Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

13.0 KB
Raw Blame
"""Async LanceDB connections must be reachable from every event loop.

The stores that use them are process-wide singletons, while the codebase keeps
creating fresh event loops (``asyncio.run`` in the coordinator's sync wrappers,
the parse path, collection_manager). Guarding a per-instance ``_async_conn``
with an ``asyncio.Lock`` deadlocked that shape: the lock binds to whichever
loop first contended it, and the release path wakes that loop's future without
``call_soon_threadsafe``, so every other loop waits forever (#2200).

Every test here joins with a timeout and asserts the thread finished, so a
regression fails the suite instead of hanging CI.
"""

from __future__ import annotations

import asyncio
import os
import threading
from typing import Any, List
from unittest.mock import patch

import pytest

from xagent.providers.vector_store import lancedb as lancedb_module
from xagent.providers.vector_store.lancedb import (
    clear_connection_cache,
    get_async_connection_from_env,
)

JOIN_TIMEOUT = 5


@pytest.fixture(autouse=True)
def _isolate_pool(tmp_path, monkeypatch):
    monkeypatch.setenv("LANCEDB_DIR", str(tmp_path))
    clear_connection_cache()
    yield
    clear_connection_cache()


class _FakeAsyncConnection:
    def __init__(self) -> None:
        self.closed = False

    def close(self) -> None:
        self.closed = True


def _slow_connect_async(delay: float = 0.2):
    """A connect_async whose latency is wide enough for openers to collide."""
    created: List[_FakeAsyncConnection] = []

    async def connect_async(_uri: str) -> _FakeAsyncConnection:
        await asyncio.sleep(delay)
        conn = _FakeAsyncConnection()
        created.append(conn)
        return conn

    return connect_async, created


def _run_in_own_loop(coro_factory, results: list, errors: list) -> threading.Thread:
    def target() -> None:
        try:
            results.append(asyncio.run(coro_factory()))
        except BaseException as exc:  # noqa: BLE001
            errors.append(f"{type(exc).__name__}: {exc}")

    thread = threading.Thread(target=target, daemon=True)
    thread.start()
    return thread


def test_async_connection_installs_the_jieba_dictionary(tmp_path, monkeypatch) -> None:
    """Placed off the event loop: copying 5 MB would otherwise block it."""
    from xagent.core.tools.core.RAG_tools.LanceDB import jieba_dictionary

    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(tmp_path / "lm"))
    monkeypatch.setattr(jieba_dictionary, "_installed", False)
    connect_async, _ = _slow_connect_async(delay=0)

    with patch.object(lancedb_module.lancedb, "connect_async", connect_async):
        asyncio.run(get_async_connection_from_env())

    assert (tmp_path / "lm" / "jieba" / "default" / "dict.txt").exists()


def test_concurrent_init_from_two_loops_does_not_deadlock() -> None:
    """Two threads, two loops, one uninitialized pool entry: both must return."""
    connect_async, created = _slow_connect_async()
    results: List[Any] = []
    errors: List[str] = []

    with patch.object(lancedb_module.lancedb, "connect_async", connect_async):
        threads = [
            _run_in_own_loop(get_async_connection_from_env, results, errors)
            for _ in range(2)
        ]
        for thread in threads:
            thread.join(timeout=JOIN_TIMEOUT)

    assert not [t for t in threads if t.is_alive()], (
        "a thread never returned from async connection init -- deadlock"
    )
    assert not errors, f"async connection init raised: {errors}"
    assert len(results) == 2
    assert results[0] is results[1], "both loops must share one pooled connection"
    # Without this the test would silently pass on plain sequential reuse,
    # never having put two openers in the window at all.
    assert len(created) == 2, (
        f"the two callers did not overlap ({len(created)} connect_async calls)"
    )
    # The opener that lost the insert race must not leak.
    leaked = [c for c in created if c is not results[0] and not c.closed]
    assert not leaked, f"{len(leaked)} superseded connections were left open"


def test_many_loops_reuse_one_pooled_connection() -> None:
    """A cached connection is handed to loops that had no part in creating it."""
    connect_async, created = _slow_connect_async(delay=0.0)
    results: List[Any] = []
    errors: List[str] = []

    with patch.object(lancedb_module.lancedb, "connect_async", connect_async):
        for _ in range(6):
            thread = _run_in_own_loop(get_async_connection_from_env, results, errors)
            thread.join(timeout=JOIN_TIMEOUT)
            assert not thread.is_alive()

    assert not errors, f"async connection init raised: {errors}"
    assert len(results) == 6
    assert all(conn is results[0] for conn in results)
    assert len(created) == 1, f"connect_async ran {len(created)} times, expected 1"


def test_clear_connection_cache_forces_a_fresh_async_connection() -> None:
    """``reset_rag_storage_for_tests`` relies on this to reset *all* state.

    Before the pool existed, the async connection went away with the store
    instance that `StorageFactory.reset_all()` dropped; now it outlives that,
    so clearing the cache is what forces the next connect. The pooled
    connection is deliberately *not* closed -- stores hold theirs across
    awaits, so closing here would strand an in-flight coroutine.
    """
    connect_async, created = _slow_connect_async(delay=0.0)
    results: List[Any] = []
    errors: List[str] = []

    with patch.object(lancedb_module.lancedb, "connect_async", connect_async):
        thread = _run_in_own_loop(get_async_connection_from_env, results, errors)
        thread.join(timeout=JOIN_TIMEOUT)
        assert not thread.is_alive()
        assert not errors and len(results) == 1

        clear_connection_cache()
        assert not created[0].closed, "a pooled connection must not be closed"

        thread = _run_in_own_loop(get_async_connection_from_env, results, errors)
        thread.join(timeout=JOIN_TIMEOUT)
        assert not thread.is_alive()

    assert not errors, f"async connection init raised: {errors}"
    assert len(created) == 2, "cache clear must force a fresh connect_async"
    assert results[1] is not results[0]


def test_store_async_methods_share_the_pool_across_loops() -> None:
    """The shape #2200 actually breaks: one store singleton, several loops."""
    from xagent.core.tools.core.RAG_tools.storage.lancedb_stores import (
        LanceDBVectorIndexStore,
    )

    connect_async, created = _slow_connect_async()
    results: List[Any] = []
    errors: List[str] = []
    first = LanceDBVectorIndexStore()
    second = LanceDBVectorIndexStore()

    with patch.object(lancedb_module.lancedb, "connect_async", connect_async):
        threads = [
            _run_in_own_loop(store._get_async_connection, results, errors)
            for store in (first, second, first)
        ]
        for thread in threads:
            thread.join(timeout=JOIN_TIMEOUT)

    assert not [t for t in threads if t.is_alive()], (
        "a store's async connection init never returned -- deadlock"
    )
    assert not errors, f"store async init raised: {errors}"
    assert len(results) == 3
    assert all(conn is results[0] for conn in results)
    assert not hasattr(first, "_async_lock")
    assert not hasattr(second, "_async_lock")


def test_real_async_connection_survives_a_second_event_loop() -> None:
    """The premise the pool rests on, checked against real lancedb.

    Every other test here fakes ``connect_async``, so none of them would notice
    if a lancedb upgrade started binding connections to their creating loop.
    This one does the crossing for real: build and write on one loop, then read
    and write the same connection from another after the first is gone.
    """
    import lancedb

    holder: dict = {}
    errors: List[str] = []

    async def create() -> int:
        conn = await lancedb.connect_async(os.environ["LANCEDB_DIR"])
        table = await conn.create_table("crossloop", data=[{"x": 1}])
        holder["conn"] = conn
        return await table.count_rows()

    async def reuse() -> int:
        table = await holder["conn"].open_table("crossloop")
        await table.add([{"x": 2}])
        return await table.count_rows()

    first: List[Any] = []
    thread = _run_in_own_loop(create, first, errors)
    thread.join(timeout=JOIN_TIMEOUT)
    assert not thread.is_alive()
    assert not errors, f"creating loop raised: {errors}"

    second: List[Any] = []
    thread = _run_in_own_loop(reuse, second, errors)
    thread.join(timeout=JOIN_TIMEOUT)
    assert not thread.is_alive(), "reusing the connection on a second loop hung"
    assert not errors, f"second loop raised: {errors}"
    assert first == [1] and second == [2]


def test_no_await_inside_the_pool_lock() -> None:
    """``_async_cache_lock`` is a threading lock, so awaiting under it hangs.

    A regression here cannot be caught at runtime: the blocked thread stops
    the event loop too, so ``asyncio.wait_for`` never gets to fire and the
    suite deadlocks instead of failing. Hence the static check.
    """
    import ast
    import inspect
    import textwrap

    tree = ast.parse(textwrap.dedent(inspect.getsource(get_async_connection_from_env)))

    offenders = []
    for node in ast.walk(tree):
        if not isinstance(node, (ast.With, ast.AsyncWith)):
            continue
        guards = {ast.unparse(item.context_expr) for item in node.items}
        if "_async_cache_lock" not in guards:
            continue
        for inner in ast.walk(node):
            if isinstance(inner, (ast.Await, ast.AsyncWith, ast.AsyncFor)):
                offenders.append(f"line {inner.lineno}: {ast.unparse(inner)[:60]}")

    assert not offenders, (
        "await under the threading lock would deadlock the loop: "
        + "; ".join(offenders)
    )


@pytest.mark.asyncio
async def test_concurrent_init_on_one_loop_shares_one_connection() -> None:
    """Gathered callers on a single loop converge on one pooled connection."""
    connect_async, created = _slow_connect_async(delay=0.2)

    with patch.object(lancedb_module.lancedb, "connect_async", connect_async):
        conns = await asyncio.wait_for(
            asyncio.gather(*(get_async_connection_from_env() for _ in range(5))),
            timeout=JOIN_TIMEOUT,
        )

    assert all(conn is conns[0] for conn in conns)
    survivors = [c for c in created if c is conns[0] or not c.closed]
    assert survivors == [conns[0]], "every superseded connection must be closed"


def test_failed_connect_leaves_the_pool_untouched() -> None:
    """A failed connect must propagate and cache nothing."""

    async def failing_connect(_uri: str) -> Any:
        raise RuntimeError("connect refused")

    results: List[Any] = []
    errors: List[str] = []

    with patch.object(lancedb_module.lancedb, "connect_async", failing_connect):
        thread = _run_in_own_loop(get_async_connection_from_env, results, errors)
        thread.join(timeout=JOIN_TIMEOUT)

    assert not thread.is_alive()
    assert errors == ["RuntimeError: connect refused"]
    assert not results
    assert not lancedb_module._async_connection_cache, "a failure must not cache"

    # The pool still works afterwards.
    connect_async, created = _slow_connect_async(delay=0.0)
    with patch.object(lancedb_module.lancedb, "connect_async", connect_async):
        thread = _run_in_own_loop(get_async_connection_from_env, results, errors)
        thread.join(timeout=JOIN_TIMEOUT)
    assert not thread.is_alive()
    assert len(results) == 1 and len(created) == 1


@pytest.mark.parametrize("env_value", [None, "", "   "])
def test_both_planes_agree_on_the_directory_without_the_env_var(
    env_value, monkeypatch
) -> None:
    """Sync and async must resolve the same dir, including with no env var.

    This is where the two planes used to diverge: the async side read
    ``LANCEDB_DIR`` directly with its own default while the sync side went
    through ``get_default_lancedb_dir()``. Every other test here sets the
    variable, so the unset and empty cases would otherwise go unchecked.
    """
    from xagent.providers.vector_store.lancedb import LanceDBConnectionManager

    if env_value is None:
        monkeypatch.delenv("LANCEDB_DIR", raising=False)
    else:
        monkeypatch.setenv("LANCEDB_DIR", env_value)

    manager = LanceDBConnectionManager()
    if env_value is not None and env_value.strip() == "":
        with pytest.raises(ValueError, match="is empty"):
            manager.resolve_dir_from_env()
        return

    resolved = manager.resolve_dir_from_env()
    assert resolved == manager._normalize_dirpath(manager.get_default_lancedb_dir())

    connect_async, created = _slow_connect_async(delay=0.0)
    results: List[Any] = []
    errors: List[str] = []
    with patch.object(lancedb_module.lancedb, "connect_async", connect_async):
        thread = _run_in_own_loop(get_async_connection_from_env, results, errors)
        thread.join(timeout=JOIN_TIMEOUT)

    assert not errors, f"async init raised: {errors}"
    assert list(lancedb_module._async_connection_cache) == [resolved], (
        "the async pool must key on the same dir the sync plane resolves"
    )