Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

20.5 KB
Raw Blame
from __future__ import annotations

import asyncio
import contextlib
import uuid
import warnings
from collections.abc import AsyncIterator, Iterator
from dataclasses import dataclass

import pytest
import pytest_asyncio
from sqlalchemy import insert, text
from sqlalchemy.engine import make_url
from sqlalchemy.exc import DBAPIError
from sqlalchemy.ext.asyncio import (
    AsyncConnection,
    AsyncEngine,
    AsyncSession,
    async_sessionmaker,
    create_async_engine,
)
from sqlalchemy.pool import NullPool
from testcontainers.postgres import PostgresContainer

# Importing the models module registers every table on Base.metadata so
# create_all provisions the full schema (rooms, room_groups, FKs, …).
import switch_core.db.models  # noqa: F401
from switch_core import messaging_platforms
from switch_core.bridges.agent.protocol.event_buffer import EventBuffer
from switch_core.bridges.collaboration.platforms import PLATFORMS
from switch_core.db import encrypted_json
from switch_core.db.base import Base
from switch_core.db.engine import create_session_factory
from switch_core.db.models import TENANT_ZERO_ID, Organization, Tenant
from switch_core.db.runtime_role import grant_runtime_role
from switch_core.keys import Keyring
from switch_core.messages.notify import MessageListener
from switch_core.tenant_context import tenant_scope
from tests.db_buffer import db_buffer_on
from tests.switch_core.pool_checkouts import PoolCheckouts
from tests.switch_core.statement_counts import StatementCounts

# Production configures this in `main.run()`; every test that touches a
# connection config needs the same, and none needs a particular key.
TEST_KEYRING = Keyring.parse(
    "test:" + "column-encryption-secret" * 2, legacy_secret=None
)
encrypted_json.configure(TEST_KEYRING)

# Production publishes these when `main.run()` registers its adapters. Tests
# that read telemetry or render cards without building a lifecycle need the
# same platforms to exist.
for _platform in PLATFORMS:
    messaging_platforms.register(_platform.adapter.describe(_platform.key))


@pytest.fixture(autouse=True)
def _restore_messaging_platforms(monkeypatch: pytest.MonkeyPatch) -> None:
    """Undo whatever a test registers. Many register a stub adapter under a real
    platform's key, which would otherwise rename that platform for every test
    that runs after it."""
    monkeypatch.setattr(
        messaging_platforms, "_REGISTERED", dict(messaging_platforms._REGISTERED)
    )


async def _seed_tenant_zero(conn: AsyncConnection) -> None:
    """Insert the one tenant every scoped row's `tenant_id` default points at.

    Every scoped table's `tenant_id` FK requires a matching `tenants` row, so
    this must run right after the schema exists. It belongs to an
    organization of its own, as the migration gives every existing workspace
    one; the organization reuses the tenant's id.
    """
    await conn.execute(
        insert(Organization.__table__).values(
            id=TENANT_ZERO_ID, slug="default", name="Default"
        )
    )
    await conn.execute(
        insert(Tenant.__table__).values(
            id=TENANT_ZERO_ID,
            slug="default",
            name="Default",
            organization_id=TENANT_ZERO_ID,
        )
    )


# Every table, emptied in one statement. `CASCADE` is what lets a single
# TRUNCATE cover tables that reference each other, and `RESTART IDENTITY`
# keeps sequence-backed columns (`messages.seq`) starting where a test that
# hard-codes an expected value needs them to.
_TRUNCATE_ALL = (
    "TRUNCATE "
    + ", ".join(f'"{table.name}"' for table in Base.metadata.sorted_tables)
    + " RESTART IDENTITY CASCADE"
)


# How long the reset waits for its locks before ending what an earlier test
# left holding them, and then how long it waits before giving up.
_TRUNCATE_LOCK_TIMEOUT = "10s"
_TRUNCATE_LOCK_TIMEOUT_AFTER_CLEARING = "60s"

_LOCK_NOT_AVAILABLE = "55P03"

# Every other session on this database, for a reset that could not take its
# locks: the ones it ends, or the ones that kept it waiting.
_OTHER_SESSIONS = (
    "SELECT pid, state, now() - xact_start AS in_transaction_for, "
    "left(query, 300) AS last_query "
    "FROM pg_stat_activity "
    "WHERE datname = current_database() AND pid <> pg_backend_pid()"
)

_END_LEFT_IN_TRANSACTION = text(
    f"SELECT *, pg_terminate_backend(pid) FROM ({_OTHER_SESSIONS}) AS other "
    "WHERE state LIKE 'idle in transaction%'"
)


async def _truncate(engine: AsyncEngine, lock_timeout: str) -> bool:
    """Empty the database, waiting at most `lock_timeout` for the locks.
    Returns whether it got them."""
    try:
        async with engine.begin() as conn:
            await conn.execute(text(f"SET LOCAL lock_timeout = '{lock_timeout}'"))
            await conn.execute(text(_TRUNCATE_ALL))
            await _seed_tenant_zero(conn)
    except DBAPIError as exc:
        if getattr(exc.orig, "sqlstate", None) != _LOCK_NOT_AVAILABLE:
            raise
        return False
    return True


async def empty_the_database(engine: AsyncEngine) -> None:
    """Leave the schema in place and take every row out of it, tenant zero apart.

    The state a test wants from `create_all` is an empty database, which
    building the schema from nothing is only one way to reach. Emptying it
    instead is the same starting point for a fraction of the DDL, and it is
    what every per-test fixture here uses.

    TRUNCATE takes every table's exclusive lock, so a connection left in a
    transaction would make it wait forever. One is left when a test's client
    loop (a `TestClient`'s thread, say) stops while a query of its stream is
    in flight: the task is abandoned rather than cancelled, and its
    transaction never ends. Each test worker has a database of its own and
    runs one test at a time, so a session idle in a transaction while a test
    is still being set up belongs to a test that has finished. The reset ends
    those, with a warning naming them, and carries on; and fails, naming every
    session still connected, if it then still cannot take its locks.
    """
    if await _truncate(engine, _TRUNCATE_LOCK_TIMEOUT):
        return
    async with engine.connect() as conn:
        ended = (await conn.execute(_END_LEFT_IN_TRANSACTION)).mappings().all()
    if ended:
        warnings.warn(
            "emptying the database ended sessions an earlier test left in a "
            "transaction:\n" + "\n".join(str(dict(session)) for session in ended),
            stacklevel=2,
        )
    if await _truncate(engine, _TRUNCATE_LOCK_TIMEOUT_AFTER_CLEARING):
        return
    async with engine.connect() as conn:
        sessions = (await conn.execute(text(_OTHER_SESSIONS))).mappings().all()
    raise AssertionError(
        "emptying the database could not take its locks. Still connected:\n"
        + "\n".join(str(dict(session)) for session in sessions)
    )


@pytest.fixture(scope="session")
def postgres_url() -> Iterator[str]:
    """A throwaway PostgreSQL instance for the whole test session.

    Store tests run against real Postgres (not SQLite/mocks) so behaviours like
    `ON DELETE SET NULL` and check constraints are exercised for real.

    Durability is off. The container is deleted when the session ends, so a
    crash has nothing to recover to and every fsync it would perform is work
    thrown away — which matters here because the per-test reset is almost
    entirely write traffic. Nothing about the behaviour under test changes:
    these settings govern what survives a crash, not what a transaction sees.
    """
    container = PostgresContainer("postgres:16-alpine").with_command(
        "postgres -c fsync=off -c synchronous_commit=off -c full_page_writes=off"
    )
    with container as pg:
        host = pg.get_container_host_ip()
        port = pg.get_exposed_port(5432)
        yield (
            f"postgresql+asyncpg://{pg.username}:{pg.password}"
            f"@{host}:{port}/{pg.dbname}"
        )


@pytest.fixture(scope="session")
def postgres_schema(postgres_url: str) -> str:
    """The schema, built once for the whole session.

    Returns `postgres_url`: it is the same database, and asking for this
    fixture rather than that one is how a test says it wants the tables there.
    The tests that want a schema of their own — migration parity, the backfill
    migrations, the frozen-DDL comparison — create a separate database off
    `postgres_url` and are untouched by this.

    Synchronous, with an event loop of its own, because it has to outlive the
    per-test loop pytest-asyncio opens: the schema belongs to the container,
    not to any one test's loop, and the engine that builds it is disposed
    before the first test runs.
    """

    async def build() -> None:
        engine = create_async_engine(postgres_url)
        try:
            async with engine.begin() as conn:
                await conn.run_sync(Base.metadata.create_all)
        finally:
            await engine.dispose()

    asyncio.run(build())
    return postgres_url


@pytest_asyncio.fixture
async def session_factory(
    postgres_schema: str, request: pytest.FixtureRequest
) -> AsyncIterator[async_sessionmaker[AsyncSession]]:
    """An empty database per test, on a schema built once for the session.

    Emptied on the way in rather than the way out, so a test that fails leaves
    its rows in the container for a `psql` to look at, and so the guarantee
    holds for the first test as much as for the rest.

    Also binds tenant zero for the duration of the test, unless the test is
    marked `no_ambient_tenant`. In production a tenant is always bound before
    store code runs — an authenticated request, or one of the background call
    sites `db/session_scope.py` covers — so a store method is never called
    with nothing bound outside a deliberately unscoped path. Most of this
    suite constructs scoped rows directly rather than through a request,
    though, so without this default they would be calling those methods with
    nothing bound — not a scenario the design means to exercise, and not what
    any of these tests are about.

    The marker exists for the tests that *are* about exactly that: a test
    asserting what is or is not bound before, after or around a call — the
    tenant-context machinery itself, not a store — needs `current_tenant_id()`
    to start `None`, and this default would otherwise stand in its way. A test
    doing ordinary multi-tenant work instead binds its own (`tenant_scope`,
    `tenant_session`, `no_tenant`) over the part it cares about, which simply
    overrides this default for that block — only the "nothing bound at all,
    including before I do anything" assertions need the marker.
    """
    engine = create_async_engine(postgres_schema)
    await empty_the_database(engine)
    ambient_tenant = (
        contextlib.nullcontext()
        if request.node.get_closest_marker("no_ambient_tenant")
        else tenant_scope(TENANT_ZERO_ID)
    )
    try:
        with ambient_tenant:
            # Goes through the same factory constructor production wiring
            # uses, so this fixture — which backs most of the store test
            # suite — differs from production in as little as possible.
            yield create_session_factory(engine)
    finally:
        await engine.dispose()


@pytest.fixture
def pool_checkouts(
    session_factory: async_sessionmaker[AsyncSession],
) -> Iterator[PoolCheckouts]:
    """Connections borrowed from `session_factory`'s pool, counted per request.

    Wrap an app under test with `pool_checkouts.wrap(app)`, or a block with
    `pool_checkouts.scope(label)`, and assert `pool_checkouts.over() == []`:
    no request held two connections at once. See `pool_checkouts.py`.
    """
    tracker = PoolCheckouts(session_factory.kw["bind"])
    try:
        yield tracker
    finally:
        tracker.close()


@pytest.fixture
def statement_counts(
    session_factory: async_sessionmaker[AsyncSession],
) -> Iterator[StatementCounts]:
    """SQL statements run against `session_factory`'s engine, counted per
    request. Wrap an app under test with `statement_counts.wrap(app)`; see
    `statement_counts.py`."""
    counts = StatementCounts(session_factory.kw["bind"])
    try:
        yield counts
    finally:
        counts.close()


@dataclass(frozen=True)
class RLSHarness:
    """Two ways into the same database, for row-level-security tests that
    need both (CHOO-2623).

    `owner` connects the same way `session_factory` above does — as the
    container's superuser, which bypasses every policy by ownership, the same
    hatch a real system session relies on (see the design doc's "bootstrap
    problem" section). Test setup uses it to create cross-tenant fixture data
    no tenant-scoped session could ever legitimately write: a new `tenants`
    row, for one, since its own policy compares on `id` — inserting tenant B
    while bound to tenant A can never satisfy `with check`, and inserting it
    unscoped can never satisfy `using` either, because creating a tenant is
    necessarily a system operation.

    `restricted` is the role the policies actually apply to: every assertion
    about isolation is made through it, never through `owner` or the plain
    `session_factory` fixture, both of which would pass regardless of whether
    a single policy in `db/rls_ddl.py` exists.
    """

    owner: async_sessionmaker[AsyncSession]
    restricted: async_sessionmaker[AsyncSession]
    # The engines behind the two factories, for the boot self-check
    # (`db/runtime_role.verify_restricted_role`), which is asked of a
    # connection rather than of a session. Exposed rather than rebuilt in the
    # test so that what it inspects is the connection the assertions above
    # were made through.
    owner_engine: AsyncEngine
    restricted_engine: AsyncEngine


@pytest_asyncio.fixture
async def rls_harness(postgres_schema: str) -> AsyncIterator[RLSHarness]:
    """A session factory connected as a role row-level security applies to.

    `session_factory` connects as the container's superuser — the same shape
    local Compose, the chart and the test containers all use today — and **a
    superuser ignores row-level security unconditionally**. A test built only
    on that fixture cannot tell a working policy from a schema with none, so
    `docs/old/multi-tenancy-phase1-db.md`'s "Done when" is unreachable without
    something else.

    This is that something else. Per test, it empties the session's schema
    (built by `create_all`, so it carries the same policies a migration would
    attach, exactly as `session_factory` sees), and creates a throwaway login
    role that is not the owner (so it doesn't inherit the owner's blanket
    exemption from its own tables' policies), not a superuser, and carries no
    `BYPASSRLS`.

    **It is granted by `db/runtime_role.grant_runtime_role`, the same function
    boot runs against a real deployment**, rather than by a second list of
    grants written out here. That is deliberate: a fixture that granted more
    than production does would let a test pass on a privilege the deployment
    has not got, and one that granted less would fail for a reason the
    deployment never hits. Sharing the definition makes "the restricted role
    can do its job" a property this suite actually proves.

    Only the tests that exist to prove the policies correct ask for this
    fixture — the rest of the suite keeps using `session_factory` exactly as
    it does today, connected as the owner. What this proves and what it does
    not is worth stating: it proves the *policies* and the *exemption* are
    correct against a role they apply to. Whether the deployment connects as
    such a role is a separate question, asked at boot by
    `db/runtime_role.verify_restricted_role` rather than here, because CI has
    no production connection to ask it of.
    """
    owner_engine = create_async_engine(postgres_schema)
    role = f"switch_rls_test_{uuid.uuid4().hex[:12]}"
    password = uuid.uuid4().hex

    await empty_the_database(owner_engine)
    async with owner_engine.begin() as conn:
        # CREATE ROLE does not accept a bind parameter for PASSWORD (it is not
        # a normal DML statement), so the password is interpolated directly.
        # It is a fresh, random, throwaway value scoped to this one test.
        await conn.execute(
            text(
                f"CREATE ROLE \"{role}\" LOGIN PASSWORD '{password}' "
                "NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS"
            )
        )
        await grant_runtime_role(conn, role)

    restricted_url = make_url(postgres_schema).set(username=role, password=password)
    restricted_engine = create_async_engine(restricted_url)
    try:
        yield RLSHarness(
            owner=create_session_factory(owner_engine),
            restricted=create_session_factory(restricted_engine),
            owner_engine=owner_engine,
            restricted_engine=restricted_engine,
        )
    finally:
        await restricted_engine.dispose()
        async with owner_engine.begin() as conn:
            # The tables outlive the test, so their grants to this role would
            # too. `DROP OWNED BY` revokes every privilege still held by the
            # role — the table grants and the schema-level `GRANT USAGE` alike
            # (it owns no objects, so there is nothing for it to drop) — which
            # is both what leaves the shared schema as it was found and what
            # lets `DROP ROLE` succeed right after.
            await conn.execute(text(f'DROP OWNED BY "{role}"'))
            await conn.execute(text(f'DROP ROLE "{role}"'))
            # `grant_runtime_role` also revokes PUBLIC's default EXECUTE on
            # functions, and that one is recorded against the *owner* rather
            # than against the throwaway role — so `DROP OWNED BY` above does
            # not touch it and it outlives this test, governing every function
            # every later test in the session creates. Left in place it makes
            # the checks about it pass for the wrong reason: the second test
            # to ask whether a new function is PUBLIC-executable would be
            # answered by the first test's leftover rather than by the grants
            # under test. Granting it back restores the built-in default,
            # which Postgres records by deleting the row.
            await conn.execute(
                text("ALTER DEFAULT PRIVILEGES GRANT EXECUTE ON FUNCTIONS TO PUBLIC")
            )
            # The same argument, for the schema itself rather than the default
            # privileges on it. The tests this fixture exists for break the
            # schema on purpose — forcing row level security, granting EXECUTE
            # back to PUBLIC — to watch the boot check refuse it, and the
            # schema now outlives them. Put it back the way `create_all`
            # leaves it: nothing forced, and PUBLIC holding the EXECUTE
            # Postgres grants on a new function. `grant_runtime_role` above
            # revokes that again for every test, so what the assertions run
            # against is unchanged.
            forced = (
                (
                    await conn.execute(
                        text(
                            "SELECT relname FROM pg_class c "
                            "JOIN pg_namespace n ON n.oid = c.relnamespace "
                            "WHERE c.relforcerowsecurity AND n.nspname = 'public'"
                        )
                    )
                )
                .scalars()
                .all()
            )
            for table in forced:
                await conn.execute(
                    text(f'ALTER TABLE "{table}" NO FORCE ROW LEVEL SECURITY')
                )
            await conn.execute(
                text("GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO PUBLIC")
            )
        await owner_engine.dispose()


@pytest.fixture
def db_buffer(session_factory: async_sessionmaker[AsyncSession]) -> EventBuffer:
    """The db-buffer on this test's database. Seed the agents it queues for
    with `tests.db_buffer.seed_agent`."""
    return db_buffer_on(session_factory)


@pytest_asyncio.fixture
async def room_bells(postgres_schema: str) -> AsyncIterator[MessageListener]:
    """The room announcements of this test's database, listened to: for a
    test whose stream reads a room's conversation and has to be woken when a
    row is written there (`db_buffer_on(..., rooms=room_bells)`)."""
    listener = MessageListener(
        lambda: create_async_engine(postgres_schema, poolclass=NullPool)
    )
    await listener.start()
    try:
        await asyncio.wait_for(listener.connected.wait(), 10.0)
        yield listener
    finally:
        await listener.stop()