Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

17.3 KB
Raw Blame
"""Observability surfaces for the interaction rollout gate: /health,
/ready, and the admin diagnostics endpoint.

/health and /ready are exercised by calling the route functions directly
(``asyncio.run`` / ``await``), the same pattern
``tests/web/test_health_degradations.py`` uses -- it avoids booting the
full ASGI lifespan (and the real startup event) just to reach two plain
async functions. The admin diagnostics endpoint is exercised the same way
``tests/web/api/test_admin_users_hidden.py`` calls its own admin routes:
directly, with a hand-built ``User``/``Session`` pair, rather than through
TestClient + JWT headers.
"""

from __future__ import annotations

from importlib import import_module

import pytest
from fastapi import HTTPException
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

import xagent.web.models.database as database_module
import xagent.web.services.interaction_rollout as ir
from tests.web.services.task_interaction_schema_shared import (
    assert_accepted,
    make_row,
    make_task,
    make_trace_event,
    make_user,
    tables_excluding_interaction_requests,
)
from xagent.web.api.admin_interaction_rollout import (
    get_interaction_rollout_diagnostics,
)
from xagent.web.models.database import Base
from xagent.web.models.user import User
from xagent.web.services.ops_signals import (
    INTERACTION_ROLLOUT_SCHEMA_ABSENT,
    INTERACTION_ROLLOUT_UNKNOWN_TASK_SOURCE,
    active_degradations,
    clear_degradation,
)

app_module = import_module("xagent.web.app")


@pytest.fixture(autouse=True)
def _reset_state(monkeypatch):
    monkeypatch.setattr(ir, "_policy", None)
    monkeypatch.setattr(ir, "_native_schema_ready", False)
    ir._counters.clear()
    clear_degradation(INTERACTION_ROLLOUT_SCHEMA_ABSENT)
    clear_degradation(INTERACTION_ROLLOUT_UNKNOWN_TASK_SOURCE)
    yield
    monkeypatch.setattr(ir, "_policy", None)
    monkeypatch.setattr(ir, "_native_schema_ready", False)
    ir._counters.clear()
    clear_degradation(INTERACTION_ROLLOUT_SCHEMA_ABSENT)
    clear_degradation(INTERACTION_ROLLOUT_UNKNOWN_TASK_SOURCE)


def _set_policy(
    monkeypatch,
    *,
    mode="legacy",
    native_sources=frozenset(),
    shared_task_execution=True,
):
    monkeypatch.setattr(
        ir,
        "_policy",
        ir.InteractionRolloutPolicy(
            mode=mode,
            native_sources=frozenset(native_sources),
            native_protocol_version=1,
            shared_task_execution=shared_task_execution,
        ),
    )


class _PoisonSessionLocal:
    def __call__(self):
        raise AssertionError("must not open a DB session at this step")


# ---------------------------------------------------------------------------
# /health surfaces the two new signal names, never their detail
# ---------------------------------------------------------------------------


async def test_to1_health_surfaces_new_signal_names_but_not_detail():
    import xagent.web.services.ops_signals as ops_signals

    ops_signals.register_degradation(
        INTERACTION_ROLLOUT_SCHEMA_ABSENT, "task_interaction_requests table absent"
    )
    ops_signals.register_degradation(
        INTERACTION_ROLLOUT_UNKNOWN_TASK_SOURCE, "task 42: unrecognized source"
    )

    payload = await app_module.health_check()

    assert payload["status"] == "ok"
    assert INTERACTION_ROLLOUT_SCHEMA_ABSENT in payload["degradations"]
    assert INTERACTION_ROLLOUT_UNKNOWN_TASK_SOURCE in payload["degradations"]
    assert "task 42" not in str(payload)
    assert "table absent" not in str(payload)


# ---------------------------------------------------------------------------
# /ready: default-mode passthrough, the native-mode schema check, and its
# one-way latch in both directions
# ---------------------------------------------------------------------------


async def test_to2_default_mode_ready_behavior_is_unchanged_zero_queries(
    monkeypatch,
):
    _set_policy(monkeypatch, mode="legacy")
    monkeypatch.setattr(database_module, "_SessionLocal", _PoisonSessionLocal())
    app_module.app.state.file_storage_startup_sync_task = None
    app_module.app.state.file_storage_startup_sync_error = None

    response = await app_module.readiness_check()

    assert response.status_code == 200


async def test_to3_native_mode_table_absent_returns_503_without_table_name(
    monkeypatch, tmp_path
):
    _set_policy(monkeypatch, mode="native", native_sources={"sdk"})
    app_module.app.state.file_storage_startup_sync_task = None
    app_module.app.state.file_storage_startup_sync_error = None

    engine = create_engine(f"sqlite:///{tmp_path / 'no_table.db'}")
    Base.metadata.create_all(
        bind=engine, tables=tables_excluding_interaction_requests()
    )
    session_local = sessionmaker(bind=engine)
    monkeypatch.setattr(database_module, "_SessionLocal", session_local)

    response = await app_module.readiness_check()

    assert response.status_code == 503
    body = response.body.decode()
    assert "Interaction rollout schema not ready" in body
    assert "task_interaction_requests" not in body


async def test_to4_native_mode_table_present_returns_200_and_clears_degradation(
    monkeypatch, tmp_path
):
    _set_policy(monkeypatch, mode="native", native_sources={"sdk"})
    app_module.app.state.file_storage_startup_sync_task = None
    app_module.app.state.file_storage_startup_sync_error = None
    import xagent.web.services.ops_signals as ops_signals

    ops_signals.register_degradation(
        INTERACTION_ROLLOUT_SCHEMA_ABSENT, "task_interaction_requests table absent"
    )

    engine = create_engine(f"sqlite:///{tmp_path / 'with_table.db'}")
    Base.metadata.create_all(bind=engine)
    session_local = sessionmaker(bind=engine)
    monkeypatch.setattr(database_module, "_SessionLocal", session_local)

    response = await app_module.readiness_check()

    assert response.status_code == 200
    assert INTERACTION_ROLLOUT_SCHEMA_ABSENT not in active_degradations()


async def test_to4b_latch_positive_means_zero_queries_on_next_probe(
    monkeypatch, tmp_path
):
    _set_policy(monkeypatch, mode="native", native_sources={"sdk"})
    app_module.app.state.file_storage_startup_sync_task = None
    app_module.app.state.file_storage_startup_sync_error = None

    engine = create_engine(f"sqlite:///{tmp_path / 'with_table.db'}")
    Base.metadata.create_all(bind=engine)
    session_local = sessionmaker(bind=engine)
    monkeypatch.setattr(database_module, "_SessionLocal", session_local)

    first = await app_module.readiness_check()
    assert first.status_code == 200
    assert ir.is_native_schema_ready() is True

    # Second probe: swap in a poison session-factory. If the latch were not
    # honored, this would raise instead of returning 200.
    monkeypatch.setattr(database_module, "_SessionLocal", _PoisonSessionLocal())
    second = await app_module.readiness_check()
    assert second.status_code == 200


async def test_to4c_latch_negative_rechecks_every_probe(monkeypatch, tmp_path):
    _set_policy(monkeypatch, mode="native", native_sources={"sdk"})
    app_module.app.state.file_storage_startup_sync_task = None
    app_module.app.state.file_storage_startup_sync_error = None

    engine = create_engine(f"sqlite:///{tmp_path / 'absent.db'}")
    Base.metadata.create_all(
        bind=engine, tables=tables_excluding_interaction_requests()
    )
    session_local = sessionmaker(bind=engine)
    monkeypatch.setattr(database_module, "_SessionLocal", session_local)

    first = await app_module.readiness_check()
    assert first.status_code == 503
    assert ir.is_native_schema_ready() is False

    # Table shows up between probes (a migration applied while the process
    # is running) -- the negative latch must re-query and see it.
    Base.metadata.create_all(bind=engine)
    second = await app_module.readiness_check()
    assert second.status_code == 200
    assert ir.is_native_schema_ready() is True


# ---------------------------------------------------------------------------
# Admin diagnostics endpoint
#
# SQLite only, deliberately: the query itself
# (SELECT count(*), min(created_at) FROM task_interaction_requests WHERE
# status = 'active' AND active_slot IS NOT NULL) uses no PostgreSQL-specific
# syntax -- no JSON operators, no ::casts, no RETURNING, nothing beyond
# ANSI count()/min()/WHERE -- so the SQL dialect itself does not vary by
# backend. What *does* vary is how each driver marshals the min(created_at)
# result back into Python: PostgreSQL hands back a tz-aware datetime,
# SQLite hands back a plain str. That difference is exactly why this suite
# includes a non-empty-table cell (test_to6b below) in addition to the
# empty-table cell (test_to6) -- an empty table never puts a value through
# min(created_at) at all, so it cannot catch a mismatch between what the
# query returns and what the age computation expects.
# ---------------------------------------------------------------------------


@pytest.fixture()
def sqlite_session_with_table(tmp_path):
    engine = create_engine(f"sqlite:///{tmp_path / 'diag_with_table.db'}")
    Base.metadata.create_all(bind=engine)
    session = sessionmaker(bind=engine)()
    yield session
    session.close()
    engine.dispose()


@pytest.fixture()
def sqlite_session_without_table(tmp_path):
    engine = create_engine(f"sqlite:///{tmp_path / 'diag_without_table.db'}")
    Base.metadata.create_all(
        bind=engine, tables=tables_excluding_interaction_requests()
    )
    session = sessionmaker(bind=engine)()
    yield session
    session.close()
    engine.dispose()


def _admin_user() -> User:
    return User(id=1, username="admin", password_hash="x", is_admin=True)


def _regular_user() -> User:
    return User(id=2, username="regular", password_hash="x", is_admin=False)


async def test_to5_non_admin_gets_403_with_exact_detail(
    monkeypatch, sqlite_session_with_table
):
    # No 401 (missing/invalid/expired credentials) cell here: this suite
    # calls the route function directly with a hand-built User, bypassing
    # the get_current_user dependency entirely (see the module docstring),
    # so there is no request to reject with 401 in the first place. That
    # dependency is shared by every admin-authenticated endpoint in this
    # codebase and its own 401 paths (missing, malformed, expired, and
    # invalid tokens) are already covered by tests/web/test_auth_dependencies.py --
    # this endpoint adds nothing to that behavior, so re-deriving it through
    # a second TestClient + JWT harness here would test the shared
    # dependency a second time, not this endpoint.
    _set_policy(monkeypatch, mode="legacy")
    with pytest.raises(HTTPException) as exc:
        await get_interaction_rollout_diagnostics(
            current_user=_regular_user(), db=sqlite_session_with_table
        )
    assert exc.value.status_code == 403
    assert exc.value.detail == "Admin access required"


async def test_to5b_admin_gets_200(monkeypatch, sqlite_session_with_table):
    _set_policy(monkeypatch, mode="legacy", shared_task_execution=False)
    response = await get_interaction_rollout_diagnostics(
        current_user=_admin_user(), db=sqlite_session_with_table
    )
    assert response["policy"]["mode"] == "legacy"
    assert response["policy"]["shared_task_execution"] is False


async def test_to6_empty_table_returns_zero_active_count_and_no_oldest_age(
    monkeypatch, sqlite_session_with_table
):
    _set_policy(monkeypatch, mode="legacy")
    response = await get_interaction_rollout_diagnostics(
        current_user=_admin_user(), db=sqlite_session_with_table
    )
    assert response["active_count"] == 0
    assert response["oldest_age_seconds"] is None
    assert "schema_absent" not in response


async def test_to6b_active_row_present_yields_positive_count_and_age(
    monkeypatch, sqlite_session_with_table
):
    """Regression cell for the SQLite str-vs-datetime marshaling gap: SQLite
    returns min(created_at) as a plain str, not a datetime, so the age
    computation must accept both shapes (see the comment above this
    section). An empty table never exercises this path.
    """
    _set_policy(monkeypatch, mode="legacy")
    user_id = make_user(sqlite_session_with_table)
    task_id = make_task(sqlite_session_with_table, user_id=user_id)
    anchor_id = make_trace_event(sqlite_session_with_table, task_id=task_id)
    assert_accepted(
        sqlite_session_with_table,
        make_row(
            task_id=task_id,
            resume_trace_event_id=anchor_id,
            db=sqlite_session_with_table,
        ),
    )

    response = await get_interaction_rollout_diagnostics(
        current_user=_admin_user(), db=sqlite_session_with_table
    )

    assert response["active_count"] == 1
    assert isinstance(response["oldest_age_seconds"], float)
    assert response["oldest_age_seconds"] > 0


async def test_to7_table_absent_returns_schema_absent_without_count_fields(
    monkeypatch, sqlite_session_without_table
):
    _set_policy(monkeypatch, mode="legacy")
    response = await get_interaction_rollout_diagnostics(
        current_user=_admin_user(), db=sqlite_session_without_table
    )
    assert response["schema_absent"] is True
    assert "active_count" not in response
    assert "oldest_age_seconds" not in response


async def test_to8_stats_query_exception_returns_503_not_a_fake_zero(
    monkeypatch, sqlite_session_with_table
):
    _set_policy(monkeypatch, mode="legacy")

    def _raise(*args, **kwargs):
        raise RuntimeError("simulated DB hiccup")

    monkeypatch.setattr(sqlite_session_with_table, "execute", _raise)

    with pytest.raises(HTTPException) as exc:
        await get_interaction_rollout_diagnostics(
            current_user=_admin_user(), db=sqlite_session_with_table
        )
    assert exc.value.status_code == 503


async def test_to8b_connection_error_before_table_check_returns_503_not_500(
    monkeypatch, sqlite_session_with_table
):
    """interaction_requests_table_exists() calls db.connection() before the
    stats query runs. A transient failure there (connection refused,
    dropped mid-request) must land in the same deliberate 503 branch as a
    failure in the stats query itself, not escape the try block and reach
    the caller as an unhandled 500.
    """
    _set_policy(monkeypatch, mode="legacy")

    def _raise(*args, **kwargs):
        raise RuntimeError("simulated connection failure")

    monkeypatch.setattr(sqlite_session_with_table, "connection", _raise)

    with pytest.raises(HTTPException) as exc:
        await get_interaction_rollout_diagnostics(
            current_user=_admin_user(), db=sqlite_session_with_table
        )
    assert exc.value.status_code == 503
    assert exc.value.detail == "Interaction rollout diagnostics stats query failed"
    assert "task_interaction_requests" not in exc.value.detail


# ---------------------------------------------------------------------------
# Counter registry snapshot isolation
# ---------------------------------------------------------------------------


def test_to9_counters_snapshot_is_a_copy_not_a_live_reference():
    ir.increment_counter(ir.COUNTER_ROLLOUT_DECISION_ALLOWED)
    snapshot = ir.counters_snapshot()
    snapshot[ir.COUNTER_ROLLOUT_DECISION_ALLOWED] = 999
    snapshot["bogus"] = 1

    fresh = ir.counters_snapshot()
    assert fresh[ir.COUNTER_ROLLOUT_DECISION_ALLOWED] == 1
    assert "bogus" not in fresh


def test_to9b_counters_snapshot_concurrent_increments_total_correctly():
    import threading

    def _bump():
        for _ in range(100):
            ir.increment_counter(ir.COUNTER_ROLLOUT_DECISION_ALLOWED)

    threads = [threading.Thread(target=_bump) for _ in range(8)]
    for t in threads:
        t.start()
    for t in threads:
        t.join()

    assert ir.counters_snapshot()[ir.COUNTER_ROLLOUT_DECISION_ALLOWED] == 800


# ---------------------------------------------------------------------------
# Unconditional startup INFO log
# ---------------------------------------------------------------------------


def test_to10_startup_logs_resolved_policy_unconditionally(monkeypatch, caplog):
    import logging

    import xagent.config as config

    monkeypatch.delenv(config.INTERACTION_PROTOCOL_MODE, raising=False)
    monkeypatch.delenv(config.INTERACTION_NATIVE_SOURCES, raising=False)
    monkeypatch.setattr(ir, "_policy", None)

    with caplog.at_level(
        logging.INFO, logger="xagent.web.services.interaction_rollout"
    ):
        ir.validate_interaction_rollout_at_startup()

    assert any(
        "Interaction rollout policy configured" in r.message for r in caplog.records
    )
    assert all(
        r.levelno == logging.INFO
        for r in caplog.records
        if "Interaction rollout policy configured" in r.message
    )


# ---------------------------------------------------------------------------
# Truncated repr of the offending Task.source in degradation detail
# ---------------------------------------------------------------------------


def test_to11_unknown_source_detail_is_repr_and_truncated_to_64_chars(
    monkeypatch, sqlite_session_with_table
):
    class _FakeTask:
        def __init__(self, source):
            self.source = source
            self.channel_id = None
            self.id = 7

    _set_policy(monkeypatch, mode="native", native_sources={"sdk"})
    long_source = "x" * 500
    ir.evaluate_native_publication(sqlite_session_with_table, _FakeTask(long_source))

    detail = active_degradations()[INTERACTION_ROLLOUT_UNKNOWN_TASK_SOURCE]
    assert "..." in detail
    # The embedded repr must be capped -- nowhere near the full 500 chars.
    assert len(detail) < 200