Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

24.3 KB
Raw Blame
from __future__ import annotations

import hashlib
import json
import logging
from collections import deque
from functools import partial
from pathlib import Path
from types import FunctionType, MethodType, ModuleType, SimpleNamespace
from typing import Any
from uuid import uuid4

import pytest
from sqlalchemy import text
from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError
from sqlalchemy.orm import Session
from sqlalchemy.pool import QueuePool

from tests.e2e.app_harness import (
    create_e2e_user,
    disable_external_app_services,
    init_e2e_db,
    run_e2e_app_client,
)
from xagent.core.agent.service import AgentService
from xagent.core.tools.adapters.vibe.factory import ToolFactory
from xagent.core.tools.adapters.vibe.selection_spec import ToolSelectionSpec
from xagent.web.models.database import get_engine
from xagent.web.models.skill import UserSkill, UserSkillFile
from xagent.web.models.user import User
from xagent.web.tools.config import WebToolConfig

pytestmark = [pytest.mark.e2e, pytest.mark.docker]

SKILL_INDEX_SENTINEL = "Pool handoff regression fixture"
QUESTION = "Which deployment target should I use?"
# Mirrors interaction_types.DEFAULT_WAITING_INTERACTION as a literal on
# purpose: a change to the published shape must fail here, not follow the
# source.
DEFAULT_WAITING_FIELD = {
    "type": "text_input",
    "field": "response",
    "label": "Your response",
    "placeholder": "Type your answer",
    "multiline": True,
}
EXPECTED_SKILL_INDEX_LINE = (
    "- session-safe: Pool handoff regression fixture "
    "When to use: Test authenticated Skill database reads"
)
EXPECTED_LOAD_SKILL_PARAMETERS = {
    "properties": {
        "skill_name": {
            "description": (
                "Exact name of the skill to load, as listed in the skill index."
            ),
            "type": "string",
        }
    },
    "required": ["skill_name"],
    "type": "object",
}
AVAILABLE_TOOLS_MODEL_GETTERS = (
    "get_vision_model",
    "get_image_models",
    "get_video_models",
    "get_asr_models",
    "get_tts_models",
    "get_sound_effect_models",
    "get_music_models",
)


def _configure_postgres_app(
    monkeypatch: pytest.MonkeyPatch,
    *,
    tmp_path: Path,
    postgres_url: str,
) -> None:
    monkeypatch.setenv("DATABASE_URL", postgres_url)
    # These tests measure request/service checkouts, not worker polling.
    # A combined host dispatcher can borrow the only slot after a response
    # and make the zero-checkout assertions fail despite correct cleanup.
    monkeypatch.setenv("XAGENT_TASK_EXECUTION_ROLE", "web")
    monkeypatch.setenv("XAGENT_DB_POOL_SIZE", "1")
    monkeypatch.setenv("XAGENT_DB_MAX_OVERFLOW", "0")
    monkeypatch.setenv("XAGENT_DB_POOL_TIMEOUT_SECONDS", "1")
    monkeypatch.setenv("XAGENT_FILE_STORAGE_STARTUP_SYNC_ENABLED", "false")
    monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(tmp_path / "uploads"))
    monkeypatch.setenv("XAGENT_FILE_MATERIALIZE_DIR", str(tmp_path / "materialized"))
    monkeypatch.setenv("LANCEDB_DIR", str(tmp_path / "lancedb"))
    monkeypatch.setenv("LANCEDB_PATH", str(tmp_path / "lancedb-path"))
    monkeypatch.setenv("LANCEDB_AUTO_MIGRATE", "false")


def _seed_personal_skill(db: Any, *, user_id: int) -> str:
    name = "session-safe"
    content = b"""---
name: session-safe
description: Pool handoff regression fixture
when_to_use: Test authenticated Skill database reads
---

# Session Safe
"""
    skill = UserSkill(
        user_id=user_id,
        name=name,
        origin="custom",
        created_by_user_id=user_id,
        updated_by_user_id=user_id,
    )
    skill.files.append(
        UserSkillFile(
            path="SKILL.md",
            content=content,
            size_bytes=len(content),
            sha256=hashlib.sha256(content).hexdigest(),
            media_type="text/markdown",
        )
    )
    db.add(skill)
    db.commit()
    return name


class _WaitingLLM:
    model_name = "waiting-regression"
    context_window = 32_000

    def __init__(self, *, call_id: str) -> None:
        self.call_id = call_id
        self.calls: list[dict[str, Any]] = []

    async def chat(self, **kwargs: Any) -> dict[str, Any]:
        self.calls.append(kwargs)
        return {
            "content": "I need one required choice.",
            "tool_calls": [
                {
                    "id": self.call_id,
                    "function": {
                        "name": "ask_user_question",
                        "arguments": json.dumps(
                            {
                                "message": QUESTION,
                                "interactions": [],
                            }
                        ),
                    },
                }
            ],
        }


def _owned_references(value: object) -> list[object]:
    if value is None or type(value) in {str, bytes, int, float, bool}:
        return []
    if isinstance(value, ModuleType | type):
        return []
    if isinstance(value, dict):
        return [*value.keys(), *value.values()]
    if isinstance(value, list | tuple | set | frozenset | deque):
        return list(value)
    if isinstance(value, partial):
        return [value.func, *value.args, *(value.keywords or {}).values()]
    if isinstance(value, MethodType):
        return [value.__self__, value.__func__]
    if isinstance(value, FunctionType):
        references: list[object] = [
            *(value.__defaults__ or ()),
            *(value.__kwdefaults__ or {}).values(),
        ]
        for cell in value.__closure__ or ():
            try:
                references.append(cell.cell_contents)
            except ValueError:
                pass
        return references
    try:
        return list(vars(value).values())
    except TypeError:
        return []


def _assert_resources_not_reachable(
    *,
    roots: dict[str, object],
    forbidden: dict[str, object],
) -> None:
    forbidden_by_id = {id(value): name for name, value in forbidden.items()}
    queue = deque((name, value) for name, value in roots.items())
    seen: set[int] = set()
    while queue:
        path, value = queue.popleft()
        value_id = id(value)
        if value_id in seen:
            continue
        seen.add(value_id)
        forbidden_name = forbidden_by_id.get(value_id)
        assert forbidden_name is None, f"{path} retains caller {forbidden_name}"
        for index, reference in enumerate(_owned_references(value)):
            queue.append((f"{path}[{index}]", reference))


def _assert_detached_retained_service(
    *,
    caller_db: Session,
    request: object,
    user: User,
    user_id: int,
    config: WebToolConfig,
    service: AgentService,
) -> None:
    assert get_engine().pool.checkedout() == 0, "verified factory handoff leaked"
    assert config is service.tool_config
    assert config._live_db is None
    assert config._lazy_db is None
    assert config.request is None
    assert config._user is None
    assert service.skill_scope_context.user_id == user_id
    assert not hasattr(service.skill_scope_context, "db")
    assert not hasattr(service.skill_scope_context, "request")
    assert not hasattr(service.skill_scope_context, "user")
    assert caller_db is not config._live_db
    assert request is not config.request
    assert user is not config._user
    _assert_resources_not_reachable(
        roots={
            "tools": service.tools,
            "config": config,
            "service": service,
        },
        forbidden={
            "Session": caller_db,
            "request": request,
            "ORM User": user,
        },
    )


def test_authenticated_skill_routes_handoff_one_slot_postgres_pool(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    postgres_url: str,
) -> None:
    _configure_postgres_app(
        monkeypatch,
        tmp_path=tmp_path,
        postgres_url=postgres_url,
    )
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()
    with SessionLocal() as db:
        user = create_e2e_user(db, username="skill-runtime-user")
        skill_name = _seed_personal_skill(db, user_id=user.id)

    with run_e2e_app_client(
        monkeypatch,
        username=user.username,
        user_id=user.id,
    ) as app:
        skills_response = app.client.get("/api/skills/", headers=app.headers)
        assert skills_response.status_code == 200, skills_response.text
        assert skill_name in {item["name"] for item in skills_response.json()}
        assert get_engine().pool.checkedout() == 0

        installed_response = app.client.get(
            "/api/skill-hub/installed",
            headers=app.headers,
        )
        assert installed_response.status_code == 200, installed_response.text
        assert skill_name in {item["name"] for item in installed_response.json()}
        assert get_engine().pool.checkedout() == 0


def test_available_tools_route_uses_retained_models_before_request_db_reads(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    postgres_url: str,
    caplog: pytest.LogCaptureFixture,
) -> None:
    _configure_postgres_app(
        monkeypatch,
        tmp_path=tmp_path,
        postgres_url=postgres_url,
    )
    disable_external_app_services(monkeypatch)
    session_factory = init_e2e_db()
    with session_factory() as seed_db:
        seeded_user = create_e2e_user(
            seed_db,
            username="available-tools-pool-user",
        )

    engine = get_engine()
    pool = engine.pool
    assert engine.dialect.name == "postgresql"
    assert isinstance(pool, QueuePool)
    assert pool.size() == 1
    assert pool._max_overflow == 0  # noqa: SLF001
    assert pool.timeout() == 1
    assert pool.checkedout() == 0

    getter_call_counts: dict[int, dict[str, int]] = {}
    factory_returns: list[tuple[WebToolConfig, bool, bool, dict[str, int]]] = []
    real_getters = {
        name: getattr(WebToolConfig, name) for name in AVAILABLE_TOOLS_MODEL_GETTERS
    }

    def _getter_spy(name: str, real_getter: Any) -> Any:
        def _delegate(config: WebToolConfig) -> Any:
            counts = getter_call_counts.setdefault(id(config), {})
            counts[name] = counts.get(name, 0) + 1
            return real_getter(config)

        return _delegate

    for getter_name, real_getter in real_getters.items():
        monkeypatch.setattr(
            WebToolConfig,
            getter_name,
            _getter_spy(getter_name, real_getter),
        )

    real_create_all_tools = ToolFactory.create_all_tools

    async def _factory_spy(
        config: WebToolConfig,
        apply_user_override_filter: bool = True,
    ) -> list[Any]:
        tools = await real_create_all_tools(
            config,
            apply_user_override_filter=apply_user_override_filter,
        )
        factory_returns.append(
            (
                config,
                config._retained_factory_model_state is not None,
                config._factory_runtime_handed_off,
                dict(getter_call_counts.get(id(config), {})),
            )
        )
        return tools

    monkeypatch.setattr(
        ToolFactory,
        "create_all_tools",
        staticmethod(_factory_spy),
    )

    def _assert_route_getters_called_once(
        config: WebToolConfig,
        factory_counts: dict[str, int],
    ) -> None:
        total_counts = getter_call_counts[id(config)]
        assert total_counts == {
            name: factory_counts.get(name, 0) + 1
            for name in AVAILABLE_TOOLS_MODEL_GETTERS
        }

    with run_e2e_app_client(
        monkeypatch,
        username=seeded_user.username,
        user_id=seeded_user.id,
    ) as app:
        baseline_response = app.client.get(
            "/api/tools/available",
            headers=app.headers,
        )
        assert baseline_response.status_code == 200, baseline_response.text
        baseline_body = baseline_response.json()
        assert isinstance(baseline_body["tools"], list)
        assert isinstance(baseline_body["count"], int)
        assert baseline_body["count"] == len(baseline_body["tools"])
        assert baseline_body["count"] > 0
        assert all(
            isinstance(tool, dict)
            and isinstance(tool.get("name"), str)
            and isinstance(tool.get("usage_count"), int)
            for tool in baseline_body["tools"]
        )
        assert len(factory_returns) == 1

        (
            baseline_config,
            baseline_retained_at_factory_return,
            baseline_handed_off_at_factory_return,
            baseline_factory_counts,
        ) = factory_returns[0]
        assert type(baseline_config) is WebToolConfig
        assert baseline_retained_at_factory_return is True
        assert baseline_handed_off_at_factory_return is True
        _assert_route_getters_called_once(
            baseline_config,
            baseline_factory_counts,
        )
        assert baseline_config._retained_factory_model_state is None
        assert baseline_config._lazy_db is None
        assert pool.checkedout() == 0

        with app.session_factory() as probe_db:
            assert probe_db.execute(text("SELECT 1")).scalar_one() == 1
        assert pool.checkedout() == 0

        real_handoff = WebToolConfig.handoff_factory_runtime

        def _restore_legacy_post_handoff_reads(config: WebToolConfig) -> None:
            # Preserve verified detach while restoring only the stale read mode.
            real_handoff(config)
            config._retained_factory_model_state = None
            config._factory_runtime_handed_off = False

        monkeypatch.setattr(
            WebToolConfig,
            "handoff_factory_runtime",
            _restore_legacy_post_handoff_reads,
        )
        caplog.clear()
        tools_logger = logging.getLogger("xagent.web.api.tools")
        capture_directly = caplog.handler not in logging.getLogger().handlers
        if capture_directly:
            tools_logger.addHandler(caplog.handler)
        try:
            with (
                caplog.at_level(logging.ERROR, logger="xagent.web.api.tools"),
                pytest.raises(SQLAlchemyTimeoutError, match="QueuePool limit"),
            ):
                app.client.get(
                    "/api/tools/available",
                    headers=app.headers,
                )
        finally:
            if capture_directly:
                tools_logger.removeHandler(caplog.handler)

        assert len(factory_returns) == 2
        (
            mutation_config,
            mutation_retained_at_factory_return,
            mutation_handed_off_at_factory_return,
            mutation_factory_counts,
        ) = factory_returns[1]
        assert type(mutation_config) is type(baseline_config)
        assert mutation_config is not baseline_config
        assert mutation_retained_at_factory_return is False
        assert mutation_handed_off_at_factory_return is False
        _assert_route_getters_called_once(
            mutation_config,
            mutation_factory_counts,
        )
        usage_errors = [
            record.getMessage()
            for record in caplog.records
            if record.name == "xagent.web.api.tools"
            and record.getMessage().startswith("Failed to fetch tool usage stats:")
        ]
        # Authentication now acquires the request session before the route
        # handler can query usage.  The retained checkout therefore times out
        # in the auth dependency and never reaches the route-level usage log.
        assert usage_errors == []

        # Route cleanup must release the lazy checkout even after the timeout.
        assert mutation_config._retained_factory_model_state is None
        assert mutation_config._factory_runtime_handed_off is True
        assert mutation_config._lazy_db is None
        assert pool.checkedout() == 0
        with app.session_factory() as probe_db:
            assert probe_db.execute(text("SELECT 1")).scalar_one() == 1
        assert pool.checkedout() == 0


@pytest.mark.asyncio
async def test_retained_agent_services_wait_without_holding_postgres_pool(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    postgres_url: str,
) -> None:
    from xagent.web.services.agent_service_manager import create_default_tools

    _configure_postgres_app(
        monkeypatch,
        tmp_path=tmp_path,
        postgres_url=postgres_url,
    )
    disable_external_app_services(monkeypatch)
    session_factory = init_e2e_db()
    with session_factory() as seed_db:
        seeded_user = create_e2e_user(
            seed_db,
            username="retained-skill-runtime-user",
        )
        skill_name = _seed_personal_skill(seed_db, user_id=seeded_user.id)

    engine = get_engine()
    pool = engine.pool
    assert engine.dialect.name == "postgresql"
    assert isinstance(pool, QueuePool)
    assert pool.size() == 1
    assert pool._max_overflow == 0  # noqa: SLF001
    assert pool.timeout() == 1
    assert pool.checkedout() == 0

    real_create_all_tools = ToolFactory.create_all_tools
    factory_calls: list[WebToolConfig] = []

    async def _count_create_all_tools(
        config: WebToolConfig,
        apply_user_override_filter: bool = True,
    ) -> list[Any]:
        factory_calls.append(config)
        return await real_create_all_tools(
            config,
            apply_user_override_filter=apply_user_override_filter,
        )

    real_load_mcp = WebToolConfig._load_mcp_server_configs
    mcp_loads: list[tuple[WebToolConfig, list[dict[str, Any]]]] = []

    async def _spy_load_mcp(
        config: WebToolConfig,
    ) -> list[dict[str, Any]]:
        loaded = await real_load_mcp(config)
        mcp_loads.append((config, loaded))
        return loaded

    monkeypatch.setattr(
        ToolFactory,
        "create_all_tools",
        staticmethod(_count_create_all_tools),
    )
    monkeypatch.setattr(
        WebToolConfig,
        "_load_mcp_server_configs",
        _spy_load_mcp,
    )

    selection = ToolSelectionSpec.from_raw(tool_categories=["skill", "mcp"])
    retained: list[
        tuple[list[Any], WebToolConfig, AgentService, _WaitingLLM, Session]
    ] = []
    try:
        for index in range(2):
            caller_db = session_factory()
            orm_user = caller_db.get(User, seeded_user.id)
            assert orm_user is not None
            assert orm_user.username == seeded_user.username
            assert pool.checkedout() == 1
            request = SimpleNamespace(
                marker=f"request-{index}-{uuid4().hex}",
                user=orm_user,
            )

            tools, config = await create_default_tools(
                caller_db,
                request=request,
                user=orm_user,
                task_id=f"retained-skill-runtime-{index}",
                allowed_skills=[skill_name],
                tool_selection_spec=selection,
            )
            llm = _WaitingLLM(call_id=f"ask-{index}")
            service = AgentService(
                name=f"retained-skill-runtime-{index}",
                id=f"retained-skill-runtime-{index}",
                pattern="react",
                llm=llm,
                tools=tools,
                tool_config=config,
                enable_workspace=False,
            )

            _assert_detached_retained_service(
                caller_db=caller_db,
                request=request,
                user=orm_user,
                user_id=seeded_user.id,
                config=config,
                service=service,
            )
            retained.append((tools, config, service, llm, caller_db))

        assert len(factory_calls) == 2
        assert len(mcp_loads) == 2
        assert all(loaded == [] for _config, loaded in mcp_loads)
        assert [config for config, _loaded in mcp_loads] == [
            retained[0][1],
            retained[1][1],
        ]

        for index, (tools, config, service, llm, _caller_db) in enumerate(retained):
            result = await service.execute_task(
                f"Run the session-safe skill, service {index}.",
                task_id=f"retained-skill-runtime-{index}",
            )

            assert result["success"] is False
            assert result["status"] == "waiting_for_user"
            assert result["message"] == QUESTION
            assert result["message_type"] == "question"
            assert result["interactions"] == [DEFAULT_WAITING_FIELD]
            assert result["chat_response"] == {
                "message": QUESTION,
                "interactions": [DEFAULT_WAITING_FIELD],
            }
            assert (
                service.get_execution_status(f"retained-skill-runtime-{index}")[
                    "status"
                ]
                == "waiting_for_user"
            )
            assert len(llm.calls) == 1
            first_call = llm.calls[0]
            system_text = "\n".join(
                str(message.get("content", ""))
                for message in first_call["messages"]
                if message.get("role") == "system"
            )
            assert SKILL_INDEX_SENTINEL in system_text
            assert EXPECTED_SKILL_INDEX_LINE in system_text
            load_skill_schemas = [
                schema
                for schema in first_call["tools"]
                if schema["function"]["name"] == "load_skill"
            ]
            assert load_skill_schemas == [
                {
                    "type": "function",
                    "function": {
                        "name": "load_skill",
                        "description": (
                            "Load a skill's full instructions into the system context.\n"
                            "\n"
                            "The available skills are listed in the system context "
                            "with one-line summaries. Call this when one of them "
                            "clearly matches the current task; its detailed guidance "
                            "becomes available from the next step on. Do not load "
                            "skills that are unrelated to the task."
                        ),
                        "parameters": EXPECTED_LOAD_SKILL_PARAMETERS,
                    },
                }
            ]
            assert service.tools == tools
            assert config is service.tool_config
            assert pool.checkedout() == 0

        assert len(factory_calls) == 2
        with session_factory() as probe_db:
            assert probe_db.execute(text("SELECT 1")).scalar_one() == 1
        assert pool.checkedout() == 0

        real_handoff = WebToolConfig.handoff_factory_runtime
        mutation_db = session_factory()
        mutation_config: WebToolConfig | None = None
        try:
            mutation_user = mutation_db.get(User, seeded_user.id)
            assert mutation_user is not None
            mutation_request = SimpleNamespace(
                marker=f"mutation-{uuid4().hex}",
                user=mutation_user,
            )
            monkeypatch.setattr(
                WebToolConfig,
                "handoff_factory_runtime",
                WebToolConfig.discard_prepared_factory_runtime,
            )
            _mutation_tools, mutation_config = await create_default_tools(
                mutation_db,
                request=mutation_request,
                user=mutation_user,
                task_id="retained-skill-runtime-mutation",
                allowed_skills=[skill_name],
                tool_selection_spec=selection,
            )
            with pytest.raises(AssertionError, match="verified factory handoff leaked"):
                _assert_detached_retained_service(
                    caller_db=mutation_db,
                    request=mutation_request,
                    user=mutation_user,
                    user_id=seeded_user.id,
                    config=mutation_config,
                    service=AgentService(
                        name="retained-skill-runtime-mutation",
                        id="retained-skill-runtime-mutation",
                        pattern="react",
                        llm=_WaitingLLM(call_id="ask-mutation"),
                        tools=_mutation_tools,
                        tool_config=mutation_config,
                        enable_workspace=False,
                    ),
                )
        finally:
            monkeypatch.setattr(
                WebToolConfig,
                "handoff_factory_runtime",
                real_handoff,
            )
            mutation_db.close()
            if mutation_config is not None:
                mutation_config.close()
    finally:
        for _tools, config, _service, _llm, caller_db in retained:
            caller_db.close()
            config.close()
        engine.dispose()