Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

68.1 KB
Raw Blame
"""Integration tests for automatic LanceDB migration on application startup.

This module tests the automatic migration logic that runs during application
startup to add user_id fields to existing LanceDB tables.
"""

from __future__ import annotations

import asyncio
import logging
import os
import sys
import tempfile
import threading
from types import ModuleType

import pyarrow as pa
import pytest

from xagent.core.tools.core.RAG_tools.core.config import MIN_INT64
from xagent.core.tools.core.RAG_tools.LanceDB.schema_manager import (
    check_table_needs_migration,
)
from xagent.migrations.lancedb.backfill_user_id import (
    ORPHANED_PERMANENT,
    ORPHANED_TEMPORARY,
    backfill_all,
    backfill_chunks_table,
    backfill_orphaned_chunks,
    backfill_orphaned_embeddings,
)
from xagent.providers.vector_store.lancedb import get_connection_from_env
from xagent.sandbox.base import SandboxRuntimeConflictError


def _patch_channel_modules_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
    """Avoid chat-channel startup work in migration tests.

    ``startup_event`` optionally starts the Telegram, Feishu, and Slack managers.

    We inject lightweight stub modules into ``sys.modules`` instead of
    ``monkeypatch.setattr("...telegram.bot...", ...)``, because importing the real
    ``telegram.bot`` or ``feishu.bot`` pulls optional dependencies.
    """

    class _FakeTelegramChannel:
        enabled = False

        async def start(self) -> None:
            return None

        async def stop(self) -> None:
            return None

    class _FakeChannelManager:
        enabled = False  # Disabled to prevent task creation in tests

        async def start(self) -> None:
            return None

        async def stop(self) -> None:
            return None

    # Create fake telegram.bot module
    fake_telegram_bot = ModuleType("xagent.web.channels.telegram.bot")
    fake_telegram_bot.get_telegram_channel = lambda: _FakeTelegramChannel()
    monkeypatch.setitem(
        sys.modules, "xagent.web.channels.telegram.bot", fake_telegram_bot
    )

    # Create fake feishu.bot module
    fake_feishu_bot = ModuleType("xagent.web.channels.feishu.bot")
    fake_feishu_bot.get_feishu_channel = lambda: _FakeChannelManager()
    monkeypatch.setitem(sys.modules, "xagent.web.channels.feishu.bot", fake_feishu_bot)

    fake_slack_bot = ModuleType("xagent.web.channels.slack.bot")
    fake_slack_bot.get_slack_channel = lambda: _FakeChannelManager()
    monkeypatch.setitem(sys.modules, "xagent.web.channels.slack.bot", fake_slack_bot)


def _patch_task_command_dispatcher_disabled(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Keep LanceDB startup tests isolated from the durable DB dispatcher."""
    monkeypatch.setattr(
        "xagent.web.services.task_command_transport.start_task_command_dispatcher",
        lambda _executor: None,
    )


@pytest.fixture(autouse=True)
def unregister_startup_task_extensions():
    """Undo the task extensions ``startup_event()`` registers.

    These tests call ``startup_event()`` without the matching shutdown, so the
    providers it registers (e.g. ``local_browser``) would otherwise stay in the
    process-wide registry and send later tests down the provider branch.
    """
    from xagent.web.services.task_runtime import (
        registered_task_extensions,
        unregister_task_extension,
    )

    before = set(registered_task_extensions())
    yield
    for name in registered_task_extensions():
        if name not in before:
            unregister_task_extension(name)


@pytest.fixture
def temp_lancedb_dir():
    """Create a temporary directory for LanceDB."""
    with tempfile.TemporaryDirectory() as temp_dir:
        original_env = os.environ.get("LANCEDB_DIR")
        os.environ["LANCEDB_DIR"] = temp_dir
        yield temp_dir
        if original_env is not None:
            os.environ["LANCEDB_DIR"] = original_env
        else:
            os.environ.pop("LANCEDB_DIR", None)


def test_auto_migration_detects_old_schema(temp_lancedb_dir):
    """Test that auto-migration detects tables with old schema (missing user_id)."""
    conn = get_connection_from_env()

    # Create tables with old schema (without user_id)
    old_chunks_schema = pa.schema(
        [
            pa.field("collection", pa.string()),
            pa.field("doc_id", pa.string()),
            pa.field("chunk_id", pa.string()),
            pa.field("text", pa.large_string()),
            pa.field("metadata", pa.string()),
        ]
    )
    conn.create_table("chunks", schema=old_chunks_schema)

    old_docs_schema = pa.schema(
        [
            pa.field("collection", pa.string()),
            pa.field("doc_id", pa.string()),
            pa.field("source_path", pa.string()),
        ]
    )
    conn.create_table("documents", schema=old_docs_schema)

    # Verify that migration is needed
    assert check_table_needs_migration(conn, "chunks") is True
    assert check_table_needs_migration(conn, "documents") is True


def test_auto_migration_skips_new_schema(temp_lancedb_dir):
    """Test that auto-migration skips tables with new schema (has user_id)."""
    conn = get_connection_from_env()

    # Create tables with new schema (with user_id)
    new_chunks_schema = pa.schema(
        [
            pa.field("collection", pa.string()),
            pa.field("doc_id", pa.string()),
            pa.field("chunk_id", pa.string()),
            pa.field("text", pa.large_string()),
            pa.field("metadata", pa.string()),
            pa.field("user_id", pa.int32(), nullable=True),
        ]
    )
    conn.create_table("chunks", schema=new_chunks_schema)

    # Verify that no migration is needed
    assert check_table_needs_migration(conn, "chunks") is False


def test_auto_migration_handles_missing_tables(temp_lancedb_dir):
    """Test that auto-migration handles non-existent tables gracefully."""
    conn = get_connection_from_env()

    # Check non-existent tables
    assert check_table_needs_migration(conn, "nonexistent_table") is False
    assert check_table_needs_migration(conn, "chunks") is False
    assert check_table_needs_migration(conn, "documents") is False


def test_migration_detection_for_multiple_tables(temp_lancedb_dir):
    """Test that migration detection works correctly for multiple tables."""
    conn = get_connection_from_env()

    # Create multiple tables with old schema
    old_schema = pa.schema(
        [
            pa.field("collection", pa.string()),
            pa.field("doc_id", pa.string()),
            pa.field("text", pa.string()),
        ]
    )

    conn.create_table("chunks", schema=old_schema)
    conn.create_table("documents", schema=old_schema)
    conn.create_table("parses", schema=old_schema)

    # All should need migration
    assert check_table_needs_migration(conn, "chunks") is True
    assert check_table_needs_migration(conn, "documents") is True
    assert check_table_needs_migration(conn, "parses") is True

    # Non-existent table should not need migration
    assert check_table_needs_migration(conn, "nonexistent") is False


def test_auto_migration_handles_embeddings_tables(temp_lancedb_dir):
    """Test that auto-migration detects embeddings tables correctly."""
    conn = get_connection_from_env()

    # Create embeddings table with old schema
    old_embeddings_schema = pa.schema(
        [
            pa.field("collection", pa.string()),
            pa.field("doc_id", pa.string()),
            pa.field("chunk_id", pa.string()),
            pa.field("vector", pa.list_(pa.float32())),
            pa.field("text", pa.large_string()),
        ]
    )
    conn.create_table("embeddings_test_model", schema=old_embeddings_schema)

    # Verify that migration is needed
    assert check_table_needs_migration(conn, "embeddings_test_model") is True

    # Create embeddings table with new schema
    new_embeddings_schema = pa.schema(
        [
            pa.field("collection", pa.string()),
            pa.field("doc_id", pa.string()),
            pa.field("chunk_id", pa.string()),
            pa.field("vector", pa.list_(pa.float32())),
            pa.field("text", pa.large_string()),
            pa.field("user_id", pa.int32(), nullable=True),
        ]
    )
    conn.create_table("embeddings_test_model_new", schema=new_embeddings_schema)

    # Verify that no migration is needed
    assert check_table_needs_migration(conn, "embeddings_test_model_new") is False


def test_two_phase_migration_recovers_orphaned_records(temp_lancedb_dir):
    """Test that two-phase migration recovers orphaned records when documents are created after chunks."""
    from xagent.core.tools.core.RAG_tools.LanceDB.schema_manager import (
        ensure_chunks_table,
        ensure_documents_table,
    )

    conn = get_connection_from_env()

    # Create chunks table using ensure_chunks_table (creates proper schema)
    ensure_chunks_table(conn)
    ensure_documents_table(conn)

    # Add some chunks without user_id (simulating old data)
    chunks_table = conn.open_table("chunks")
    chunks_table.add(
        [
            {
                "collection": "test",
                "doc_id": "doc_1",
                "parse_hash": "hash1",
                "chunk_id": "chunk_1",
                "index": 0,
                "text": "Test chunk 1",
                "page_number": 1,
                "section": "",
                "anchor": "",
                "json_path": "",
                "chunk_hash": "ch1",
                "config_hash": "cfg1",
                "created_at": 1700000000000,
                "metadata": "{}",
                "user_id": None,
            },
            {
                "collection": "test",
                "doc_id": "doc_2",
                "parse_hash": "hash2",
                "chunk_id": "chunk_2",
                "index": 0,
                "text": "Test chunk 2",
                "page_number": 1,
                "section": "",
                "anchor": "",
                "json_path": "",
                "chunk_hash": "ch2",
                "config_hash": "cfg2",
                "created_at": 1700000000000,
                "metadata": "{}",
                "user_id": None,
            },
        ]
    )

    # Phase 1: Run migration - chunks should be marked as orphaned (user_id = ORPHANED_TEMPORARY)
    result_phase1 = backfill_chunks_table(dry_run=False, conn=conn)
    assert result_phase1["skipped"] == 2  # Both chunks orphaned in phase 1
    assert result_phase1["backfilled"] == 0

    # Verify that chunks are marked with ORPHANED_TEMPORARY after Phase 1
    phase1_chunks = conn.open_table("chunks").search().to_arrow()
    phase1_chunk_data = phase1_chunks.to_pylist()
    for chunk in phase1_chunk_data:
        assert (
            chunk["user_id"] == ORPHANED_TEMPORARY
        )  # Should be marked as temporary orphan

    # Now add documents (simulating concurrent document creation)
    docs_table = conn.open_table("documents")
    docs_table.add(
        [
            {
                "collection": "test",
                "doc_id": "doc_1",
                "source_path": "/path/doc1.txt",
                "file_type": "txt",
                "content_hash": "h1",
                "uploaded_at": 1700000000000,
                "title": "Doc 1",
                "language": "en",
                "user_id": 100,
            },
            {
                "collection": "test",
                "doc_id": "doc_2",
                "source_path": "/path/doc2.txt",
                "file_type": "txt",
                "content_hash": "h2",
                "uploaded_at": 1700000000000,
                "title": "Doc 2",
                "language": "en",
                "user_id": 200,
            },
        ]
    )

    # Phase 2: Retry orphaned chunks
    result_phase2 = backfill_orphaned_chunks(dry_run=False, conn=conn)

    # Verify that orphaned chunks were recovered
    assert result_phase2["backfilled"] == 2  # Both chunks recovered
    assert result_phase2["skipped"] == 0  # No longer orphaned

    # Verify final state
    final_chunks = conn.open_table("chunks").search().to_arrow()
    assert len(final_chunks) == 2
    # Check that user_id values are correct
    chunk_user_ids = final_chunks.to_pylist()
    doc_1_chunk = next(c for c in chunk_user_ids if c["doc_id"] == "doc_1")
    doc_2_chunk = next(c for c in chunk_user_ids if c["doc_id"] == "doc_2")
    assert doc_1_chunk["user_id"] == 100
    assert doc_2_chunk["user_id"] == 200


def test_two_phase_migration_handles_permanently_orphaned_records(temp_lancedb_dir):
    """Test that permanently orphaned records (no matching document ever exists) are marked correctly."""
    from xagent.core.tools.core.RAG_tools.LanceDB.schema_manager import (
        ensure_chunks_table,
        ensure_documents_table,
    )

    conn = get_connection_from_env()

    # Create chunks and documents tables
    ensure_chunks_table(conn)
    ensure_documents_table(conn)

    # Add a chunk with a doc_id that will never have a matching document
    chunks_table = conn.open_table("chunks")
    chunks_table.add(
        [
            {
                "collection": "test",
                "doc_id": "deleted_doc",
                "parse_hash": "hash_deleted",
                "chunk_id": "orphaned_chunk",
                "index": 0,
                "text": "This chunk has no matching document",
                "page_number": 1,
                "section": "",
                "anchor": "",
                "json_path": "",
                "chunk_hash": "ch_deleted",
                "config_hash": "cfg_deleted",
                "created_at": 1700000000000,
                "metadata": "{}",
                "user_id": None,
            }
        ]
    )

    # Run full migration (both phases)
    result = backfill_all(dry_run=False, conn=conn)

    # Verify results
    assert result["chunks"]["backfilled"] == 0  # No chunks backfilled
    assert result["chunks"]["skipped"] == 1  # One chunk permanently orphaned

    # Verify the chunk is marked with user_id = ORPHANED_PERMANENT
    final_chunks = conn.open_table("chunks").search().to_arrow()
    chunk_data = final_chunks.to_pylist()[0]
    assert chunk_data["user_id"] == ORPHANED_PERMANENT  # Marked as permanently orphaned


def test_two_phase_migration_with_mixed_scenarios(temp_lancedb_dir):
    """Test two-phase migration with a mix of normal, temporarily orphaned, and permanently orphaned records."""
    from xagent.core.tools.core.RAG_tools.LanceDB.schema_manager import (
        ensure_chunks_table,
        ensure_documents_table,
    )

    conn = get_connection_from_env()

    # Create chunks and documents tables
    ensure_chunks_table(conn)
    ensure_documents_table(conn)

    # Add documents for doc_1 and doc_3 (doc_2 will be added later, doc_4 never exists)
    docs_table = conn.open_table("documents")
    docs_table.add(
        [
            {
                "collection": "test",
                "doc_id": "doc_1",
                "source_path": "/path/doc1.txt",
                "file_type": "txt",
                "content_hash": "h1",
                "uploaded_at": 1700000000000,
                "title": "Doc 1",
                "language": "en",
                "user_id": 100,
            },
            {
                "collection": "test",
                "doc_id": "doc_3",
                "source_path": "/path/doc3.txt",
                "file_type": "txt",
                "content_hash": "h3",
                "uploaded_at": 1700000000000,
                "title": "Doc 3",
                "language": "en",
                "user_id": 300,
            },
        ]
    )

    # Add chunks with proper schema
    chunks_table = conn.open_table("chunks")
    chunks_table.add(
        [
            {
                "collection": "test",
                "doc_id": "doc_1",
                "parse_hash": "hash1",
                "chunk_id": "chunk_1",
                "index": 0,
                "text": "Chunk with doc",
                "page_number": 1,
                "section": "",
                "anchor": "",
                "json_path": "",
                "chunk_hash": "ch1",
                "config_hash": "cfg1",
                "created_at": 1700000000000,
                "metadata": "{}",
                "user_id": None,
            },
            {
                "collection": "test",
                "doc_id": "doc_2",
                "parse_hash": "hash2",
                "chunk_id": "chunk_2",
                "index": 0,
                "text": "Chunk without doc yet",
                "page_number": 1,
                "section": "",
                "anchor": "",
                "json_path": "",
                "chunk_hash": "ch2",
                "config_hash": "cfg2",
                "created_at": 1700000000000,
                "metadata": "{}",
                "user_id": None,
            },
            {
                "collection": "test",
                "doc_id": "doc_3",
                "parse_hash": "hash3",
                "chunk_id": "chunk_3",
                "index": 0,
                "text": "Another chunk with doc",
                "page_number": 1,
                "section": "",
                "anchor": "",
                "json_path": "",
                "chunk_hash": "ch3",
                "config_hash": "cfg3",
                "created_at": 1700000000000,
                "metadata": "{}",
                "user_id": None,
            },
            {
                "collection": "test",
                "doc_id": "doc_4",
                "parse_hash": "hash4",
                "chunk_id": "chunk_4",
                "index": 0,
                "text": "Chunk with deleted doc",
                "page_number": 1,
                "section": "",
                "anchor": "",
                "json_path": "",
                "chunk_hash": "ch4",
                "config_hash": "cfg4",
                "created_at": 1700000000000,
                "metadata": "{}",
                "user_id": None,
            },
        ]
    )

    # Phase 1: Initial migration
    result_phase1 = backfill_chunks_table(dry_run=False, conn=conn)

    # chunk_1 and chunk_3 should be backfilled
    # chunk_2 and chunk_4 should be marked orphaned
    assert result_phase1["backfilled"] == 2
    assert result_phase1["skipped"] == 2

    # Verify that chunk_2 and chunk_4 are marked with ORPHANED_TEMPORARY after Phase 1
    phase1_chunks = conn.open_table("chunks").search().to_arrow()
    phase1_chunk_data = phase1_chunks.to_pylist()
    chunk_2_phase1 = next(c for c in phase1_chunk_data if c["chunk_id"] == "chunk_2")
    chunk_4_phase1 = next(c for c in phase1_chunk_data if c["chunk_id"] == "chunk_4")
    assert chunk_2_phase1["user_id"] == ORPHANED_TEMPORARY
    assert chunk_4_phase1["user_id"] == ORPHANED_TEMPORARY

    # Now add document for doc_2 (simulating late-arriving document)
    docs_table.add(
        [
            {
                "collection": "test",
                "doc_id": "doc_2",
                "source_path": "/path/doc2.txt",
                "file_type": "txt",
                "content_hash": "h2",
                "uploaded_at": 1700000000000,
                "title": "Doc 2",
                "language": "en",
                "user_id": 200,
            }
        ]
    )

    # Phase 2: Retry orphaned chunks
    result_phase2 = backfill_orphaned_chunks(dry_run=False, conn=conn)

    # chunk_2 should be recovered
    # chunk_4 remains orphaned
    assert result_phase2["backfilled"] == 1
    assert result_phase2["skipped"] == 1

    # Verify final state
    final_chunks = conn.open_table("chunks").search().to_arrow()
    chunk_data = final_chunks.to_pylist()

    # Find each chunk and verify user_id
    chunk_1 = next(c for c in chunk_data if c["chunk_id"] == "chunk_1")
    chunk_2 = next(c for c in chunk_data if c["chunk_id"] == "chunk_2")
    chunk_3 = next(c for c in chunk_data if c["chunk_id"] == "chunk_3")
    chunk_4 = next(c for c in chunk_data if c["chunk_id"] == "chunk_4")

    assert chunk_1["user_id"] == 100  # Backfilled in phase 1
    assert chunk_2["user_id"] == 200  # Recovered in phase 2
    assert chunk_3["user_id"] == 300  # Backfilled in phase 1
    assert chunk_4["user_id"] == ORPHANED_PERMANENT  # Permanently orphaned


def test_two_phase_migration_with_no_orphaned_records(temp_lancedb_dir):
    """Test that phase 2 is skipped when there are no orphaned records."""
    from xagent.core.tools.core.RAG_tools.LanceDB.schema_manager import (
        ensure_chunks_table,
        ensure_documents_table,
    )

    conn = get_connection_from_env()

    # Create chunks and documents tables
    ensure_chunks_table(conn)
    ensure_documents_table(conn)

    # Add document and chunk
    docs_table = conn.open_table("documents")
    docs_table.add(
        [
            {
                "collection": "test",
                "doc_id": "doc_1",
                "source_path": "/path/doc1.txt",
                "file_type": "txt",
                "content_hash": "h1",
                "uploaded_at": 1700000000000,
                "title": "Doc 1",
                "language": "en",
                "user_id": 100,
            }
        ]
    )

    chunks_table = conn.open_table("chunks")
    chunks_table.add(
        [
            {
                "collection": "test",
                "doc_id": "doc_1",
                "parse_hash": "hash1",
                "chunk_id": "chunk_1",
                "index": 0,
                "text": "Test chunk",
                "page_number": 1,
                "section": "",
                "anchor": "",
                "json_path": "",
                "chunk_hash": "ch1",
                "config_hash": "cfg1",
                "created_at": 1700000000000,
                "metadata": "{}",
                "user_id": None,
            }
        ]
    )

    # Run full migration
    result = backfill_all(dry_run=False, conn=conn)

    # All records should be backfilled in phase 1, phase 2 should be skipped
    assert result["chunks"]["backfilled"] == 1
    assert result["chunks"]["skipped"] == 0  # No orphaned records
    assert result["chunks"].get("failed", 0) == 0

    # Verify final state
    final_chunks = conn.open_table("chunks").search().to_arrow()
    chunk_data = final_chunks.to_pylist()[0]
    assert chunk_data["user_id"] == 100


def test_backfill_all_remaps_legacy_minus_one_orphans(temp_lancedb_dir):
    """Legacy -1 orphan markers are remapped and then retried in phase 2."""
    from xagent.core.tools.core.RAG_tools.LanceDB.schema_manager import (
        ensure_chunks_table,
        ensure_documents_table,
    )

    conn = get_connection_from_env()
    ensure_chunks_table(conn)
    ensure_documents_table(conn)

    docs_table = conn.open_table("documents")
    docs_table.add(
        [
            {
                "collection": "test",
                "doc_id": "doc_legacy",
                "source_path": "/path/doc_legacy.txt",
                "file_type": "txt",
                "content_hash": "hash_legacy",
                "uploaded_at": 1700000000000,
                "title": "Legacy Doc",
                "language": "en",
                "user_id": 777,
            }
        ]
    )

    chunks_table = conn.open_table("chunks")
    chunks_table.add(
        [
            {
                "collection": "test",
                "doc_id": "doc_legacy",
                "parse_hash": "hash_legacy",
                "chunk_id": "chunk_legacy",
                "index": 0,
                "text": "legacy orphaned chunk",
                "page_number": 1,
                "section": "",
                "anchor": "",
                "json_path": "",
                "chunk_hash": "ch_legacy",
                "config_hash": "cfg_legacy",
                "created_at": 1700000000000,
                "metadata": "{}",
                "user_id": -1,  # historical transitional value
            }
        ]
    )

    result = backfill_all(dry_run=False, conn=conn)

    # Legacy -1 rows should be remapped to MIN_INT64 then recovered in phase 2.
    assert result["chunks"]["backfilled"] == 1
    assert result["chunks"]["skipped"] == 0

    final_chunks = conn.open_table("chunks").search().to_arrow().to_pylist()
    assert len(final_chunks) == 1
    assert final_chunks[0]["user_id"] == 777
    assert final_chunks[0]["user_id"] != -1
    assert final_chunks[0]["user_id"] != MIN_INT64


def test_phase2_ensures_tables_exist(temp_lancedb_dir):
    """Phase 2 helpers should be callable without prior Phase 1 table creation."""
    conn = get_connection_from_env()

    # Should not raise even if tables don't exist yet.
    res_chunks = backfill_orphaned_chunks(dry_run=True, conn=conn)
    assert res_chunks["table"] == "chunks"

    res_embeddings = backfill_orphaned_embeddings(dry_run=True, conn=conn)
    assert res_embeddings["table"] == "embeddings"


@pytest.mark.asyncio
async def test_startup_event_skips_when_auto_migrate_disabled(
    monkeypatch: pytest.MonkeyPatch, temp_lancedb_dir
):
    """Startup should not create migration task when auto migration is disabled."""
    import importlib

    _patch_channel_modules_disabled(monkeypatch)
    _patch_task_command_dispatcher_disabled(monkeypatch)
    web_app_module = importlib.import_module("xagent.web.app")

    class _FakeManager:
        async def initialize(self) -> None:
            return None

        async def list_skills(self) -> list[str]:
            return []

        async def list_templates(self) -> list[str]:
            return []

    class _FakeMemoryStoreManager:
        def get_store_info(self) -> dict[str, object]:
            return {
                "is_lancedb": True,
                "embedding_model_id": "test-model",
                "similarity_threshold": 0.5,
            }

    class _FakeSandboxManager:
        async def _resolve_backend_probe(self) -> bool:
            return False

        async def cleanup(self) -> None:
            return None

        async def warmup(self) -> None:
            return None

    class _FakeConn:
        pass

    migration_called = {"value": False}
    lease_recovery_starts = []
    runtime_start_order: list[str] = []
    created_tasks: list[asyncio.Task] = []
    original_create_task = asyncio.create_task

    monkeypatch.setenv("LANCEDB_AUTO_MIGRATE", "false")
    monkeypatch.setattr(web_app_module, "init_db", lambda: None)
    task_execution_module = importlib.import_module(
        "xagent.web.services.task_execution"
    )
    monkeypatch.setattr(
        task_execution_module.background_task_manager,
        "start_accepting",
        lambda: runtime_start_order.append("admission"),
    )
    monkeypatch.setattr(
        web_app_module, "start_file_storage_startup_sync_task", lambda _app, **_kw: None
    )
    monkeypatch.setattr(
        web_app_module,
        "start_trigger_dispatcher_task",
        lambda _app: runtime_start_order.append("trigger"),
    )
    monkeypatch.setattr(
        web_app_module,
        "start_task_lease_recovery_task",
        lambda app_instance: lease_recovery_starts.append(app_instance),
    )
    monkeypatch.setattr(web_app_module, "_migration_task", None)
    monkeypatch.setattr(
        "xagent.skills.utils.create_skill_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.templates.utils.create_template_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.web.dynamic_memory_store.get_memory_store_manager",
        lambda: _FakeMemoryStoreManager(),
    )
    monkeypatch.setattr(
        "xagent.web.sandbox_manager.get_sandbox_manager",
        lambda: _FakeSandboxManager(),
    )
    monkeypatch.setattr(
        "xagent.providers.vector_store.lancedb.get_connection_from_env",
        lambda: _FakeConn(),
    )
    monkeypatch.setattr(
        "xagent.core.tools.core.RAG_tools.LanceDB.schema_manager.check_table_needs_migration",
        lambda _conn, table_name: table_name == "chunks",
    )
    monkeypatch.setattr(
        "xagent.core.tools.core.RAG_tools.utils.lancedb_query_utils.list_embeddings_table_names",
        lambda _conn: [],
    )

    def _fake_backfill_all(*, dry_run: bool = False, conn=None) -> dict:
        migration_called["value"] = True
        return {
            "chunks": {"backfilled": 1, "skipped": 0, "failed": 0},
            "embeddings": {"backfilled": 0, "skipped": 0, "failed": 0},
            "locked": True,
        }

    async def _fake_to_thread(fn, *args, **kwargs):
        return fn(*args, **kwargs)

    def _track_create_task(coro):
        task = original_create_task(coro)
        created_tasks.append(task)
        return task

    monkeypatch.setattr(
        "xagent.migrations.lancedb.backfill_user_id.backfill_all",
        _fake_backfill_all,
    )
    monkeypatch.setattr(web_app_module.asyncio, "to_thread", _fake_to_thread)
    monkeypatch.setattr(web_app_module.asyncio, "create_task", _track_create_task)
    monkeypatch.setattr(
        web_app_module,
        "start_app_process_diagnostics",
        lambda _app: runtime_start_order.append("diagnostics"),
    )

    await web_app_module.startup_event()
    if created_tasks:
        await asyncio.gather(*created_tasks)

    assert created_tasks == []
    assert migration_called["value"] is False
    assert lease_recovery_starts == [web_app_module.app]
    assert runtime_start_order == ["admission", "trigger", "diagnostics"]


@pytest.mark.asyncio
async def test_startup_event_triggers_background_auto_migration(
    monkeypatch: pytest.MonkeyPatch, temp_lancedb_dir
):
    """Startup should create task and execute backfill when enabled and needed."""
    import importlib

    _patch_channel_modules_disabled(monkeypatch)
    _patch_task_command_dispatcher_disabled(monkeypatch)
    web_app_module = importlib.import_module("xagent.web.app")

    class _FakeManager:
        async def initialize(self) -> None:
            return None

        async def list_skills(self) -> list[str]:
            return []

        async def list_templates(self) -> list[str]:
            return []

    class _FakeMemoryStoreManager:
        def get_store_info(self) -> dict[str, object]:
            return {
                "is_lancedb": True,
                "embedding_model_id": "test-model",
                "similarity_threshold": 0.5,
            }

    class _FakeSandboxManager:
        async def _resolve_backend_probe(self) -> bool:
            return False

        async def cleanup(self) -> None:
            return None

        async def warmup(self) -> None:
            return None

    class _FakeConn:
        pass

    migration_called = {"value": False}
    created_tasks: list[asyncio.Task] = []
    original_create_task = asyncio.create_task

    monkeypatch.setenv("LANCEDB_AUTO_MIGRATE", "true")
    monkeypatch.setattr(web_app_module, "init_db", lambda: None)
    monkeypatch.setattr(
        web_app_module, "start_file_storage_startup_sync_task", lambda _app, **_kw: None
    )
    monkeypatch.setattr(web_app_module, "_migration_task", None)
    monkeypatch.setattr(
        "xagent.skills.utils.create_skill_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.templates.utils.create_template_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.web.dynamic_memory_store.get_memory_store_manager",
        lambda: _FakeMemoryStoreManager(),
    )
    monkeypatch.setattr(
        "xagent.web.sandbox_manager.get_sandbox_manager",
        lambda: _FakeSandboxManager(),
    )
    monkeypatch.setattr(
        "xagent.providers.vector_store.lancedb.get_connection_from_env",
        lambda: _FakeConn(),
    )
    monkeypatch.setattr(
        "xagent.core.tools.core.RAG_tools.LanceDB.schema_manager.check_table_needs_migration",
        lambda _conn, table_name: table_name == "chunks",
    )
    monkeypatch.setattr(
        "xagent.core.tools.core.RAG_tools.utils.lancedb_query_utils.list_embeddings_table_names",
        lambda _conn: [],
    )

    def _fake_backfill_all(*, dry_run: bool = False, conn=None) -> dict:
        migration_called["value"] = True
        return {
            "chunks": {"backfilled": 1, "skipped": 0, "failed": 0},
            "embeddings": {"backfilled": 0, "skipped": 0, "failed": 0},
            "locked": True,
        }

    async def _fake_to_thread(fn, *args, **kwargs):
        return fn(*args, **kwargs)

    def _track_create_task(coro):
        task = original_create_task(coro)
        created_tasks.append(task)
        return task

    monkeypatch.setattr(
        "xagent.migrations.lancedb.backfill_user_id.backfill_all",
        _fake_backfill_all,
    )
    monkeypatch.setattr(web_app_module.asyncio, "to_thread", _fake_to_thread)
    monkeypatch.setattr(web_app_module.asyncio, "create_task", _track_create_task)

    # The cleanup task itself is gated off under pytest (it walks the real
    # uploads tree), so it contributes no entry to created_tasks and the count
    # assertion below cannot see it. Spy on the call site so deleting it from
    # startup_event fails a test instead of silently disabling the sweep in
    # production.
    cleanup_scheduled: list[object] = []

    def _spy_start_temp_file_cleanup_task(app_instance):
        cleanup_scheduled.append(app_instance)
        return None

    monkeypatch.setattr(
        web_app_module,
        "start_temp_file_cleanup_task",
        _spy_start_temp_file_cleanup_task,
    )

    await web_app_module.startup_event()
    if created_tasks:
        await asyncio.gather(*created_tasks)

    # We expect 2 tasks: backfill migration + uploaded files reconcile. The
    # orphaned temp-file cleanup task is gated off under pytest (it walks the
    # real uploads tree); its wiring/shutdown lifecycle is covered directly by
    # test_shutdown_event_stops_temp_file_cleanup_without_cancel below.
    assert len(created_tasks) == 2
    assert migration_called["value"] is True
    assert cleanup_scheduled == [web_app_module.app]


@pytest.mark.asyncio
async def test_startup_event_no_task_when_no_table_needs_migration(
    monkeypatch: pytest.MonkeyPatch, temp_lancedb_dir
):
    """Startup should not create migration task when no table needs migration."""
    import importlib

    _patch_channel_modules_disabled(monkeypatch)
    _patch_task_command_dispatcher_disabled(monkeypatch)
    web_app_module = importlib.import_module("xagent.web.app")

    class _FakeManager:
        async def initialize(self) -> None:
            return None

        async def list_skills(self) -> list[str]:
            return []

        async def list_templates(self) -> list[str]:
            return []

    class _FakeMemoryStoreManager:
        def get_store_info(self) -> dict[str, object]:
            return {
                "is_lancedb": True,
                "embedding_model_id": "test-model",
                "similarity_threshold": 0.5,
            }

    class _FakeSandboxManager:
        async def _resolve_backend_probe(self) -> bool:
            return False

        async def cleanup(self) -> None:
            return None

        async def warmup(self) -> None:
            return None

    class _FakeConn:
        pass

    migration_called = {"value": False}
    created_tasks: list[asyncio.Task] = []
    original_create_task = asyncio.create_task

    monkeypatch.setenv("LANCEDB_AUTO_MIGRATE", "true")
    monkeypatch.setattr(web_app_module, "init_db", lambda: None)
    monkeypatch.setattr(
        web_app_module, "start_file_storage_startup_sync_task", lambda _app, **_kw: None
    )
    monkeypatch.setattr(web_app_module, "_migration_task", None)
    monkeypatch.setattr(
        "xagent.skills.utils.create_skill_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.templates.utils.create_template_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.web.dynamic_memory_store.get_memory_store_manager",
        lambda: _FakeMemoryStoreManager(),
    )
    monkeypatch.setattr(
        "xagent.web.sandbox_manager.get_sandbox_manager",
        lambda: _FakeSandboxManager(),
    )
    monkeypatch.setattr(
        "xagent.providers.vector_store.lancedb.get_connection_from_env",
        lambda: _FakeConn(),
    )
    monkeypatch.setattr(
        "xagent.core.tools.core.RAG_tools.LanceDB.schema_manager.check_table_needs_migration",
        lambda _conn, _table_name: False,
    )
    monkeypatch.setattr(
        "xagent.core.tools.core.RAG_tools.utils.lancedb_query_utils.list_embeddings_table_names",
        lambda _conn: [],
    )

    def _fake_backfill_all(*, dry_run: bool = False, conn=None) -> dict:
        migration_called["value"] = True
        return {
            "chunks": {"backfilled": 0, "skipped": 0, "failed": 0},
            "embeddings": {"backfilled": 0, "skipped": 0, "failed": 0},
            "locked": True,
        }

    async def _fake_to_thread(fn, *args, **kwargs):
        return fn(*args, **kwargs)

    def _track_create_task(coro):
        task = original_create_task(coro)
        created_tasks.append(task)
        return task

    monkeypatch.setattr(
        "xagent.migrations.lancedb.backfill_user_id.backfill_all",
        _fake_backfill_all,
    )
    monkeypatch.setattr(web_app_module.asyncio, "to_thread", _fake_to_thread)
    monkeypatch.setattr(web_app_module.asyncio, "create_task", _track_create_task)

    await web_app_module.startup_event()
    if created_tasks:
        await asyncio.gather(*created_tasks)

    # We expect 1 task: uploaded files reconcile (runs under auto_migrate even
    # without migration needs). The orphaned temp-file cleanup task is gated off
    # under pytest; its lifecycle is covered by the dedicated shutdown test.
    assert len(created_tasks) == 1
    assert migration_called["value"] is False


@pytest.mark.asyncio
async def test_shutdown_event_stops_temp_file_cleanup_without_cancel(
    monkeypatch: pytest.MonkeyPatch,
):
    """Shutdown must signal the cleanup stop flag first and NOT cancel the task.

    Covers the two round-2 majors on the shutdown path: the stop flag is set
    before any other teardown (so an earlier hang cannot strand the executor
    thread), and the task is awaited via asyncio.wait rather than cancelled.

    On the happy path (the walk finishes almost immediately) asyncio.wait and
    asyncio.wait_for behave identically, so that alone can't tell them apart --
    a prior version of this test made exactly that mistake. The walk task
    below deliberately blocks past a (monkeypatched, tiny) shutdown timeout
    without checking `stop`, forcing a genuine timeout: asyncio.wait leaves
    the task pending past it, while asyncio.wait_for would cancel it.
    """
    import importlib

    _patch_channel_modules_disabled(monkeypatch)
    web_app_module = importlib.import_module("xagent.web.app")
    app = web_app_module.app

    async def _anoop(*args, **kwargs):
        return None

    def _noop(*args, **kwargs):
        return None

    # Neutralize the unrelated teardown steps so only the temp-cleanup stop/await
    # path is exercised.
    for name in (
        "stop_orphan_upload_gc_task",
        "stop_uploaded_file_recovery_task",
        "stop_task_lease_recovery_task",
    ):
        monkeypatch.setattr(web_app_module, name, _anoop)
    monkeypatch.setattr(web_app_module, "unregister_local_browser_runtime", _noop)
    for name in (
        "_task_command_dispatcher_task",
        "_sandbox_idle_sweep_task",
        "_file_storage_startup_sync_task",
        "_trigger_dispatcher_task",
        "_migration_task",
    ):
        monkeypatch.setattr(web_app_module, name, None)

    class _FakeBackgroundTaskManager:
        async def shutdown(self) -> None:
            return None

    monkeypatch.setattr(
        "xagent.web.services.task_execution.background_task_manager",
        _FakeBackgroundTaskManager(),
    )
    monkeypatch.setattr(
        "xagent.web.services.task_lease_service.wait_for_heartbeat_manager_idle",
        _anoop,
    )
    monkeypatch.setattr(
        "xagent.web.services.task_runtime.shutdown_task_runtime_hook_executor",
        _noop,
    )
    monkeypatch.setattr(
        "xagent.web.sandbox_manager.get_sandbox_manager",
        lambda: None,
    )

    # Real but tiny, so the wait genuinely times out without slowing the suite.
    monkeypatch.setattr(
        web_app_module,
        "get_temp_file_cleanup_shutdown_timeout_seconds",
        lambda: 0.05,
    )

    stop = threading.Event()
    flush_saw_stop = {}

    def _spy_flush() -> None:
        # N2: the stop flag must already be set by the time the first teardown
        # step (flush_langfuse) runs.
        flush_saw_stop["set"] = stop.is_set()

    monkeypatch.setattr(web_app_module, "flush_langfuse", _spy_flush)

    started = asyncio.Event()
    resume = asyncio.Event()

    async def _cooperative_walk() -> None:
        started.set()
        # Simulate being mid-scan, past the point where `stop` would next be
        # polled, until the test releases `resume` below -- this is what
        # forces the shutdown wait to genuinely time out.
        await resume.wait()

    task = asyncio.create_task(_cooperative_walk())
    await started.wait()

    app.state.temp_file_cleanup_stop = stop
    app.state.temp_file_cleanup_task = task

    await web_app_module.shutdown_event()

    try:
        assert flush_saw_stop["set"] is True
        assert stop.is_set()
        # The timeout above genuinely elapsed while the walk was still blocked on
        # `resume`. asyncio.wait_for would have cancelled the task by now;
        # asyncio.wait leaves it running, untouched.
        assert not task.done()
        assert not task.cancelled()
        # The walk outlived the bounded wait, so its handles must be RETAINED.
        # Clearing them here would discard the only stop handle and let a
        # re-entrant startup schedule a second concurrent sweep over the same
        # tree -- the exact case start_temp_file_cleanup_task's guard exists for.
        assert app.state.temp_file_cleanup_task is task
        assert app.state.temp_file_cleanup_stop is stop

        resume.set()
        await task

        # Once the walk has actually finished, a later shutdown releases them.
        await web_app_module.shutdown_event()
        assert app.state.temp_file_cleanup_task is None
        assert app.state.temp_file_cleanup_stop is None
    finally:
        # Never leave a pending task or populated app.state behind for the rest
        # of the session, even if an assertion above fails.
        resume.set()
        if not task.done():
            await task
        app.state.temp_file_cleanup_task = None
        app.state.temp_file_cleanup_stop = None


@pytest.mark.asyncio
async def test_start_temp_file_cleanup_task_wires_stop_event(
    monkeypatch: pytest.MonkeyPatch,
):
    """The pytest-gated wiring block itself must actually wire things up.

    Regression guard: an earlier commit on this PR asserted
    app.state.temp_file_cleanup_task is not None and
    isinstance(app.state.temp_file_cleanup_stop, threading.Event) directly
    against startup_event(), but those assertions were dropped rather than
    reworked when the PYTEST_CURRENT_TEST gate was introduced, leaving this
    block untested. Calling start_temp_file_cleanup_task directly (rather
    than the full startup_event) exercises its own pytest gate without also
    un-gating the unrelated trigger/lease-recovery/orphan-GC loops that share
    the same env-var check elsewhere in startup_event.
    """
    import importlib

    web_app_module = importlib.import_module("xagent.web.app")
    app = web_app_module.app

    monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
    app.state.temp_file_cleanup_task = None
    app.state.temp_file_cleanup_stop = None

    seen_stop_events: list[threading.Event] = []

    def _fake_cleanup_orphaned_temp_files(*, stop_event=None) -> int:
        seen_stop_events.append(stop_event)
        return 0

    monkeypatch.setattr(
        "xagent.web.services.storage_maintenance.cleanup_orphaned_temp_files",
        _fake_cleanup_orphaned_temp_files,
    )

    task = web_app_module.start_temp_file_cleanup_task(app)
    stop_event = app.state.temp_file_cleanup_stop

    assert task is not None
    assert task is app.state.temp_file_cleanup_task
    assert isinstance(stop_event, threading.Event)

    await task

    # Pins that the exact stop-event object stored on app.state is the one
    # actually threaded into the walk -- a wiring bug here would silently
    # make shutdown's stop signal a no-op.
    assert seen_stop_events == [stop_event]

    app.state.temp_file_cleanup_task = None
    app.state.temp_file_cleanup_stop = None


@pytest.mark.asyncio
async def test_failed_startup_leaves_no_unsignaled_temp_file_cleanup(
    monkeypatch: pytest.MonkeyPatch, temp_lancedb_dir
):
    """A startup that fails must not leave the sweep walking unsignaled.

    Starlette's default lifespan is an ``async with``: when startup raises,
    ``__aexit__`` -- and therefore ``shutdown_event`` -- never runs, so the
    only ``stop_event.set()`` is skipped. Sandbox static readiness raises
    deliberately on a mount conflict, so the walk would keep sweeping the
    whole uploads tree in an executor thread that ``asyncio.run()``'s teardown
    then joins (with no timeout at all before 3.12), turning a fast
    config-error crash into a multi-minute one.

    Asserts the invariant rather than a call position: after the failure there
    must be no cleanup task that is both unfinished and unsignaled. Scheduling
    the sweep only after every failure-prone startup step satisfies it, and so
    would signalling the flag on the way out.

    The invariant alone is weak here -- today it holds because the sweep is
    never reached, so ``task is None`` satisfies it -- so this also pins the
    reason: the induced failure must land *before* the scheduling point. That
    is the regression worth catching, because moving the scheduling earlier is
    what would leave a walk running with no way to stop it.
    """
    import importlib

    _patch_channel_modules_disabled(monkeypatch)
    _patch_task_command_dispatcher_disabled(monkeypatch)
    web_app_module = importlib.import_module("xagent.web.app")
    app = web_app_module.app

    class _FakeManager:
        async def initialize(self) -> None:
            return None

        async def list_skills(self) -> list[str]:
            return []

        async def list_templates(self) -> list[str]:
            return []

    class _FakeMemoryStoreManager:
        def get_store_info(self) -> dict[str, object]:
            return {
                "is_lancedb": True,
                "embedding_model_id": "test-model",
                "similarity_threshold": 0.5,
            }

    class _FakeSandboxManager:
        async def _resolve_backend_probe(self) -> bool:
            return True

        def requires_host_workspace_access(self) -> bool:
            return True

        def validate_host_workspace_access(self) -> None:
            return None

        async def cleanup(self) -> None:
            return None

        async def warmup(self) -> None:
            return None

    class _FakeConn:
        pass

    created_tasks: list[asyncio.Task] = []
    original_create_task = asyncio.create_task
    real_to_thread = asyncio.to_thread

    # The sweep only runs under the auto-migrate gate, so it has to be on for
    # the failure path under test to be reachable at all.
    monkeypatch.setenv("LANCEDB_AUTO_MIGRATE", "true")
    # The same genuine custom-volume conflict as
    # test_startup_event_raises_on_readiness_conflict_with_probe_true. Static
    # readiness runs well before the sweep is scheduled, so this failure lands
    # ahead of every position the sweep must not be scheduled at.
    monkeypatch.setenv("SANDBOX_VOLUMES", "/foo:/guest1:ro;/foo:/guest2:ro")
    monkeypatch.setattr(web_app_module, "init_db", lambda: None)
    monkeypatch.setattr(
        web_app_module, "start_file_storage_startup_sync_task", lambda _app, **_kw: None
    )
    monkeypatch.setattr(web_app_module, "_migration_task", None)
    monkeypatch.setattr(
        "xagent.skills.utils.create_skill_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.templates.utils.create_template_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.web.dynamic_memory_store.get_memory_store_manager",
        lambda: _FakeMemoryStoreManager(),
    )
    monkeypatch.setattr(
        "xagent.web.sandbox_manager.get_sandbox_manager",
        lambda: _FakeSandboxManager(),
    )
    monkeypatch.setattr(
        "xagent.providers.vector_store.lancedb.get_connection_from_env",
        lambda: _FakeConn(),
    )
    monkeypatch.setattr(
        "xagent.core.tools.core.RAG_tools.LanceDB.schema_manager.check_table_needs_migration",
        lambda _conn, table_name: False,
    )
    monkeypatch.setattr(
        "xagent.core.tools.core.RAG_tools.utils.lancedb_query_utils.list_embeddings_table_names",
        lambda _conn: [],
    )

    # The uploaded-files reconcile task is scheduled just before the sweep and
    # is not what this test is about; stub its units of work so it cannot spray
    # unrelated LanceDB/DB tracebacks over this test's output.
    class _FakeSession:
        def close(self) -> None:
            return None

    monkeypatch.setattr(
        "xagent.migrations.lancedb.backfill_uploaded_file_links.backfill_all",
        lambda *args, **kwargs: {},
    )
    monkeypatch.setattr(
        "xagent.web.services.kb_file_service.reconcile_uploaded_files",
        lambda *args, **kwargs: {},
    )
    monkeypatch.setattr("xagent.web.models.database._SessionLocal", _FakeSession)

    def _blocking_cleanup(*, stop_event=None) -> int:
        # Stand in for a real sweep of a large tree: an executor walk that only
        # unwinds once the cooperative stop flag is set. The timeout is a test
        # safety net so a regression cannot hang the suite, not part of the
        # contract under test.
        assert stop_event is not None
        stop_event.wait(timeout=30)
        return 0

    monkeypatch.setattr(
        "xagent.web.services.storage_maintenance.cleanup_orphaned_temp_files",
        _blocking_cleanup,
    )

    real_start = web_app_module.start_temp_file_cleanup_task
    scheduling_reached = []

    def _start_ungated(app_instance):
        # Exercise the REAL wrapper, whose own PYTEST_CURRENT_TEST gate would
        # otherwise skip the sweep entirely. Clearing the variable only around
        # this call keeps the five unrelated gates in startup_event (trigger
        # dispatcher, lease/upload recovery, orphan GC, metadata rebuild) in
        # force, so the failure path is the only thing this test un-gates.
        scheduling_reached.append(app_instance)
        saved = os.environ.pop("PYTEST_CURRENT_TEST", None)
        try:
            return real_start(app_instance)
        finally:
            if saved is not None:
                os.environ["PYTEST_CURRENT_TEST"] = saved

    monkeypatch.setattr(web_app_module, "start_temp_file_cleanup_task", _start_ungated)
    # The Composio reconcile pass has the same executor-thread shape, so it
    # must not be scheduled before the failing step either.
    reconcile_reached = []
    monkeypatch.setattr(
        web_app_module,
        "start_composio_reconcile_task",
        lambda app_instance: reconcile_reached.append(app_instance),
    )

    async def _selective_to_thread(fn, *args, **kwargs):
        # The sweep must run in a real thread: the stub above blocks until the
        # stop flag is set, which can never happen if it runs inline on the
        # event loop. Everything else stays inline, like the sibling tests.
        if fn is _blocking_cleanup:
            return await real_to_thread(fn, *args, **kwargs)
        return fn(*args, **kwargs)

    def _track_create_task(coro):
        task = original_create_task(coro)
        created_tasks.append(task)
        return task

    monkeypatch.setattr(web_app_module.asyncio, "to_thread", _selective_to_thread)
    monkeypatch.setattr(web_app_module.asyncio, "create_task", _track_create_task)

    try:
        with pytest.raises(SandboxRuntimeConflictError):
            await web_app_module.startup_event()

        # WHY: pins the reason the invariant below holds -- the raising step
        # comes first, so the sweep was never scheduled. Without this, moving
        # the scheduling earlier could still satisfy it and pass on luck.
        assert scheduling_reached == []
        assert reconcile_reached == []

        task = getattr(app.state, "temp_file_cleanup_task", None)
        stop = getattr(app.state, "temp_file_cleanup_stop", None)
        assert task is None or task.done() or (stop is not None and stop.is_set())
    finally:
        # Never leave a blocked executor thread, a pending task, or populated
        # app.state behind for the rest of the session.
        stop = getattr(app.state, "temp_file_cleanup_stop", None)
        if stop is not None:
            stop.set()
        app.state.temp_file_cleanup_task = None
        app.state.temp_file_cleanup_stop = None
        if created_tasks:
            await asyncio.gather(*created_tasks, return_exceptions=True)


@pytest.mark.asyncio
async def test_startup_event_runs_sandbox_readiness_before_cleanup_and_warmup(
    monkeypatch: pytest.MonkeyPatch, temp_lancedb_dir
):
    """Sandbox readiness must run before cleanup(), which must run before
    warmup() -- a static mount-conflict must fail startup before any
    container is touched."""
    import importlib

    _patch_channel_modules_disabled(monkeypatch)
    _patch_task_command_dispatcher_disabled(monkeypatch)
    web_app_module = importlib.import_module("xagent.web.app")

    class _FakeManager:
        async def initialize(self) -> None:
            return None

        async def list_skills(self) -> list[str]:
            return []

        async def list_templates(self) -> list[str]:
            return []

    class _FakeMemoryStoreManager:
        def get_store_info(self) -> dict[str, object]:
            return {
                "is_lancedb": True,
                "embedding_model_id": "test-model",
                "similarity_threshold": 0.5,
            }

    call_order: list[str] = []

    class _FakeSandboxManager:
        async def _resolve_backend_probe(self) -> bool:
            call_order.append("probe")
            return False

        async def cleanup(self) -> None:
            call_order.append("cleanup")

        async def warmup(self) -> None:
            call_order.append("warmup")

    class _FakeConn:
        pass

    created_tasks: list[asyncio.Task] = []
    original_create_task = asyncio.create_task

    monkeypatch.setenv("LANCEDB_AUTO_MIGRATE", "false")
    monkeypatch.setattr(web_app_module, "init_db", lambda: None)
    monkeypatch.setattr(
        web_app_module, "start_file_storage_startup_sync_task", lambda _app, **_kw: None
    )
    monkeypatch.setattr(web_app_module, "_migration_task", None)
    monkeypatch.setattr(
        "xagent.skills.utils.create_skill_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.templates.utils.create_template_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.web.dynamic_memory_store.get_memory_store_manager",
        lambda: _FakeMemoryStoreManager(),
    )
    monkeypatch.setattr(
        "xagent.web.sandbox_manager.get_sandbox_manager",
        lambda: _FakeSandboxManager(),
    )
    monkeypatch.setattr(
        "xagent.providers.vector_store.lancedb.get_connection_from_env",
        lambda: _FakeConn(),
    )

    async def _fake_to_thread(fn, *args, **kwargs):
        return fn(*args, **kwargs)

    def _track_create_task(coro):
        task = original_create_task(coro)
        created_tasks.append(task)
        return task

    monkeypatch.setattr(web_app_module.asyncio, "to_thread", _fake_to_thread)
    monkeypatch.setattr(web_app_module.asyncio, "create_task", _track_create_task)

    await web_app_module.startup_event()
    if created_tasks:
        await asyncio.gather(*created_tasks)

    assert call_order == ["probe", "cleanup", "warmup"]


@pytest.mark.asyncio
async def test_startup_event_raises_on_readiness_conflict_with_probe_true(
    monkeypatch: pytest.MonkeyPatch, temp_lancedb_dir
):
    """probe=True plus a genuine custom-volume conflict must
    fail startup outright, before cleanup()/warmup() ever run -- the actual
    static-conflict path ``check_sandbox_static_readiness`` exists to guard,
    not just the probe=False short-circuit the sibling test above covers."""
    import importlib

    _patch_channel_modules_disabled(monkeypatch)
    _patch_task_command_dispatcher_disabled(monkeypatch)
    web_app_module = importlib.import_module("xagent.web.app")

    class _FakeManager:
        async def initialize(self) -> None:
            return None

        async def list_skills(self) -> list[str]:
            return []

        async def list_templates(self) -> list[str]:
            return []

    class _FakeMemoryStoreManager:
        def get_store_info(self) -> dict[str, object]:
            return {
                "is_lancedb": True,
                "embedding_model_id": "test-model",
                "similarity_threshold": 0.5,
            }

    call_order: list[str] = []

    class _FakeSandboxManager:
        async def _resolve_backend_probe(self) -> bool:
            call_order.append("probe")
            return True

        def requires_host_workspace_access(self) -> bool:
            return True

        def validate_host_workspace_access(self) -> None:
            return None

        async def cleanup(self) -> None:
            call_order.append("cleanup")

        async def warmup(self) -> None:
            call_order.append("warmup")

    class _FakeConn:
        pass

    monkeypatch.setenv("LANCEDB_AUTO_MIGRATE", "false")
    # A genuine host-path conflict between two SANDBOX_VOLUMES
    # entries, same shape as test_sandbox_manager_readiness.py's
    # test_readiness_raises_on_host_conflict.
    monkeypatch.setenv("SANDBOX_VOLUMES", "/foo:/guest1:ro;/foo:/guest2:ro")
    monkeypatch.setattr(web_app_module, "init_db", lambda: None)
    monkeypatch.setattr(
        web_app_module, "start_file_storage_startup_sync_task", lambda _app, **_kw: None
    )
    monkeypatch.setattr(web_app_module, "_migration_task", None)
    monkeypatch.setattr(
        "xagent.skills.utils.create_skill_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.templates.utils.create_template_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.web.dynamic_memory_store.get_memory_store_manager",
        lambda: _FakeMemoryStoreManager(),
    )
    monkeypatch.setattr(
        "xagent.web.sandbox_manager.get_sandbox_manager",
        lambda: _FakeSandboxManager(),
    )
    monkeypatch.setattr(
        "xagent.providers.vector_store.lancedb.get_connection_from_env",
        lambda: _FakeConn(),
    )

    async def _fake_to_thread(fn, *args, **kwargs):
        return fn(*args, **kwargs)

    monkeypatch.setattr(web_app_module.asyncio, "to_thread", _fake_to_thread)

    with pytest.raises(SandboxRuntimeConflictError):
        await web_app_module.startup_event()

    assert call_order == ["probe"]


@pytest.mark.asyncio
async def test_startup_event_skips_sandbox_readiness_when_manager_is_none(
    monkeypatch: pytest.MonkeyPatch, temp_lancedb_dir
):
    """``get_sandbox_manager()`` can swallow a factory exception and return
    None (sandbox disabled or init failed); startup must skip readiness
    (and cleanup/warmup) entirely rather than raise."""
    import importlib

    _patch_channel_modules_disabled(monkeypatch)
    _patch_task_command_dispatcher_disabled(monkeypatch)
    web_app_module = importlib.import_module("xagent.web.app")

    class _FakeManager:
        async def initialize(self) -> None:
            return None

        async def list_skills(self) -> list[str]:
            return []

        async def list_templates(self) -> list[str]:
            return []

    class _FakeMemoryStoreManager:
        def get_store_info(self) -> dict[str, object]:
            return {
                "is_lancedb": True,
                "embedding_model_id": "test-model",
                "similarity_threshold": 0.5,
            }

    class _FakeConn:
        pass

    created_tasks: list[asyncio.Task] = []
    original_create_task = asyncio.create_task

    monkeypatch.setenv("LANCEDB_AUTO_MIGRATE", "false")
    monkeypatch.setattr(web_app_module, "init_db", lambda: None)
    monkeypatch.setattr(
        web_app_module, "start_file_storage_startup_sync_task", lambda _app, **_kw: None
    )
    monkeypatch.setattr(web_app_module, "_migration_task", None)
    monkeypatch.setattr(
        "xagent.skills.utils.create_skill_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.templates.utils.create_template_manager",
        lambda: _FakeManager(),
    )
    monkeypatch.setattr(
        "xagent.web.dynamic_memory_store.get_memory_store_manager",
        lambda: _FakeMemoryStoreManager(),
    )
    monkeypatch.setattr(
        "xagent.web.sandbox_manager.get_sandbox_manager",
        lambda: None,
    )
    readiness_called = {"value": False}

    async def _fail_if_called(_sandbox_mgr) -> None:
        readiness_called["value"] = True

    monkeypatch.setattr(
        "xagent.web.sandbox_manager.check_sandbox_static_readiness",
        _fail_if_called,
    )
    monkeypatch.setattr(
        "xagent.providers.vector_store.lancedb.get_connection_from_env",
        lambda: _FakeConn(),
    )

    async def _fake_to_thread(fn, *args, **kwargs):
        return fn(*args, **kwargs)

    def _track_create_task(coro):
        task = original_create_task(coro)
        created_tasks.append(task)
        return task

    monkeypatch.setattr(web_app_module.asyncio, "to_thread", _fake_to_thread)
    monkeypatch.setattr(web_app_module.asyncio, "create_task", _track_create_task)

    await web_app_module.startup_event()
    if created_tasks:
        await asyncio.gather(*created_tasks)

    assert readiness_called["value"] is False


@pytest.mark.asyncio
async def test_startup_reports_a_still_unwinding_previous_cleanup(
    monkeypatch: pytest.MonkeyPatch,
):
    """A retained generation that is already stop-signalled is not "running".

    A shutdown whose bounded wait expires deliberately keeps the pending task
    and its already-set stop event, so the next startup on the same app object
    hits the re-entrancy guard. Scheduling a second walk there is still wrong,
    but reporting a live sweep is a lie: that generation is unwinding and this
    startup gets none. Only reachable when one app object runs lifespan twice
    (the test suite's module-level app), never in a deployment that exits
    after shutdown.
    """
    import importlib

    web_app_module = importlib.import_module("xagent.web.app")
    app = web_app_module.app

    monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
    app.state.temp_file_cleanup_task = None
    app.state.temp_file_cleanup_stop = None

    unwind = threading.Event()
    sweeps_started: list[threading.Event] = []

    def _stalled_cleanup(*, stop_event=None) -> int:
        sweeps_started.append(stop_event)
        unwind.wait(timeout=5)
        return 0

    monkeypatch.setattr(
        "xagent.web.services.storage_maintenance.cleanup_orphaned_temp_files",
        _stalled_cleanup,
    )

    try:
        first = web_app_module.start_temp_file_cleanup_task(app)
        assert first is not None
        first_stop = app.state.temp_file_cleanup_stop

        # A shutdown whose bounded wait expired: signalled, but still pending.
        first_stop.set()
        assert not first.done()

        # A handler on the module logger rather than caplog: this app configures
        # root logging on import, which displaces caplog's own root handler.
        records: list[logging.LogRecord] = []

        class _Collect(logging.Handler):
            def emit(self, record: logging.LogRecord) -> None:
                records.append(record)

        collector = _Collect()
        web_app_module.logger.addHandler(collector)
        try:
            second = web_app_module.start_temp_file_cleanup_task(app)
        finally:
            web_app_module.logger.removeHandler(collector)

        assert second is first
        assert app.state.temp_file_cleanup_stop is first_stop
        assert [
            r for r in records if r.levelno == logging.WARNING and "unwinding" in r.msg
        ]
        assert not [r for r in records if "Scheduled background" in r.msg]
    finally:
        unwind.set()
        await first
        app.state.temp_file_cleanup_task = None
        app.state.temp_file_cleanup_stop = None

    # The guard held: exactly one sweep ran, using the first generation's flag.
    assert sweeps_started == [first_stop]


@pytest.mark.asyncio
async def test_background_cleanup_logs_a_failing_sweep_with_its_traceback(
    monkeypatch: pytest.MonkeyPatch,
):
    """A sweep that raises must be reported with its traceback, not swallowed.

    The walk runs unattended in an executor thread: nothing awaits the task
    for a result and no request path sees the exception, so this WARNING is
    the failure's only surface. Without it a sweep that never cleans anything
    looks exactly like one with nothing to clean.
    """
    import importlib

    web_app_module = importlib.import_module("xagent.web.app")
    app = web_app_module.app

    monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
    app.state.temp_file_cleanup_task = None
    app.state.temp_file_cleanup_stop = None

    def _failing_cleanup(*, stop_event=None) -> int:
        raise OSError("uploads root vanished mid-sweep")

    monkeypatch.setattr(
        "xagent.web.services.storage_maintenance.cleanup_orphaned_temp_files",
        _failing_cleanup,
    )

    records: list[logging.LogRecord] = []

    class _Collect(logging.Handler):
        def emit(self, record: logging.LogRecord) -> None:
            records.append(record)

    collector = _Collect()
    web_app_module.logger.addHandler(collector)
    try:
        task = web_app_module.start_temp_file_cleanup_task(app)
        assert task is not None
        await task
        # Nothing retrieves this task's result in production, so a failure
        # left on it would surface only as an "exception was never retrieved"
        # warning at interpreter exit -- long after the sweep mattered.
        assert task.exception() is None
    finally:
        web_app_module.logger.removeHandler(collector)
        app.state.temp_file_cleanup_task = None
        app.state.temp_file_cleanup_stop = None

    failures = [
        r
        for r in records
        if r.levelno == logging.WARNING and "cleanup failed" in r.getMessage()
    ]
    assert len(failures) == 1
    assert failures[0].exc_info is not None
    assert "uploads root vanished mid-sweep" in logging.Formatter().formatException(
        failures[0].exc_info
    )


@pytest.mark.asyncio
async def test_startup_reuses_a_live_unsignaled_cleanup_generation(
    monkeypatch: pytest.MonkeyPatch,
):
    """A live, unsignalled sweep is handed back, not joined by a second walk.

    Sibling of the still-unwinding case above: same guard, opposite state.
    Two concurrent walks over one tree would race each other's unlinks, and
    the newer generation's stop event would overwrite the older one's on
    app.state -- orphaning the only handle that can stop the walk in flight.
    """
    import importlib

    web_app_module = importlib.import_module("xagent.web.app")
    app = web_app_module.app

    monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
    app.state.temp_file_cleanup_task = None
    app.state.temp_file_cleanup_stop = None

    release = threading.Event()
    sweeps_started: list[threading.Event] = []

    def _stalled_cleanup(*, stop_event=None) -> int:
        sweeps_started.append(stop_event)
        release.wait(timeout=5)
        return 0

    monkeypatch.setattr(
        "xagent.web.services.storage_maintenance.cleanup_orphaned_temp_files",
        _stalled_cleanup,
    )

    try:
        first = web_app_module.start_temp_file_cleanup_task(app)
        assert first is not None
        first_stop = app.state.temp_file_cleanup_stop

        # Still sweeping and never signalled -- unlike the unwinding sibling.
        assert not first.done()
        assert not first_stop.is_set()

        second = web_app_module.start_temp_file_cleanup_task(app)

        assert second is first
        assert app.state.temp_file_cleanup_task is first
        assert app.state.temp_file_cleanup_stop is first_stop
        assert not first_stop.is_set()
    finally:
        release.set()
        await first
        app.state.temp_file_cleanup_task = None
        app.state.temp_file_cleanup_stop = None

    # The guard held: one walk, still holding the flag shutdown would set.
    assert sweeps_started == [first_stop]