Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

63.2 KB
Raw Blame
"""Test cases for TaskTracker and TaskTrackerManager."""

import asyncio
import threading
from concurrent.futures import ThreadPoolExecutor
from unittest.mock import MagicMock, patch

import pytest
from sqlalchemy import create_engine, event
from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import QueuePool

from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from tests.web.pool_contention_shared import (
    GUARD_TIMEOUT,
    LOOP_LIVENESS_TICKS,
    gated_pool_checkout,
    wait_for_ticks,
)
from xagent.core.model.chat.token_context import (
    TokenUsage,
    add_token_usage,
    get_token_usage,
)
from xagent.web.models.task import Task, TaskStatus
from xagent.web.services.db_runtime import drain_async_task_cancellation_safe
from xagent.web.tracking import TaskTracker, TaskTrackerManager


def _create_tracker_concurrency_db(tmp_path, filename):
    engine = create_engine(
        f"sqlite:///{tmp_path / filename}",
        connect_args={"check_same_thread": False},
        isolation_level="AUTOCOMMIT",
    )
    Task.__table__.create(engine)
    session_factory = sessionmaker(
        autocommit=False,
        autoflush=False,
        expire_on_commit=False,
        bind=engine,
    )
    with session_factory() as db:
        db.add(
            Task(
                id=123,
                user_id=1,
                title="tracker ownership race",
                status=TaskStatus.RUNNING,
                run_id="run-a",
                runner_id="runner-a",
                input_tokens=4,
                output_tokens=2,
                total_tokens=6,
                llm_calls=1,
            )
        )
        db.commit()
    return engine, session_factory


def _run_tracker_ownership_race(
    monkeypatch,
    tmp_path,
    filename,
    operation,
    *,
    pause_after_ownership_read=False,
):
    from xagent.web.models import database
    from xagent.web.tracking import task_tracker as tracker_module

    engine, session_factory = _create_tracker_concurrency_db(tmp_path, filename)
    monkeypatch.setattr(database, "_SessionLocal", session_factory)
    fence_ready = threading.Event()
    takeover_done = threading.Event()
    pause_lock = threading.Lock()
    worker_thread_id: int | None = None
    paused = False
    result: dict[str, bool] = {}
    errors: list[BaseException] = []
    original_task_for_run = tracker_module._task_for_run

    def pause_for_takeover() -> None:
        nonlocal paused
        with pause_lock:
            if paused:
                return
            paused = True
        fence_ready.set()
        assert takeover_done.wait(timeout=DB_PROGRESS_TIMEOUT)

    def pause_before_usage_update(
        _connection,
        _cursor,
        statement,
        _parameters,
        _context,
        _executemany,
    ):
        if (
            threading.get_ident() == worker_thread_id
            and statement.lstrip().upper().startswith("UPDATE TASKS SET")
            and "input_tokens" in statement.lower()
        ):
            pause_for_takeover()

    def pause_after_legacy_ownership_read(*args, **kwargs):
        task = original_task_for_run(*args, **kwargs)
        if threading.get_ident() == worker_thread_id:
            pause_for_takeover()
        return task

    event.listen(engine, "before_cursor_execute", pause_before_usage_update)
    if pause_after_ownership_read:
        monkeypatch.setattr(
            tracker_module,
            "_task_for_run",
            pause_after_legacy_ownership_read,
        )

    def run_as_old_runner():
        nonlocal worker_thread_id
        worker_thread_id = threading.get_ident()
        try:
            result["owned"] = operation(tracker_module)
        except BaseException as error:
            errors.append(error)

    worker = threading.Thread(target=run_as_old_runner)
    worker.start()
    try:
        assert fence_ready.wait(timeout=DB_PROGRESS_TIMEOUT)
        with session_factory() as takeover_db:
            replacement = takeover_db.get(Task, 123)
            assert replacement is not None
            replacement.runner_id = "runner-b"
            replacement.input_tokens = 100
            replacement.output_tokens = 50
            replacement.total_tokens = 150
            takeover_db.commit()
    finally:
        takeover_done.set()
        worker.join(timeout=DB_PROGRESS_TIMEOUT)
        event.remove(engine, "before_cursor_execute", pause_before_usage_update)

    assert not worker.is_alive()
    assert errors == []
    with session_factory() as verify_db:
        replacement = verify_db.get(Task, 123)
        assert replacement is not None
        persisted = (
            replacement.runner_id,
            replacement.input_tokens,
            replacement.output_tokens,
            replacement.total_tokens,
        )
    engine.dispose()
    return result["owned"], persisted


def _create_tracker_pool_db(tmp_path, filename):
    engine = create_engine(
        f"sqlite:///{tmp_path / filename}",
        connect_args={"check_same_thread": False},
        poolclass=QueuePool,
        pool_size=1,
        max_overflow=0,
        pool_timeout=1,
    )
    Task.__table__.create(engine)
    session_factory = sessionmaker(
        autocommit=False,
        autoflush=False,
        bind=engine,
    )
    with session_factory() as db:
        db.add(
            Task(
                id=123,
                user_id=1,
                title="pool isolation",
                status=TaskStatus.RUNNING,
            )
        )
        db.commit()
    return engine, session_factory


async def _assert_tracker_operation_keeps_loop_responsive_under_pool_pressure(
    monkeypatch,
    tmp_path,
    filename,
    sync_helper_name,
    prepare,
    operation,
    verify,
    *,
    operation_scheduled=None,
):
    """Exercise one tracker DB operation while its QueuePool is exhausted."""
    from xagent.web.models import database
    from xagent.web.tracking import task_tracker as tracker_module

    engine, session_factory = _create_tracker_pool_db(tmp_path, filename)
    monkeypatch.setattr(database, "_SessionLocal", session_factory)
    tracker = TaskTracker(task_id=123, update_interval_seconds=60)
    operation_task = None
    ticker_task = None
    held_connection = None
    ticker_stop = threading.Event()
    ticker_ticks = 0
    event_loop_thread_id = threading.get_ident()
    worker_thread_id: int | None = None
    original_helper = getattr(tracker_module, sync_helper_name)

    def traced_helper(*args, **kwargs):
        nonlocal worker_thread_id
        worker_thread_id = threading.get_ident()
        return original_helper(*args, **kwargs)

    async def ticker():
        nonlocal ticker_ticks
        while not ticker_stop.is_set():
            await asyncio.sleep(0.01)
            ticker_ticks += 1

    try:
        await prepare(tracker)
        monkeypatch.setattr(tracker_module, sync_helper_name, traced_helper)
        held_connection = engine.connect()
        with gated_pool_checkout(engine) as gate:
            ticker_task = asyncio.create_task(ticker())
            operation_task = asyncio.create_task(operation(tracker))
            if operation_scheduled is not None:
                operation_scheduled.set()
            try:
                await gate.wait_until_contending()
                observed = await wait_for_ticks(lambda: ticker_ticks)
                assert observed >= LOOP_LIVENESS_TICKS
                assert not operation_task.done()
                assert worker_thread_id is not None
                assert worker_thread_id != event_loop_thread_id
            finally:
                held_connection.close()
                held_connection = None
                gate.let_through()
                # Drain even when cancelled before the worker reaches checkout.
                await drain_async_task_cancellation_safe(operation_task)
            result = operation_task.result()
            verify(tracker, session_factory, result)
    finally:
        ticker_stop.set()
        try:
            if held_connection is not None:
                held_connection.close()
            if ticker_task is not None:
                await drain_async_task_cancellation_safe(ticker_task)
        finally:
            try:
                if tracker.is_tracking:
                    await tracker.stop_periodic_updates()
            finally:
                engine.dispose()


class TestTaskTracker:
    """Test cases for TaskTracker."""

    @pytest.fixture
    def db_session(self, monkeypatch):
        """Fixture providing a mock database session."""
        from xagent.web.models import database

        session = MagicMock()

        # Mock Task object
        mock_task = MagicMock(spec=Task)
        mock_task.id = 123
        mock_task.status = TaskStatus.RUNNING
        mock_task.input_tokens = 0
        mock_task.output_tokens = 0
        mock_task.total_tokens = 0
        mock_task.llm_calls = 0
        mock_task.token_usage_details = None

        # Mock query to return the task
        session.query.return_value.filter.return_value.first.return_value = mock_task

        def apply_atomic_update(values, *, synchronize_session):
            assert synchronize_session is False
            for column, value in values.items():
                setattr(mock_task, column.key, value)
            return 1

        session.query.return_value.filter.return_value.update.side_effect = (
            apply_atomic_update
        )
        monkeypatch.setattr(database, "_SessionLocal", lambda: session)

        return session

    @pytest.fixture
    def task_tracker(self, db_session):
        """Fixture providing a TaskTracker instance."""
        return TaskTracker(task_id=123, db_session=db_session)

    @pytest.mark.asyncio
    async def test_init_task_tracker(self, db_session):
        """Test TaskTracker initialization."""
        tracker = TaskTracker(task_id=123, db_session=db_session)

        assert tracker.task_id == 123
        assert not hasattr(tracker, "db_session")
        assert not hasattr(tracker, "task")
        assert tracker.update_interval_seconds == 15  # default
        assert not tracker.is_tracking

    @pytest.mark.asyncio
    async def test_start_tracking_does_not_block_event_loop_when_pool_is_full(
        self, monkeypatch, tmp_path
    ):
        """A pool wait belongs in a worker thread, never on the event loop."""

        async def prepare(_tracker):
            return None

        def verify(tracker, _session_factory, _result):
            assert tracker.is_tracking

        await _assert_tracker_operation_keeps_loop_responsive_under_pool_pressure(
            monkeypatch,
            tmp_path,
            "tracker-start.db",
            "_load_task_seed_sync",
            prepare,
            lambda tracker: tracker.start_tracking(),
            verify,
        )

    @pytest.mark.asyncio
    async def test_periodic_update_does_not_block_event_loop_when_pool_is_full(
        self, monkeypatch, tmp_path
    ):
        """A periodic usage write must wait for the pool off the event loop."""

        async def prepare(tracker):
            await tracker.start_tracking()
            add_token_usage(input_tokens=10, output_tokens=5)

        def verify(tracker, session_factory, _result):
            assert tracker.is_tracking
            with session_factory() as db:
                task = db.get(Task, 123)
                assert task is not None
                assert (task.input_tokens, task.output_tokens, task.total_tokens) == (
                    10,
                    5,
                    15,
                )

        await _assert_tracker_operation_keeps_loop_responsive_under_pool_pressure(
            monkeypatch,
            tmp_path,
            "tracker-periodic.db",
            "_write_task_usage_sync",
            prepare,
            lambda tracker: tracker.periodic_update(),
            verify,
        )

    @pytest.mark.asyncio
    async def test_complete_tracking_does_not_block_event_loop_when_pool_is_full(
        self, monkeypatch, tmp_path
    ):
        """Final usage persistence must wait for the pool off the event loop."""

        async def prepare(tracker):
            await tracker.start_tracking()
            add_token_usage(input_tokens=20, output_tokens=10)

        def verify(tracker, session_factory, result):
            assert not tracker.is_tracking
            assert (result.input_tokens, result.output_tokens, result.total_tokens) == (
                20,
                10,
                30,
            )
            with session_factory() as db:
                task = db.get(Task, 123)
                assert task is not None
                assert (task.input_tokens, task.output_tokens, task.total_tokens) == (
                    20,
                    10,
                    30,
                )

        await _assert_tracker_operation_keeps_loop_responsive_under_pool_pressure(
            monkeypatch,
            tmp_path,
            "tracker-complete.db",
            "_complete_task_usage_sync",
            prepare,
            lambda tracker: tracker.complete_tracking(),
            verify,
        )

    @pytest.mark.parametrize(
        "inject_cleanup_error",
        [False, True],
        ids=["normal-cleanup", "cleanup-error"],
    )
    @pytest.mark.asyncio
    async def test_tracker_pool_helper_drains_delayed_worker(
        self, monkeypatch, tmp_path, inject_cleanup_error
    ):
        """Cancellation must drain a worker queued before it reaches checkout."""
        loop = asyncio.get_running_loop()
        executor = ThreadPoolExecutor(max_workers=1)
        original_executor = getattr(loop, "_default_executor")
        blocker_started = asyncio.Event()
        release_worker = threading.Event()
        operation_scheduled = asyncio.Event()
        worker_finished = threading.Event()
        from xagent.web.tracking import task_tracker as tracker_module

        original_seed = tracker_module._load_task_seed_sync

        def observed_seed(*args, **kwargs):
            try:
                return original_seed(*args, **kwargs)
            finally:
                worker_finished.set()

        monkeypatch.setattr(tracker_module, "_load_task_seed_sync", observed_seed)

        def occupy_default_executor():
            loop.call_soon_threadsafe(blocker_started.set)
            assert release_worker.wait(timeout=GUARD_TIMEOUT)

        async def prepare(_tracker):
            return None

        def verify(_tracker, _session_factory, _result):
            pytest.fail("cancelled helper completed normally")

        async def run_probe():
            blocker = loop.run_in_executor(None, occupy_default_executor)
            helper_task = None
            try:
                await asyncio.wait_for(blocker_started.wait(), timeout=GUARD_TIMEOUT)
                helper_task = asyncio.create_task(
                    _assert_tracker_operation_keeps_loop_responsive_under_pool_pressure(
                        monkeypatch,
                        tmp_path,
                        "tracker-delayed-worker.db",
                        "_load_task_seed_sync",
                        prepare,
                        lambda tracker: tracker.start_tracking(),
                        verify,
                        operation_scheduled=operation_scheduled,
                    )
                )
                await asyncio.wait_for(
                    operation_scheduled.wait(), timeout=GUARD_TIMEOUT
                )
                helper_task.cancel()
                release_worker.set()
                await blocker
                with pytest.raises(asyncio.CancelledError):
                    await helper_task

                assert helper_task.cancelled()
                assert worker_finished.is_set()
            finally:
                try:
                    release_worker.set()
                    if helper_task is not None and not helper_task.done():
                        helper_task.cancel()
                        await asyncio.gather(helper_task, return_exceptions=True)
                    if not blocker.done():
                        await blocker
                finally:
                    if inject_cleanup_error:
                        raise RuntimeError("injected cleanup failure")

        monkeypatch.setattr(loop, "_default_executor", executor)
        try:
            if inject_cleanup_error:
                with pytest.raises(RuntimeError, match="injected cleanup failure"):
                    await run_probe()
            else:
                await run_probe()
        finally:
            setattr(loop, "_default_executor", original_executor)
            executor.shutdown(wait=True)

        assert getattr(loop, "_default_executor") is original_executor
        with pytest.raises(RuntimeError):
            executor.submit(lambda: None)

    @pytest.mark.asyncio
    async def test_final_usage_does_not_overwrite_a_replacement_run(
        self, monkeypatch, tmp_path
    ):
        from xagent.web.models import database
        from xagent.web.services import quota_hooks

        engine = create_engine(
            f"sqlite:///{tmp_path / 'tracker-run-fence.db'}",
            connect_args={"check_same_thread": False},
        )
        Task.__table__.create(engine)
        session_factory = sessionmaker(
            autocommit=False,
            autoflush=False,
            bind=engine,
        )
        with session_factory() as db:
            db.add(
                Task(
                    id=123,
                    user_id=1,
                    title="run fence",
                    status=TaskStatus.RUNNING,
                    run_id="run-a",
                    input_tokens=4,
                    output_tokens=2,
                    total_tokens=6,
                    llm_calls=1,
                )
            )
            db.commit()

        monkeypatch.setattr(database, "_SessionLocal", session_factory)
        usage_hook_calls: list[int] = []
        quota_hooks.set_usage_record_hook(
            lambda _db, _user_id, _details, _actions, *, task_id=None: (
                usage_hook_calls.append(1)
            )
        )
        try:
            tracker = TaskTracker(task_id=123, expected_run_id="run-a")
            await tracker.start_tracking()
            add_token_usage(input_tokens=10, output_tokens=5)

            with session_factory() as replacement_db:
                replacement = replacement_db.get(Task, 123)
                assert replacement is not None
                replacement.run_id = "run-b"
                replacement.input_tokens = 100
                replacement.output_tokens = 50
                replacement.total_tokens = 150
                replacement_db.commit()

            await tracker.complete_tracking()
        finally:
            quota_hooks.set_usage_record_hook(None)

        with session_factory() as verify_db:
            replacement = verify_db.get(Task, 123)
            assert replacement is not None
            assert replacement.run_id == "run-b"
            assert replacement.input_tokens == 100
            assert replacement.output_tokens == 50
            assert replacement.total_tokens == 150
        assert usage_hook_calls == []
        engine.dispose()

    @pytest.mark.asyncio
    @pytest.mark.parametrize("takeover", ["runner", "attempt"])
    async def test_runner_fence_rejects_same_run_takeover_at_every_db_boundary(
        self, monkeypatch, tmp_path, takeover
    ):
        """A stale tracker must not seed or write after runner ownership changes."""
        from xagent.web.models import database
        from xagent.web.services import quota_hooks

        engine = create_engine(
            f"sqlite:///{tmp_path / 'tracker-runner-fence.db'}",
            connect_args={"check_same_thread": False},
        )
        Task.__table__.create(engine)
        session_factory = sessionmaker(
            autocommit=False,
            autoflush=False,
            bind=engine,
        )
        with session_factory() as db:
            for task_id in (123, 124, 125):
                db.add(
                    Task(
                        id=task_id,
                        user_id=1,
                        title=f"runner fence {task_id}",
                        status=TaskStatus.RUNNING,
                        run_id="run-a",
                        runner_id="runner-a",
                        lease_attempt_id="attempt-a",
                        input_tokens=4,
                        output_tokens=2,
                        total_tokens=6,
                        llm_calls=1,
                    )
                )
            db.commit()

        monkeypatch.setattr(database, "_SessionLocal", session_factory)
        usage_hook_calls: list[int] = []
        quota_hooks.set_usage_record_hook(
            lambda _db, _user_id, _details, _actions, *, task_id=None: (
                usage_hook_calls.append(1)
            )
        )
        periodic_tracker = None
        try:
            periodic_tracker = TaskTracker(
                task_id=123,
                update_interval_seconds=60,
                expected_run_id="run-a",
                expected_runner_id="runner-a",
                expected_attempt_id="attempt-a",
            )
            final_tracker = TaskTracker(
                task_id=124,
                update_interval_seconds=60,
                expected_run_id="run-a",
                expected_runner_id="runner-a",
                expected_attempt_id="attempt-a",
            )
            await periodic_tracker.start_tracking()
            await final_tracker.start_tracking()

            with session_factory() as takeover_db:
                for task_id in (123, 124, 125):
                    replacement = takeover_db.get(Task, task_id)
                    assert replacement is not None
                    if takeover == "runner":
                        replacement.runner_id = "runner-b"
                    else:
                        replacement.lease_attempt_id = "attempt-b"
                    replacement.input_tokens = 100
                    replacement.output_tokens = 50
                    replacement.total_tokens = 150
                takeover_db.commit()

            add_token_usage(input_tokens=10, output_tokens=5)
            await periodic_tracker.periodic_update()
            await final_tracker.complete_tracking()

            seed_tracker = TaskTracker(
                task_id=125,
                expected_run_id="run-a",
                expected_runner_id="runner-a",
                expected_attempt_id="attempt-a",
            )
            with pytest.raises(ValueError, match="Task 125.*not found"):
                await seed_tracker.start_tracking()
        finally:
            quota_hooks.set_usage_record_hook(None)
            if periodic_tracker is not None and periodic_tracker.is_tracking:
                await periodic_tracker.stop_periodic_updates()

        with session_factory() as verify_db:
            for task_id in (123, 124, 125):
                replacement = verify_db.get(Task, task_id)
                assert replacement is not None
                assert replacement.run_id == "run-a"
                assert replacement.runner_id == (
                    "runner-b" if takeover == "runner" else "runner-a"
                )
                assert replacement.input_tokens == 100
                assert replacement.output_tokens == 50
                assert replacement.total_tokens == 150
        assert not periodic_tracker.is_tracking
        assert usage_hook_calls == []
        engine.dispose()

    def test_periodic_write_cannot_overwrite_runner_takeover_after_ownership_read(
        self, monkeypatch, tmp_path
    ):
        """The ownership predicate and counter write must be one SQL statement."""
        owned, persisted = _run_tracker_ownership_race(
            monkeypatch,
            tmp_path,
            "tracker-periodic-cas.db",
            lambda tracker_module: tracker_module._write_task_usage_sync(
                123,
                TokenUsage(input_tokens=20, output_tokens=10, llm_calls=2),
                "run-a",
                "runner-a",
            ),
        )

        assert owned is False
        assert persisted == ("runner-b", 100, 50, 150)

    def test_final_usage_write_loses_takeover_ownership_race(
        self, monkeypatch, tmp_path
    ):
        """A stale runner must lose the durable fence before final persistence."""
        owned, persisted = _run_tracker_ownership_race(
            monkeypatch,
            tmp_path,
            "tracker-final-cas.db",
            lambda tracker_module: tracker_module._complete_task_usage_sync(
                123,
                TokenUsage(input_tokens=20, output_tokens=10, llm_calls=2),
                "run-a",
                "runner-a",
            ),
            pause_after_ownership_read=True,
        )

        assert owned is False
        assert persisted == ("runner-b", 100, 50, 150)

    @pytest.mark.asyncio
    async def test_usage_hook_pool_timeout_is_best_effort_after_counter_commit(
        self, monkeypatch, tmp_path, caplog
    ):
        """A metering checkout timeout must not reclassify a committed write."""
        import logging

        from xagent.web.models import database
        from xagent.web.services import quota_hooks

        engine = create_engine(
            f"sqlite:///{tmp_path / 'tracker-hook-pool-timeout.db'}",
            connect_args={"check_same_thread": False},
            poolclass=QueuePool,
            pool_size=1,
            max_overflow=0,
            pool_timeout=0.05,
        )
        Task.__table__.create(engine)
        session_factory = sessionmaker(
            autocommit=False,
            autoflush=False,
            bind=engine,
        )
        with session_factory() as db:
            db.add(
                Task(
                    id=123,
                    user_id=1,
                    title="best-effort usage hook",
                    status=TaskStatus.RUNNING,
                    run_id="run-a",
                    runner_id="runner-a",
                    lease_attempt_id="attempt-a",
                    input_tokens=4,
                    output_tokens=2,
                    total_tokens=6,
                    llm_calls=1,
                )
            )
            db.commit()

        monkeypatch.setattr(database, "_SessionLocal", session_factory)

        def pool_pressured_usage_hook(db, *_args, task_id=None):
            # The fenced counter commit has returned its connection. Occupy that
            # only slot before the hook's compatibility-session read, reproducing
            # a real QueuePool checkout timeout inside best-effort metering.
            held_connection = engine.connect()
            try:
                db.query(Task.id).first()
            finally:
                held_connection.close()

        quota_hooks.set_usage_record_hook(pool_pressured_usage_hook)
        try:
            tracker = TaskTracker(
                task_id=123,
                update_interval_seconds=60,
                expected_run_id="run-a",
                expected_runner_id="runner-a",
                expected_attempt_id="attempt-a",
            )
            await tracker.start_tracking()
            add_token_usage(input_tokens=16, output_tokens=8)

            with caplog.at_level(
                logging.WARNING,
                logger="xagent.web.tracking.task_tracker",
            ):
                await tracker.complete_tracking()

            assert "Quota usage recording failed for task 123" in caplog.text
            assert "QueuePool limit of size 1 overflow 0 reached" in caplog.text

            with session_factory() as verify_db:
                task = verify_db.get(Task, 123)
                assert task is not None
                assert task.input_tokens == 20
                assert task.output_tokens == 10
                assert task.total_tokens == 30
                assert task.llm_calls == 2
        finally:
            quota_hooks.set_usage_record_hook(None)
            engine.dispose()

    @pytest.mark.asyncio
    async def test_db_writes_run_off_loop_without_moving_quota_hooks(self, monkeypatch):
        """Database writes use workers without changing quota hook affinity."""
        from xagent.web.models import database
        from xagent.web.services import quota_hooks

        event_loop_thread = threading.get_ident()
        session_threads = []
        sessions = []
        hook_threads = []
        hook_sessions = []

        def create_session():
            session_threads.append(threading.get_ident())
            session = MagicMock()
            task = MagicMock(spec=Task)
            task.id = 123
            task.user_id = 7
            task.input_tokens = 0
            task.output_tokens = 0
            task.total_tokens = 0
            task.llm_calls = 0
            task.token_usage_details = None
            session.query.return_value.filter.return_value.first.return_value = task
            sessions.append(session)
            return session

        monkeypatch.setattr(database, "_SessionLocal", create_session)

        def progress_hook(db, user_id, delta_details, delta_actions):
            hook_threads.append(threading.get_ident())
            hook_sessions.append(db)
            return None

        def usage_hook(db, user_id, delta_details, delta_actions, *, task_id=None):
            hook_threads.append(threading.get_ident())
            hook_sessions.append(db)

        quota_hooks.set_run_progress_gate_hook(progress_hook)
        quota_hooks.set_usage_record_hook(usage_hook)
        try:
            tracker = TaskTracker(task_id=123, update_interval_seconds=60)
            await tracker.start_tracking()
            await tracker.periodic_update()
            assert await tracker.interrupt_reason_for_quota() is None
            await tracker.complete_tracking()
        finally:
            quota_hooks.set_run_progress_gate_hook(None)
            quota_hooks.set_usage_record_hook(None)

        assert len(hook_threads) == 2
        assert hook_threads == [event_loop_thread, event_loop_thread]
        assert len(hook_sessions) == 2
        assert len(sessions) == 5
        assert session_threads.count(event_loop_thread) == 2
        hook_session_ids = {id(session) for session in hook_sessions}
        for hook_session in hook_sessions:
            session_index = sessions.index(hook_session)
            assert session_threads[session_index] == event_loop_thread
        for session_index, session in enumerate(sessions):
            if id(session) not in hook_session_ids:
                assert session_threads[session_index] != event_loop_thread
        for session in sessions:
            session.close.assert_called_once_with()

    @pytest.mark.asyncio
    async def test_completion_waits_for_inflight_periodic_worker(
        self, db_session, monkeypatch
    ):
        """An uncancellable thread write must finish before the final write."""
        from xagent.web.tracking import task_tracker as tracker_module

        periodic_started = threading.Event()
        release_periodic = threading.Event()
        writes = []

        def blocking_periodic_write(*_args):
            periodic_started.set()
            assert release_periodic.wait(timeout=1)
            writes.append("periodic")
            return True

        def final_write(*_args):
            writes.append("complete")
            return True

        monkeypatch.setattr(
            tracker_module, "_write_task_usage_sync", blocking_periodic_write
        )
        monkeypatch.setattr(tracker_module, "_complete_task_usage_sync", final_write)

        tracker = TaskTracker(
            task_id=123,
            db_session=db_session,
            update_interval_seconds=0.01,
        )
        await tracker.start_tracking()
        assert await asyncio.to_thread(periodic_started.wait, 1)

        completion = asyncio.create_task(tracker.complete_tracking())
        try:
            await asyncio.sleep(0.02)
            assert not completion.done()
            release_periodic.set()
            await asyncio.wait_for(completion, timeout=1)
        finally:
            release_periodic.set()
            if not completion.done():
                await asyncio.wait_for(completion, timeout=1)

        assert writes == ["periodic", "complete"]

    @pytest.mark.asyncio
    async def test_periodic_pool_timeout_does_not_checkout_again_or_stop_tracking(
        self, db_session, monkeypatch
    ):
        """A transient pool timeout is one failed write, not a second probe."""
        from xagent.web.tracking import task_tracker as tracker_module

        attempts = 0

        def write_with_one_timeout(*_args):
            nonlocal attempts
            attempts += 1
            if attempts == 1:
                raise SQLAlchemyTimeoutError("pool exhausted")
            return True

        tracker = TaskTracker(
            task_id=123,
            db_session=db_session,
            update_interval_seconds=60,
        )
        await tracker.start_tracking()
        monkeypatch.setattr(
            tracker_module,
            "_write_task_usage_sync",
            write_with_one_timeout,
        )
        monkeypatch.setattr(
            tracker_module,
            "_new_short_session",
            lambda: pytest.fail("periodic failure performed a second checkout"),
        )

        try:
            await tracker.periodic_update()
            assert tracker.is_tracking

            await tracker.periodic_update()
            assert tracker.is_tracking
            assert attempts == 2
        finally:
            await tracker.stop_periodic_updates()

    @pytest.mark.asyncio
    async def test_final_pool_timeout_is_reported_to_the_lease_owner(
        self, db_session, monkeypatch
    ):
        """The lease owner must be able to avoid a second checkout after timeout."""
        from xagent.web.tracking import task_tracker as tracker_module

        attempts = 0

        def final_write(*_args):
            nonlocal attempts
            attempts += 1
            raise SQLAlchemyTimeoutError("pool exhausted")

        tracker = TaskTracker(
            task_id=123,
            db_session=db_session,
            update_interval_seconds=60,
        )
        await tracker.start_tracking()
        monkeypatch.setattr(
            tracker_module,
            "_complete_task_usage_sync",
            final_write,
        )

        with pytest.raises(SQLAlchemyTimeoutError, match="pool exhausted"):
            await tracker.complete_tracking()

        assert attempts == 1
        assert not tracker.is_tracking

    @pytest.mark.asyncio
    async def test_periodic_missing_row_stops_tracking_without_a_followup_checkout(
        self, db_session, monkeypatch
    ):
        """The write helper's False result is the only missing-row signal."""
        from xagent.web.tracking import task_tracker as tracker_module

        tracker = TaskTracker(
            task_id=123,
            db_session=db_session,
            update_interval_seconds=60,
        )
        await tracker.start_tracking()
        monkeypatch.setattr(
            tracker_module,
            "_write_task_usage_sync",
            lambda *_args: False,
        )
        monkeypatch.setattr(
            tracker_module,
            "_new_short_session",
            lambda: pytest.fail("missing-row result performed a second checkout"),
        )

        try:
            await tracker.periodic_update()
            assert not tracker.is_tracking
        finally:
            await tracker.stop_periodic_updates()

    @pytest.mark.asyncio
    async def test_cancelled_completion_drains_periodic_then_persists_final_usage(
        self, db_session, monkeypatch
    ):
        """Cancellation is propagated only after stale writes can no longer win."""
        from xagent.web.tracking import task_tracker as tracker_module

        periodic_started = threading.Event()
        release_periodic = threading.Event()
        writes = []

        def blocking_periodic_write(*_args):
            periodic_started.set()
            assert release_periodic.wait(timeout=1)
            writes.append("periodic")
            return True

        def final_write(*_args):
            writes.append("complete")
            return True

        monkeypatch.setattr(
            tracker_module,
            "_write_task_usage_sync",
            blocking_periodic_write,
        )
        monkeypatch.setattr(tracker_module, "_complete_task_usage_sync", final_write)

        tracker = TaskTracker(
            task_id=123,
            db_session=db_session,
            update_interval_seconds=0.01,
        )
        await tracker.start_tracking()
        assert await asyncio.to_thread(periodic_started.wait, 1)

        completion = asyncio.create_task(tracker.complete_tracking())
        try:
            await asyncio.sleep(0.02)
            completion.cancel()
            await asyncio.sleep(0.02)

            assert not completion.done()
            assert writes == []

            release_periodic.set()
            with pytest.raises(asyncio.CancelledError):
                await asyncio.wait_for(completion, timeout=1)
        finally:
            release_periodic.set()
            if not completion.done():
                try:
                    await asyncio.wait_for(asyncio.shield(completion), timeout=1)
                except BaseException:
                    pass

        assert completion.cancelled()
        assert writes == ["periodic", "complete"]

    @pytest.mark.asyncio
    async def test_complete_tracking_reports_only_current_turn_delta(self, db_session):
        """The usage-record hook must receive only this turn's delta, not the
        re-seeded prior-turn baseline (multi-turn tasks seed from the DB)."""
        from xagent.core.model.chat.token_context import add_tool_call_usage
        from xagent.web.services import quota_hooks

        task = db_session.query.return_value.filter.return_value.first.return_value
        task.user_id = 42
        # Prior-turn state seeded from the DB row.
        task.input_tokens = 100
        task.output_tokens = 50
        task.llm_calls = 1
        task.token_usage_details = [
            {"type": "input", "tokens": 100, "model": "m", "call_type": "chat"},
            {"type": "output", "tokens": 50, "model": "m", "call_type": "chat"},
        ]

        captured = {}

        def _hook(db, user_id, delta_details, delta_actions, *, task_id=None):
            captured.update(
                user_id=user_id,
                details=delta_details,
                actions=delta_actions,
                task_id=task_id,
            )

        quota_hooks.set_usage_record_hook(_hook)
        try:
            tracker = TaskTracker(task_id=123, db_session=db_session)
            await tracker.start_tracking()
            # This turn's usage, appended on top of the seeded baseline.
            add_token_usage(
                input_tokens=10, output_tokens=5, model="m", call_type="chat"
            )
            add_tool_call_usage(3)
            await tracker.complete_tracking()
        finally:
            quota_hooks.set_usage_record_hook(None)

        assert captured["user_id"] == 42
        # The metered run's task, so the hook can attribute the usage to it.
        assert captured["task_id"] == 123
        assert captured["actions"] == 3  # only this turn's tool calls (baseline was 0)
        # Only the two entries appended this turn, not the two seeded ones.
        assert len(captured["details"]) == 2
        assert sorted(d["tokens"] for d in captured["details"]) == [5, 10]

    @pytest.mark.asyncio
    async def test_interrupt_reason_for_quota_passes_turn_delta(self, db_session):
        """The per-step quota gate must see the same this-turn delta the metering
        path computes, and surface the gate's reason (or None when open)."""
        from xagent.core.model.chat.token_context import add_tool_call_usage
        from xagent.web.services import quota_hooks

        task = db_session.query.return_value.filter.return_value.first.return_value
        task.user_id = 42
        task.input_tokens = 100
        task.output_tokens = 50
        task.llm_calls = 1
        task.token_usage_details = [
            {"type": "input", "tokens": 100, "model": "m", "call_type": "chat"},
        ]

        captured = {}

        def _gate(db, user_id, delta_details, delta_actions):
            captured.update(
                user_id=user_id, details=delta_details, actions=delta_actions
            )
            return "over credits" if delta_actions >= 2 else None

        quota_hooks.set_run_progress_gate_hook(_gate)
        try:
            tracker = TaskTracker(task_id=123, db_session=db_session)
            # Before tracking starts, the checker is a no-op (fails open).
            assert await tracker.interrupt_reason_for_quota() is None
            await tracker.start_tracking()
            add_token_usage(
                input_tokens=10, output_tokens=5, model="m", call_type="chat"
            )
            add_tool_call_usage(2)
            reason = await tracker.interrupt_reason_for_quota()
        finally:
            quota_hooks.set_run_progress_gate_hook(None)

        assert reason == "over credits"
        # The reason is recorded so the run's caller can surface why it stopped.
        assert tracker.quota_interrupt_reason == "over credits"
        assert captured["user_id"] == 42
        assert captured["actions"] == 2  # only this turn's tool calls
        # Only this turn's input+output entries, not the seeded baseline one.
        assert len(captured["details"]) == 2
        assert sorted(d["tokens"] for d in captured["details"]) == [5, 10]

    @pytest.mark.asyncio
    async def test_quota_gate_caches_user_id_and_logs_once(self, db_session, caplog):
        """The tracker caches user_id and logs gate failures once per run."""
        import logging

        from xagent.web.services import quota_hooks

        task = db_session.query.return_value.filter.return_value.first.return_value
        task.user_id = 7

        def _boom(db, user_id, dd, da):
            raise RuntimeError("gate infra down")

        quota_hooks.set_run_progress_gate_hook(_boom)
        try:
            tracker = TaskTracker(task_id=123, db_session=db_session)
            assert tracker._user_id == 7
            await tracker.start_tracking()
            with caplog.at_level(logging.WARNING):
                assert await tracker.interrupt_reason_for_quota() is None
                assert await tracker.interrupt_reason_for_quota() is None
                assert await tracker.interrupt_reason_for_quota() is None
        finally:
            quota_hooks.set_run_progress_gate_hook(None)

        warnings = [r for r in caplog.records if "failed open" in r.getMessage()]
        assert len(warnings) == 1
        assert tracker.quota_interrupt_reason is None  # never tripped → no reason

    @pytest.mark.asyncio
    async def test_runtime_should_interrupt_drives_tracker_gate(self, db_session):
        """End-to-end seam: the runtime's should_interrupt — the exact call the
        pattern loop makes at each safe point — wired to the tracker's real gate
        method fires and records the reason when a registered hook trips."""
        from xagent.core.agent.runtime import PatternRuntime
        from xagent.web.services import quota_hooks

        task = db_session.query.return_value.filter.return_value.first.return_value
        task.user_id = 5

        quota_hooks.set_run_progress_gate_hook(lambda db, uid, dd, da: "Out of credits")
        try:
            tracker = TaskTracker(task_id=123, db_session=db_session)
            await tracker.start_tracking()
            runtime = PatternRuntime(
                interrupt_checker=tracker.interrupt_reason_for_quota
            )
            assert await runtime.should_interrupt() is True
            assert runtime.interrupt_reason == "Out of credits"
            assert tracker.quota_interrupt_reason == "Out of credits"
        finally:
            quota_hooks.set_run_progress_gate_hook(None)

    @pytest.mark.asyncio
    async def test_init_task_tracker_with_custom_interval(self, db_session):
        """Test TaskTracker with custom update interval."""
        tracker = TaskTracker(
            task_id=123, db_session=db_session, update_interval_seconds=60
        )

        assert tracker.update_interval_seconds == 60

    @pytest.mark.asyncio
    async def test_init_task_tracker_task_not_found(self, db_session):
        """Test TaskTracker with non-existent task."""
        # Mock query to return None (task not found)
        db_session.query.return_value.filter.return_value.first.return_value = None

        with pytest.raises(ValueError, match="Task 123 not found"):
            TaskTracker(task_id=123, db_session=db_session)

    @pytest.mark.asyncio
    async def test_start_tracking(self, task_tracker):
        """Test starting token tracking."""
        # Add some tokens before starting
        add_token_usage(input_tokens=10, output_tokens=5)

        await task_tracker.start_tracking()

        # Should reset token usage
        usage = get_token_usage()
        assert usage.input_tokens == 0
        assert usage.output_tokens == 0
        assert task_tracker.is_tracking

    @pytest.mark.asyncio
    async def test_start_tracking_uses_existing_task_totals(self, db_session):
        mock_task = db_session.query.return_value.filter.return_value.first.return_value
        mock_task.input_tokens = 120
        mock_task.output_tokens = 80
        mock_task.llm_calls = 4
        mock_task.token_usage_details = [{"type": "input", "tokens": 120}]

        tracker = TaskTracker(task_id=123, db_session=db_session)
        await tracker.start_tracking()

        usage = get_token_usage()
        assert usage.input_tokens == 120
        assert usage.output_tokens == 80
        assert usage.llm_calls == 4
        assert usage.details == [{"type": "input", "tokens": 120}]

    @pytest.mark.asyncio
    async def test_start_tracking_already_tracking(self, task_tracker, caplog):
        """Test starting tracking when already tracking."""
        with patch("xagent.web.tracking.task_tracker.logger.warning") as mock_warning:
            await task_tracker.start_tracking()

            # Try to start again
            await task_tracker.start_tracking()

        # Should log warning
        assert mock_warning.called
        assert "already being tracked" in mock_warning.call_args.args[0]

    @pytest.mark.asyncio
    async def test_periodic_update(self, task_tracker, db_session):
        """Test periodic database update."""
        await task_tracker.start_tracking()

        # Add some tokens
        add_token_usage(input_tokens=100, output_tokens=50)

        # Perform periodic update
        await task_tracker.periodic_update()

        # Verify database was updated
        task = db_session.query.return_value.filter.return_value.first.return_value
        assert task.input_tokens == 100
        assert task.output_tokens == 50
        assert task.total_tokens == 150
        assert task.llm_calls == 1
        db_session.commit.assert_called_once()
        db_session.close.assert_called_once()

    @pytest.mark.asyncio
    async def test_periodic_update_not_tracking(self, task_tracker, caplog):
        """Test periodic update when not tracking."""
        # Don't start tracking

        with patch("xagent.web.tracking.task_tracker.logger.warning") as mock_warning:
            await task_tracker.periodic_update()

        # Should log warning
        assert mock_warning.called
        assert "not being tracked" in mock_warning.call_args.args[0]

    @pytest.mark.asyncio
    async def test_complete_tracking(self, task_tracker, db_session):
        """Test completing tracking."""
        await task_tracker.start_tracking()

        # Add some tokens
        add_token_usage(input_tokens=200, output_tokens=100)
        add_token_usage(input_tokens=50, output_tokens=25)

        # Complete tracking
        usage = await task_tracker.complete_tracking()

        # Verify usage was returned
        assert usage.input_tokens == 250
        assert usage.output_tokens == 125
        assert usage.total_tokens == 375
        assert usage.llm_calls == 2

        # Verify database was updated
        task = db_session.query.return_value.filter.return_value.first.return_value
        assert task.input_tokens == 250
        assert task.output_tokens == 125
        assert task.total_tokens == 375
        assert task.llm_calls == 2
        db_session.commit.assert_called_once()
        # The fenced task write and compatibility quota callback each own one
        # short-lived Session.
        assert db_session.close.call_count == 2

        assert not task_tracker.is_tracking

    @pytest.mark.asyncio
    async def test_complete_tracking_not_started(self, task_tracker):
        """Test completing tracking without starting."""
        # Don't start tracking

        with pytest.raises(RuntimeError, match="not being tracked"):
            await task_tracker.complete_tracking()

    @pytest.mark.asyncio
    async def test_get_current_usage(self, task_tracker):
        """Test getting current usage without stopping."""
        await task_tracker.start_tracking()

        add_token_usage(input_tokens=30, output_tokens=15)

        usage = task_tracker.get_current_usage()

        assert usage.input_tokens == 30
        assert usage.output_tokens == 15
        assert task_tracker.is_tracking  # Should still be tracking

    @pytest.mark.asyncio
    async def test_start_stop_periodic_updates(self, task_tracker):
        """Test starting and stopping periodic background updates."""
        await task_tracker.start_tracking()

        # Start periodic updates
        await task_tracker.start_periodic_updates()

        assert task_tracker.is_tracking

        # Stop periodic updates
        await task_tracker.stop_periodic_updates()

        assert not task_tracker.is_tracking

    @pytest.mark.asyncio
    async def test_start_periodic_updates_already_active(self, task_tracker, caplog):
        """Test starting periodic updates when already active."""
        with patch("xagent.web.tracking.task_tracker.logger.warning") as mock_warning:
            await task_tracker.start_tracking()
            await task_tracker.start_periodic_updates()

            # Try to start again
            await task_tracker.start_periodic_updates()

        # Should log warning
        assert mock_warning.called
        assert "already active" in mock_warning.call_args.args[0]

    @pytest.mark.asyncio
    async def test_turn_usage_reports_this_run_over_the_seed(self, db_session):
        """``turn_usage`` subtracts every seeded counter and groups this run's
        detail rows by model id, keeping two ids that share a name apart."""
        from xagent.core.model.chat.token_context import (
            add_tool_call_usage,
            add_tool_error_usage,
        )
        from xagent.web.tracking.task_tracker import _TaskTrackingSeed

        tracker = TaskTracker(task_id=123, db_session=db_session)
        # A prior turn left every counter non-zero, including tool errors the
        # database row has no column for.
        tracker._seed = _TaskTrackingSeed(
            user_id=None,
            usage=TokenUsage(
                input_tokens=100,
                output_tokens=50,
                llm_calls=4,
                tool_calls=6,
                tool_errors=2,
                details=[
                    {"type": "input", "tokens": 100, "model": "m", "model_id": "1"},
                    {"type": "output", "tokens": 50, "model": "m", "model_id": "1"},
                ],
            ),
        )
        await tracker.start_tracking()
        add_token_usage(
            input_tokens=10,
            output_tokens=5,
            model="shared-name",
            model_id="a",
            call_type="chat",
        )
        add_token_usage(
            input_tokens=7,
            output_tokens=3,
            model="shared-name",
            model_id="b",
            call_type="chat",
        )
        add_tool_call_usage(3)
        add_tool_error_usage(1)
        usage = await tracker.complete_tracking()

        turn = tracker.turn_usage(usage)

        assert turn.input_tokens == 17
        assert turn.output_tokens == 8
        assert turn.llm_calls == 2
        assert turn.tool_calls == 3
        assert turn.tool_errors == 1
        assert [
            (m.model_id, m.model_name, m.input_tokens, m.output_tokens)
            for m in turn.by_model
        ] == [("a", "shared-name", 10, 5), ("b", "shared-name", 7, 3)]
        # Without an argument it reads the current context's usage.
        assert tracker.turn_usage() == turn

    @pytest.mark.asyncio
    @pytest.mark.parametrize("persisted", [True, False])
    async def test_turn_usage_sink_receives_usage_once_per_completion(
        self, db_session, monkeypatch, persisted
    ):
        """The sink runs whether or not the final write was persisted, and a
        failing sink never changes what completion returns."""
        from xagent.web.tracking import task_tracker as tracker_module

        monkeypatch.setattr(
            tracker_module,
            "_complete_task_usage_sync",
            lambda *_args, **_kwargs: persisted,
        )
        received = []
        tracker = TaskTracker(
            task_id=123, db_session=db_session, turn_usage_sink=received.append
        )
        await tracker.start_tracking()
        add_token_usage(input_tokens=4, output_tokens=2, model="m", call_type="chat")

        usage = await tracker.complete_tracking()

        assert len(received) == 1
        assert received[0] == tracker.turn_usage(usage)
        assert (received[0].input_tokens, received[0].output_tokens) == (4, 2)

    @pytest.mark.asyncio
    async def test_turn_usage_sink_runs_when_the_final_write_raises(
        self, db_session, monkeypatch
    ):
        """A final write that raises something other than pool exhaustion is
        only logged: the usage was still observed, so the sink receives it."""
        from xagent.web.tracking import task_tracker as tracker_module

        def failing_write(*_args, **_kwargs):
            raise RuntimeError("db unavailable")

        monkeypatch.setattr(tracker_module, "_complete_task_usage_sync", failing_write)
        received = []
        tracker = TaskTracker(
            task_id=123, db_session=db_session, turn_usage_sink=received.append
        )
        await tracker.start_tracking()
        add_token_usage(input_tokens=4, output_tokens=2, model="m", call_type="chat")

        usage = await tracker.complete_tracking()

        assert len(received) == 1
        assert received[0] == tracker.turn_usage(usage)

    @pytest.mark.asyncio
    async def test_failing_turn_usage_sink_is_only_logged(self, db_session):
        def sink(_usage):
            raise RuntimeError("sink broke")

        tracker = TaskTracker(task_id=123, db_session=db_session, turn_usage_sink=sink)
        await tracker.start_tracking()
        add_token_usage(input_tokens=4, output_tokens=2, model="m", call_type="chat")

        with patch("xagent.web.tracking.task_tracker.logger.warning") as warning:
            usage = await tracker.complete_tracking()

        assert usage.input_tokens == 4
        assert warning.call_count == 1


class TestTaskTrackerManager:
    """Test cases for TaskTrackerManager."""

    @pytest.fixture
    def manager(self):
        """Fixture providing a TaskTrackerManager."""
        return TaskTrackerManager()

    @pytest.fixture
    def mock_session(self, monkeypatch):
        """Fixture providing a mock database session."""
        from xagent.web.models import database

        session = MagicMock()

        # Mock Task object
        mock_task = MagicMock(spec=Task)
        mock_task.id = 1
        mock_task.status = TaskStatus.RUNNING

        session.query.return_value.filter.return_value.first.return_value = mock_task
        monkeypatch.setattr(database, "_SessionLocal", lambda: session)

        return session

    @pytest.mark.asyncio
    async def test_get_or_create_tracker_new(self, manager, mock_session):
        """Test creating a new tracker."""
        tracker = manager.get_or_create_tracker(task_id=1, db_session=mock_session)

        assert tracker is not None
        assert tracker.task_id == 1
        assert 1 in manager._trackers

    @pytest.mark.asyncio
    async def test_get_or_create_tracker_existing(self, manager, mock_session):
        """Test getting existing tracker."""
        # Create tracker first
        tracker1 = manager.get_or_create_tracker(task_id=1, db_session=mock_session)

        # Get same tracker again
        tracker2 = manager.get_or_create_tracker(task_id=1, db_session=mock_session)

        # Should return the same instance
        assert tracker1 is tracker2

    @pytest.mark.asyncio
    async def test_get_tracker(self, manager, mock_session):
        """Test getting tracker without creating."""
        # Non-existent tracker
        tracker = manager.get_tracker(task_id=1)
        assert tracker is None

        # Create tracker
        manager.get_or_create_tracker(task_id=1, db_session=mock_session)

        # Now it exists
        tracker = manager.get_tracker(task_id=1)
        assert tracker is not None

    @pytest.mark.asyncio
    async def test_complete_tracker(self, manager, mock_session):
        """Test completing a specific tracker."""
        # Create and start tracker
        tracker = manager.get_or_create_tracker(task_id=1, db_session=mock_session)
        await tracker.start_tracking()

        # Add tokens
        add_token_usage(input_tokens=10, output_tokens=5)

        # Complete the tracker
        usage = await manager.complete_tracker(task_id=1)

        # Verify usage
        assert usage.input_tokens == 10
        assert usage.output_tokens == 5

        # Tracker should be removed
        assert 1 not in manager._trackers

    @pytest.mark.asyncio
    async def test_complete_tracker_nonexistent(self, manager):
        """Test completing non-existent tracker."""
        usage = await manager.complete_tracker(task_id=999)

        # Should return None
        assert usage is None

    @pytest.mark.asyncio
    async def test_complete_all(self, manager, mock_session):
        """Test completing all trackers."""
        # Create multiple trackers with independent token tracking
        # Note: In real usage, each task would have its own execution context
        # Here we simulate by manually tracking tokens per task

        # Trackers are created but tokens accumulate in shared context
        for i in range(1, 4):
            tracker = manager.get_or_create_tracker(task_id=i, db_session=MagicMock())
            await tracker.start_tracking()

        # Complete all (will have 0 tokens since we didn't add any after last reset)
        results = await manager.complete_all()

        # Verify all trackers completed
        assert len(results) == 3
        assert 1 in results
        assert 2 in results
        assert 3 in results

        # All trackers should be removed
        assert len(manager._trackers) == 0

    @pytest.mark.asyncio
    async def test_multiple_tasks_independent(self, manager):
        """Test that multiple task trackers can be created and managed independently."""
        # Mock different sessions
        session1 = MagicMock()
        session2 = MagicMock()

        for i, session in enumerate([session1, session2], 1):
            mock_task = MagicMock(spec=Task)
            mock_task.id = i
            session.query.return_value.filter.return_value.first.return_value = (
                mock_task
            )

        # Create two trackers
        tracker1 = manager.get_or_create_tracker(task_id=1, db_session=session1)
        tracker2 = manager.get_or_create_tracker(task_id=2, db_session=session2)

        # Verify both are tracked independently
        assert tracker1.task_id == 1
        assert tracker2.task_id == 2
        assert len(manager._trackers) == 2


class TestTaskTrackerIntegration:
    """Integration tests for TaskTracker with real token tracking."""

    @pytest.fixture
    def db_session(self, monkeypatch):
        """Fixture providing a mock database session."""
        from xagent.web.models import database

        session = MagicMock()

        mock_task = MagicMock(spec=Task)
        mock_task.id = 123
        mock_task.status = TaskStatus.RUNNING
        mock_task.input_tokens = 0
        mock_task.output_tokens = 0
        mock_task.total_tokens = 0
        mock_task.llm_calls = 0
        mock_task.token_usage_details = None

        session.query.return_value.filter.return_value.first.return_value = mock_task
        monkeypatch.setattr(database, "_SessionLocal", lambda: session)

        return session

    @pytest.mark.asyncio
    async def test_full_tracking_workflow(self, db_session):
        """Test complete tracking workflow."""
        tracker = TaskTracker(task_id=123, db_session=db_session)

        # Start tracking
        await tracker.start_tracking()

        # Simulate LLM calls
        add_token_usage(
            input_tokens=100, output_tokens=50, model="gpt-4", call_type="chat"
        )
        add_token_usage(
            input_tokens=200, output_tokens=100, model="gpt-4", call_type="chat"
        )

        # Check current usage
        usage = tracker.get_current_usage()
        assert usage.input_tokens == 300
        assert usage.output_tokens == 150

        # Complete tracking
        final_usage = await tracker.complete_tracking()

        # Verify final stats
        assert final_usage.input_tokens == 300
        assert final_usage.output_tokens == 150
        assert final_usage.llm_calls == 2  # Two add_token_usage calls
        assert len(final_usage.details) == 4  # 2 input + 2 output entries

        # Verify database was updated
        db_session.commit.assert_called()

    @pytest.mark.asyncio
    async def test_tracking_with_details(self, db_session):
        """Test tracking with detailed token information."""
        tracker = TaskTracker(task_id=123, db_session=db_session)

        await tracker.start_tracking()

        # Add tokens with details
        add_token_usage(
            input_tokens=100,
            output_tokens=50,
            model="gpt-4",
            call_type="chat",
        )
        add_token_usage(
            input_tokens=50,
            output_tokens=25,
            model="gpt-3.5-turbo",
            call_type="stream_chat",
        )

        final_usage = await tracker.complete_tracking()

        # Verify details are tracked (4 entries: 2 input + 2 output)
        assert len(final_usage.details) == 4

        # Details are accumulated, check both models are present
        models = [d.get("model") for d in final_usage.details]
        assert "gpt-4" in models
        assert "gpt-3.5-turbo" in models

        # Check both input and output types are present
        types = [d.get("type") for d in final_usage.details]
        assert "input" in types
        assert "output" in types


def test_check_run_progress_gate_guards():
    """The quota-gate seam is a no-op when no hook is registered or user_id is
    None, and otherwise forwards to the hook and returns its reason."""
    from xagent.web.services import quota_hooks

    # No hook registered → None regardless of args.
    quota_hooks.set_run_progress_gate_hook(None)
    assert quota_hooks.check_run_progress_gate("db", 1, [], 0) is None

    seen = []
    quota_hooks.set_run_progress_gate_hook(
        lambda db, uid, dd, da: (seen.append(uid), "OVER")[1]
    )
    try:
        # user_id None short-circuits before the hook is called.
        assert quota_hooks.check_run_progress_gate("db", None, [], 0) is None
        assert seen == []
        # Otherwise the hook runs and its reason is returned verbatim.
        assert quota_hooks.check_run_progress_gate("db", 7, [], 3) == "OVER"
        assert seen == [7]
    finally:
        quota_hooks.set_run_progress_gate_hook(None)