Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

18.7 KB
Raw Blame
"""Every thread-starting call in ``web`` and ``core/tools`` is tracked or audited.

``asyncio.to_thread`` and friends start a thread that a cancelled awaiter does
not stop, and the thread may still write after ``app.retire_writers`` reported
success. The guarantee of ``retire_writers`` is therefore exactly: the calls
that go through ``run_tracked`` (web) or ``run_blocking`` (core), plus the
files audited below. This test fails on any thread-starting call that is
neither, so a new one cannot slip in unreviewed.

Detected: ``to_thread``, ``run_in_executor``, ``anyio.to_thread.run_sync``,
``run_in_thread_with_telemetry``, ``threading.Thread(...)`` and
``ThreadPoolExecutor(...)`` (and so every ``submit`` on a pool built there).
Not detected, on purpose: ``run_db_io_cancellation_safe`` and
``run_auth_db_worker``. They hold the worker until it has returned, even when
the awaiting task is cancelled, so a cancelled and awaited task owns no live
thread.

Each entry below is keyed by file with the exact number of call sites audited.
The count is exact in both directions: adding a call to an audited file fails,
and so does tracking one (lower the number, so the audit stays true).

The reasons name which of the documented ``retire_writers`` "Not covered"
classes a file belongs to, or why it cannot write at all. They were checked
against the code, not copied from a template; if a call site changes meaning,
change the reason with it.
"""

from __future__ import annotations

import ast
from pathlib import Path

import pytest

SRC = Path(__file__).resolve().parents[2] / "src" / "xagent"
SCANNED = ("web", "core/tools")
THREAD_CALLS = {
    "to_thread",
    "run_in_executor",
    "run_in_thread_with_telemetry",
    "Thread",
    "ThreadPoolExecutor",
    "run_sync",
}
# The tracker itself.
IMPLEMENTATION = {"web/services/writer_workers.py"}

HTTP = (
    "HTTP route handler thread. HTTP producers are listed under 'Not covered' "
    "in retire_writers: the host stops routing HTTP traffic before it relies "
    "on the retire."
)
EXEC = (
    "Runs inside a running execution (task coordinator or a tool call). It may "
    "write, but executions are owned by the task coordinators and "
    "background_task_manager that execution_status() reports and shutdown "
    "drains; retire_writers lists them under 'Not covered'."
)
FILES = (
    "Filesystem or workspace-file work for a running execution or an HTTP "
    "route (no database, vector-store or shared-state write); both are listed "
    "under 'Not covered' in retire_writers."
)
READ = "Read-only or pure computation: no database, vector-store or shared write."
NETWORK = "Outbound network call (DNS, provider or Google API read): writes no local shared state."

ALLOWED: dict[str, tuple[int, str]] = {
    # -- web/api: HTTP routes ---------------------------------------------------
    "web/api/admin_users.py": (6, HTTP),
    "web/api/auth.py": (10, HTTP),
    "web/api/chat.py": (8, HTTP),
    "web/api/files.py": (7, HTTP),
    "web/api/kb.py": (16, HTTP),
    "web/api/mcp.py": (5, HTTP),
    "web/api/model.py": (2, HTTP),
    "web/api/skill_hub.py": (6, HTTP),
    "web/api/tools.py": (2, HTTP),
    "web/api/triggers.py": (8, HTTP),
    "web/api/v1/skills.py": (1, HTTP),
    "web/api/v1/tasks.py": (1, HTTP),
    "web/api/workforces.py": (2, HTTP),
    # -- web: lifecycle and telemetry -------------------------------------------
    "web/app.py": (
        4,
        "Four calls: the startup check of the auth-attempt storage, which is "
        "awaited before the lifespan starts any writer; the temp-file "
        "walk, which deletes temp files only and is stopped through its own "
        "event (documented in retire_writers); the dedicated "
        "'startup-file-sync' pool, whose only writing job (the startup file "
        "storage sync) is submitted through run_tracked_in and so is counted "
        "until its thread returns; and the standby preflight, a read-only "
        "classification that writes nothing and is stopped through its "
        "StartupSyncControl cancel event.",
    ),
    "web/worker.py": (
        4,
        "Worker-process startup checks (schema, key and engine validation), "
        "awaited before the worker serves; the web retire does not run in a "
        "worker and never overlaps them.",
    ),
    "web/runtime_telemetry.py": (
        1,
        "Telemetry exporter shutdown; no application data.",
    ),
    "web/tracing.py": (
        1,
        "Reads one committed tool outcome from the checkpoint store for a "
        "running execution (load); writes nothing.",
    ),
    "web/tools/config.py": (1, NETWORK),
    "web/tools/mcp/composio_proxy.py": (1, NETWORK),
    "web/tools/mcp/google_api_proxy.py": (1, NETWORK),
    "web/tools/mcp/google_slides.py": (3, NETWORK),
    "web/tracking/standalone_usage.py": (
        1,
        "The host's usage gate hook, called before work starts; the usage "
        "delivery that writes is tracked separately (run_tracked in "
        "UsageScope).",
    ),
    # -- web/channels: reads and sends; the writing threads are tracked ---------
    "web/channels/feishu/bot.py": (
        4,
        "Two file downloads into a temp or staging directory and two message "
        "create/patch calls to the Feishu API; the database-writing threads of "
        "this file (input lookup and acceptance, upload staging) are tracked.",
    ),
    "web/channels/feishu/trace_handler.py": (
        2,
        "Message create/patch calls to the Feishu API; no local shared write.",
    ),
    "web/channels/telegram/bot.py": (
        1,
        "Resolves the user's ASR model in one short read-only session; the "
        "writing threads of this file (input lookup and acceptance, upload "
        "staging) are tracked.",
    ),
    # -- web/models, web/services: helpers and execution-owned work -------------
    "web/models/auth_database.py": (
        1,
        "run_auth_db_worker: drains the worker before a cancellation "
        "propagates, so a cancelled and awaited task owns no live thread.",
    ),
    "web/services/db_runtime.py": (
        1,
        "run_db_io_cancellation_safe itself: drains the worker before a "
        "cancellation propagates, so a cancelled and awaited task owns no "
        "live thread.",
    ),
    "web/services/agent_management.py": (
        2,
        "Runtime-key delivery and its compensation, both awaited through "
        "await_task_settlement (the worker is drained before a cancellation "
        "propagates); reached from the agent routes (api/agents.py, "
        "api/v1/agents.py) and workforce creation, which are HTTP producers. "
        "The builder's create_agent/update_agent tools use no thread.",
    ),
    "web/services/durable_sandbox_lifecycle.py": (
        4,
        "DurableSandboxDeleteCoordinator has no caller in this repository "
        "(an embedding host constructs it); it is not part of any loop "
        "retire_writers stops.",
    ),
    # Only reads the serving-generation row (is_serving never writes).
    "web/services/generation.py": (2, READ),
    "web/services/gmail_triggers.py": (
        5,
        "Gmail Pub/Sub webhook collection (collect_gmail_pubsub_events): "
        "webhook ingress is an HTTP producer, listed under 'Not covered'. The "
        "scheduled watch renewal and provisioning sweeps are tracked.",
    ),
    "web/services/mcp_oauth.py": (1, NETWORK),
    "web/services/task_coordinator_runtime.py": (2, EXEC),
    "web/services/task_delete.py": (
        5,
        "Task deletion, called by the delete routes (api/chat.py, "
        "api/v1/tasks.py): " + HTTP,
    ),
    "web/services/task_event_trace_handler.py": (
        3,
        "Reads one stream event's display data, the prior-message check and "
        "the task description for a websocket trace; read-only.",
    ),
    "web/services/task_execution.py": (7, EXEC),
    "web/services/task_execution_controller.py": (2, EXEC),
    "web/services/task_lease_service.py": (
        1,
        "Lease acquisition, awaited through await_task_settlement. The lease "
        "is the cross-process mutex for a run, so a stray acquire is "
        "arbitrated by the lease itself; the callers are coordinators and "
        "command attempts that execution_status() reports.",
    ),
    "web/services/task_orchestrator.py": (4, EXEC),
    "web/services/task_runtime.py": (
        2,
        "The bounded executor for task-runtime hooks of running executions "
        "(shutdown_task_runtime_hook_executor joins it at shutdown): " + EXEC,
    ),
    "web/services/trace_database.py": (
        3,
        "Trace-event persistence of running executions, awaited through "
        "drain_async_task_cancellation_safe and closed by "
        "close_trace_database_runtime at shutdown: " + EXEC,
    ),
    "web/services/trace_handlers.py": (1, READ),
    "web/services/trigger_providers/gmail.py": (
        3,
        "Trigger register/unregister from the trigger HTTP routes, and the "
        "webhook OIDC verification: HTTP producers.",
    ),
    "web/services/triggers.py": (
        2,
        "Two pools whose results are awaited inside the call (the `with` "
        "block joins them before it returns), used by the synchronous "
        "trigger CRUD that the HTTP routes run. The retire-relevant trigger "
        "run threads of this file are tracked.",
    ),
    "web/services/uploaded_file_cleanup_publication.py": (
        3,
        "Restores or copies a stored file to local disk and enters/exits the "
        "per-file cleanup lock, for HTTP routes and executions: " + FILES,
    ),
    # -- core/tools/adapters: tool calls of a running execution ------------------
    "core/tools/adapters/vibe/a2a_agent_tool.py": (1, NETWORK),
    "core/tools/adapters/vibe/command_executor.py": (1, EXEC),
    "core/tools/adapters/vibe/computer.py": (1, EXEC),
    "core/tools/adapters/vibe/factory.py": (3, EXEC),
    "core/tools/adapters/vibe/javascript_executor.py": (1, EXEC),
    "core/tools/adapters/vibe/mcp_adapter.py": (3, EXEC),
    "core/tools/adapters/vibe/output_filter_wrapper.py": (1, EXEC),
    "core/tools/adapters/vibe/python_executor.py": (1, EXEC),
    "core/tools/adapters/vibe/sandboxed_tool/sandboxed_tool_wrapper.py": (4, EXEC),
    "core/tools/adapters/vibe/vision_tool.py": (1, FILES),
    # -- core/tools/core: RAG storage, tools ------------------------------------
    "core/tools/core/browser_use.py": (1, FILES),
    "core/tools/core/RAG_tools/kb/collection_handle.py": (
        1,
        "A pool that runs the dense query beside the sparse one and is joined "
        "by its `with` block before the search returns; read-only. The "
        "document-delete write of this file goes through run_blocking.",
    ),
    "core/tools/core/RAG_tools/kb/coordinator.py": (
        1,
        "_run_in_separate_loop: a thread that is joined immediately, so the "
        "caller does not return while it runs. Every to_thread of this file "
        "goes through run_blocking.",
    ),
    "core/tools/core/RAG_tools/management/collection_manager.py": (
        3,
        "Collection statistics aggregation (read-only) and a thread joined "
        "immediately in the sync-to-async bridge.",
    ),
    "core/tools/core/RAG_tools/management/collections.py": (2, READ),
    "core/tools/core/RAG_tools/management/write_leases.py": (
        3,
        "Three daemon helpers: the heartbeat that renews the operation's own "
        "lease rows while its scope runs (stopped and joined before release, "
        "and the async decorators release it off the event loop), plus the "
        "fire-and-forget compaction and expired-row GC of the lease table, "
        "neither of which writes application data. All belong to the "
        "HTTP-route and execution classes retire_writers lists under "
        "'Not covered'.",
    ),
    "core/tools/core/RAG_tools/management/migration_state.py": (
        1,
        "The freeze guard reads one state row (and opens the metadata "
        "connection to read it). Lease writes go off the loop through "
        "run_blocking, and a scope releases on its own exit.",
    ),
    "core/tools/core/RAG_tools/pipelines/document_ingestion.py": (
        2,
        "Embedding encode calls: a provider call and a pool joined by its "
        "`with` block inside the ingestion call. The ingestion call itself "
        "(the writer) is reached through run_blocking from the builder and "
        "otherwise from HTTP routes and executions.",
    ),
    "core/tools/core/RAG_tools/storage/file_reference.py": (
        2,
        "Enter/exit of the file-reference protection lock around a document "
        "upsert that the caller awaits inline; the lock itself writes no row.",
    ),
    "core/tools/core/RAG_tools/storage/lancedb_stores.py": (
        6,
        "Opens a LanceDB connection and reads collection metadata; creates no "
        "table. The metadata-table creation goes through run_blocking.",
    ),
    "core/tools/core/RAG_tools/web_crawler/crawler.py": (1, NETWORK),
    "core/tools/core/document_search.py": (1, READ),
    "core/tools/core/exa_web_search.py": (2, NETWORK),
    "core/tools/core/excel.py": (2, FILES),
    "core/tools/core/image_tool.py": (6, FILES),
    "core/tools/core/pptx_conversion.py": (3, FILES),
    "core/tools/core/pptx_render.py": (10, FILES),
    "core/tools/core/video_tool.py": (9, FILES),
    "core/tools/core/vision_tool.py": (5, FILES),
}


def _call_name(node: ast.Call) -> str | None:
    func = node.func
    if isinstance(func, ast.Attribute):
        name = func.attr
        owner = func.value
        if name == "run_sync":
            # anyio.to_thread.run_sync only.
            return (
                name
                if isinstance(owner, ast.Attribute) and owner.attr == "to_thread"
                else None
            )
        if name == "Thread":
            return (
                name
                if isinstance(owner, ast.Name) and owner.id == "threading"
                else None
            )
        return name if name in THREAD_CALLS else None
    if isinstance(func, ast.Name) and func.id in THREAD_CALLS - {"run_sync"}:
        return func.id
    return None


def _scan() -> dict[str, list[int]]:
    found: dict[str, list[int]] = {}
    for sub in SCANNED:
        for path in sorted((SRC / sub).rglob("*.py")):
            rel = path.relative_to(SRC).as_posix()
            if rel in IMPLEMENTATION:
                continue
            tree = ast.parse(path.read_text(encoding="utf-8"))
            lines = sorted(
                node.lineno
                for node in ast.walk(tree)
                if isinstance(node, ast.Call) and _call_name(node)
            )
            if lines:
                found[rel] = lines
    return found


FOUND = _scan()

_FIX = (
    "\nA thread started like this keeps running after its awaiting task is "
    "cancelled and can still write after retire_writers() reported success.\n"
    "- If it can write to the database, a vector store or other shared state: "
    "in xagent.web use `run_tracked(fn, ...)` "
    "(xagent.web.services.writer_workers); in xagent.core use "
    "`run_blocking(fn, ...)` (xagent.core.utils.blocking).\n"
    "- If it cannot (read-only, filesystem scratch, network), or belongs to a "
    "class listed under 'Not covered' in app.retire_writers, add or raise the "
    "entry for the file in ALLOWED in this test, with a reason that is true "
    "for every call in the file, and say so in the PR."
)


def test_every_thread_starting_call_is_tracked_or_audited():
    unlisted = {rel: lines for rel, lines in FOUND.items() if rel not in ALLOWED}
    assert not unlisted, (
        "Untracked, unaudited thread-starting calls:\n"
        + "\n".join(
            f"  {rel}: lines {lines}" for rel, lines in sorted(unlisted.items())
        )
        + _FIX
    )


@pytest.mark.parametrize("rel", sorted(ALLOWED))
def test_an_audited_file_keeps_exactly_its_audited_call_count(rel):
    count, reason = ALLOWED[rel]
    actual = FOUND.get(rel, [])
    assert (SRC / rel).is_file(), f"{rel} no longer exists: remove its entry"
    assert len(actual) <= count, (
        f"{rel} has {len(actual)} thread-starting calls (lines {actual}) but "
        f"{count} were audited." + _FIX
    )
    assert len(actual) == count, (
        f"{rel} now has {len(actual)} thread-starting calls, {count} audited: "
        "lower the audited count (a call was removed or moved onto "
        "run_tracked/run_blocking) so the audit stays exact."
    )
    assert len(reason) > 40, f"{rel}: the reason must say why the file is safe"


def test_the_scan_sees_the_known_call_shapes():
    """The detector itself must keep finding each shape it claims to find."""
    tree = ast.parse(
        "import asyncio, anyio, threading\n"
        "from concurrent.futures import ThreadPoolExecutor\n"
        "async def f(loop):\n"
        "    await asyncio.to_thread(g)\n"
        "    await loop.run_in_executor(None, g)\n"
        "    await anyio.to_thread.run_sync(g)\n"
        "    await run_in_thread_with_telemetry('x', g)\n"
        "    threading.Thread(target=g)\n"
        "    ThreadPoolExecutor(1)\n"
        "    other.run_sync(g)\n"
        "    other.Thread(g)\n"
    )
    names = sorted(
        name
        for node in ast.walk(tree)
        if isinstance(node, ast.Call) and (name := _call_name(node))
    )
    assert names == sorted(
        [
            "to_thread",
            "run_in_executor",
            "run_sync",
            "run_in_thread_with_telemetry",
            "Thread",
            "ThreadPoolExecutor",
        ]
    )


def test_the_tracked_call_sites_are_still_there():
    """The files the docstring of retire_writers promises are tracked still use the trackers."""
    expected = {
        "web/app.py": "run_tracked(",
        "web/api/websocket.py": "run_tracked(",
        "web/channels/feishu/bot.py": "run_tracked(",
        "web/channels/slack/bot.py": "run_tracked(",
        "web/channels/telegram/bot.py": "run_tracked(",
        "web/services/channel_runtime.py": "run_tracked(",
        "web/services/shared_channel_execution.py": "run_tracked(",
        "web/services/gmail_provisioning.py": "start_tracked_thread(",
        "web/services/triggers.py": "run_tracked(",
        "web/services/task_cleanup_obligations.py": "run_tracked(",
        "web/services/task_retention_purge.py": "run_tracked(",
        "core/tools/core/RAG_tools/kb/coordinator.py": "run_blocking(",
        "core/tools/core/RAG_tools/kb/api_compatibility.py": "run_blocking(",
        "core/tools/core/RAG_tools/pipelines/web_ingestion.py": "run_blocking(",
        "core/tools/adapters/vibe/file_ingestion_tool.py": "run_blocking(",
        "core/tools/adapters/vibe/function.py": "run_blocking(",
    }
    missing = [
        rel
        for rel, needle in expected.items()
        if needle not in (SRC / rel).read_text(encoding="utf-8")
    ]
    assert not missing, f"no longer routed through its tracker: {missing}"