Enterprise agent platform for building, deploying, and operating AI teammates.
"""``retire_writers``: the writers stop once, the process keeps serving, shutdown skips it."""
from __future__ import annotations
import asyncio
import threading
from collections.abc import Iterator
from contextlib import suppress
import pytest
from xagent.web import app as app_module
from xagent.web.services.release_hold import WritersRetired
from xagent.web.services.writer_workers import (
active_workers,
drain_workers,
run_tracked,
)
# The ``raise_errors`` flag of every stubbed ``_stop_chat_channels`` call.
CHANNEL_STOP_FLAGS: list[bool] = []
@pytest.fixture
def steps(monkeypatch: pytest.MonkeyPatch) -> Iterator[list[str]]:
calls: list[str] = []
stop_flags = CHANNEL_STOP_FLAGS
stop_flags.clear()
def stopper(name: str):
async def stop(*_args, **_kwargs) -> None:
calls.append(name)
return stop
for name in (
"stop_orphan_upload_gc_task",
"stop_composio_reconcile_task",
"stop_composio_missing_account_task",
"stop_retention_purge_task",
"stop_task_cleanup_retry_task",
"stop_uploaded_file_recovery_task",
"stop_task_auto_resume_task",
"stop_task_lease_recovery_task",
):
monkeypatch.setattr(app_module, name, stopper(name))
async def stop_chat_channels(*, raise_errors: bool) -> None:
calls.append("stop_chat_channels")
stop_flags.append(raise_errors)
monkeypatch.setattr(app_module, "_stop_chat_channels", stop_chat_channels)
monkeypatch.setattr(app_module, "_writers_retired", False)
monkeypatch.setattr(app_module, "_task_command_dispatcher_task", None)
monkeypatch.setattr(app_module, "_sandbox_idle_sweep_task", None)
monkeypatch.setattr(app_module, "_file_storage_startup_sync_task", None)
monkeypatch.setattr(app_module, "_trigger_dispatcher_task", None)
monkeypatch.setattr(app_module, "_migration_task", None)
monkeypatch.setattr(app_module, "_uploaded_file_reconcile_task", None)
monkeypatch.setattr(app_module, "_retire_writers_lock", None)
yield calls
app_module.release_hold.reset_writers_retired()
async def test_retire_writers_stops_every_writer_once_and_is_idempotent(steps):
assert not app_module.writers_retired()
await app_module.retire_writers()
first = list(steps)
await app_module.retire_writers()
assert app_module.writers_retired()
# Socket writes that remain are refused from here on.
assert app_module.release_hold.socket_writes_fenced
assert first == [
"stop_orphan_upload_gc_task",
"stop_composio_reconcile_task",
"stop_composio_missing_account_task",
"stop_retention_purge_task",
"stop_task_cleanup_retry_task",
"stop_uploaded_file_recovery_task",
"stop_task_auto_resume_task",
"stop_task_lease_recovery_task",
"stop_chat_channels",
]
assert steps == first
async def test_a_step_that_fails_leaves_the_retire_unfinished_so_it_can_continue(
steps, monkeypatch
):
async def boom(*_a, **_k) -> None:
raise RuntimeError("stop failed")
monkeypatch.setattr(app_module, "stop_task_auto_resume_task", boom)
with pytest.raises(RuntimeError):
await app_module.retire_writers()
assert not app_module.writers_retired()
async def fine(*_a, **_k) -> None:
steps.append("auto_resume")
monkeypatch.setattr(app_module, "stop_task_auto_resume_task", fine)
await app_module.retire_writers()
assert app_module.writers_retired()
assert steps[-3:] == [
"auto_resume",
"stop_task_lease_recovery_task",
"stop_chat_channels",
]
# -- socket writes are refused once the writers are retired -------------------
@pytest.fixture
def retired_hold():
from xagent.web.services.release_hold import release_hold
release_hold.mark_writers_retired()
yield release_hold
release_hold.reset_writers_retired()
def test_the_retired_flag_is_one_way_until_a_new_lifespan():
from xagent.web.services.release_hold import release_hold
assert not release_hold.socket_writes_fenced
release_hold.mark_writers_retired()
try:
release_hold.reopen()
assert release_hold.socket_writes_fenced
finally:
release_hold.reset_writers_retired()
assert not release_hold.socket_writes_fenced
async def test_a_work_starting_socket_command_is_refused_with_the_paused_error(
retired_hold,
):
from xagent.web.api import websocket as ws
from xagent.web.services.client_error_messages import ClientErrorCode
from xagent.web.services.task_command_transport import TaskCommandKind
class _User:
id = 1
is_admin = False
with pytest.raises(ws.ClientVisibleValidationError) as caught:
await ws._enqueue_websocket_task_command(
task_id=1, message_data={"user": _User()}, kind=TaskCommandKind.MESSAGE
)
assert caught.value.error_code is ClientErrorCode.EXECUTION_PAUSED
@pytest.mark.parametrize("name", ["PAUSE", "RESUME", "CANCEL"])
async def test_a_steering_command_still_gets_through_the_fence(
retired_hold, monkeypatch, name
):
from xagent.web.api import websocket as ws
from xagent.web.services.task_command_transport import TaskCommandKind
class _User:
id = 1
is_admin = False
sentinel = object()
monkeypatch.setattr(
ws, "_enqueue_websocket_task_command_sync", lambda **_k: sentinel
)
result = await ws._enqueue_websocket_task_command(
task_id=1, message_data={"user": _User()}, kind=TaskCommandKind[name]
)
assert result is sentinel
assert not retired_hold._socket_writes
async def test_execute_task_is_refused_before_it_acknowledges_or_stages(retired_hold):
from xagent.web.api import websocket as ws
sent: list[dict] = []
class _User:
id = 1
is_admin = False
async def record(message, _websocket):
sent.append(message)
original = ws.manager.send_personal_message
ws.manager.send_personal_message = record # type: ignore[assignment]
try:
await ws.handle_execute_task(object(), 1, {"user": _User()}) # type: ignore[arg-type]
finally:
ws.manager.send_personal_message = original # type: ignore[assignment]
assert all(m.get("type") != "execution_started" for m in sent)
assert any(m.get("error_code") == "execution_paused" for m in sent)
# -- the retire waits for work that outlives the tasks it cancels ---------------
async def test_the_retire_waits_for_an_admitted_socket_write_and_fences_new_ones(steps):
hold = app_module.release_hold
interrupted: list[bool] = []
token = hold.acquire_socket_write(lambda: interrupted.append(True))
retiring = asyncio.create_task(app_module.retire_writers())
await asyncio.sleep(0.2)
assert not retiring.done()
assert interrupted == [True]
assert not app_module.writers_retired()
with pytest.raises(WritersRetired):
hold.acquire_socket_write()
token.release()
await retiring
assert app_module.writers_retired()
async def test_a_socket_write_that_never_settles_fails_the_retire(steps, monkeypatch):
monkeypatch.setattr(app_module, "RETIRE_SETTLE_SECONDS", 0.2)
token = app_module.release_hold.acquire_socket_write()
try:
with pytest.raises(RuntimeError, match="socket writes"):
await app_module.retire_writers()
assert not app_module.writers_retired()
# The fence stays up for the life of the lifespan.
assert app_module.release_hold.socket_writes_fenced
finally:
token.release()
await app_module.retire_writers()
assert app_module.writers_retired()
async def test_a_detached_worker_thread_blocks_the_retire_until_it_returns(
steps, monkeypatch
):
release = threading.Event()
started = threading.Event()
def worker() -> None:
started.set()
release.wait(5)
awaiting = asyncio.create_task(run_tracked(worker))
while not started.is_set():
await asyncio.sleep(0.01)
# What stop_retention_purge_task does after its grace period.
awaiting.cancel()
with pytest.raises(asyncio.CancelledError):
await awaiting
assert active_workers() == 1
monkeypatch.setattr(app_module, "RETIRE_SETTLE_SECONDS", 0.2)
with pytest.raises(RuntimeError, match="writer threads"):
await app_module.retire_writers()
assert not app_module.writers_retired()
release.set()
monkeypatch.setattr(app_module, "RETIRE_SETTLE_SECONDS", 5)
await app_module.retire_writers()
assert app_module.writers_retired()
assert active_workers() == 0
async def test_a_cancelled_await_before_the_thread_started_leaves_no_count(
monkeypatch,
):
from xagent.web.services import writer_workers
# The thread never starts: the executor hand-off is replaced by a wait that
# only the cancellation ends, so the count has been taken when it arrives.
async def never_started(*_a, **_k):
await asyncio.Event().wait()
monkeypatch.setattr(writer_workers.asyncio, "to_thread", never_started)
ran: list[bool] = []
task = asyncio.create_task(run_tracked(lambda: ran.append(True)))
await asyncio.sleep(0)
assert active_workers() == 1
task.cancel()
with suppress(asyncio.CancelledError):
await task
assert active_workers() == 0
assert await drain_workers(1.0)
assert ran == []
async def test_the_uploaded_file_reconcile_task_is_cancelled_by_the_retire(
steps, monkeypatch
):
started = asyncio.Event()
async def background() -> None:
started.set()
await asyncio.sleep(60)
task = asyncio.create_task(background())
await started.wait()
monkeypatch.setattr(app_module, "_uploaded_file_reconcile_task", task)
await app_module.retire_writers()
assert task.cancelled()
async def test_the_socket_command_enqueue_holds_a_token_until_its_thread_settles(
monkeypatch,
):
from xagent.web.api import websocket as ws
from xagent.web.services.task_command_transport import TaskCommandKind
seen: list[tuple[int, int]] = []
def fake_enqueue(**_kwargs):
seen.append((len(app_module.release_hold._socket_writes), active_workers()))
raise RuntimeError("enqueue failed")
class _User:
id = 1
is_admin = False
monkeypatch.setattr(ws, "_enqueue_websocket_task_command_sync", fake_enqueue)
with pytest.raises(RuntimeError):
await ws._enqueue_websocket_task_command(
task_id=1,
message_data={"user": _User()},
kind=TaskCommandKind.MESSAGE,
)
# One token and one tracked worker while it ran; both gone afterwards.
assert seen == [(1, 1)]
assert not app_module.release_hold._socket_writes
assert active_workers() == 0
async def test_an_unauthenticated_enqueue_is_refused_without_a_token():
from xagent.web.api import websocket as ws
from xagent.web.services.task_command_transport import TaskCommandKind
with pytest.raises(ws.ClientVisibleValidationError):
await ws._enqueue_websocket_task_command(
task_id=1, message_data={}, kind=TaskCommandKind.MESSAGE
)
assert not app_module.release_hold._socket_writes
async def test_an_asr_connection_to_a_retired_process_is_accepted_before_it_is_closed(
monkeypatch,
):
from starlette.websockets import WebSocketState
from xagent.web.api import speech
order: list[str] = []
class _Socket:
application_state = WebSocketState.CONNECTING
async def close(self, code: int = 1000, reason: str = "") -> None:
order.append(f"close:{code}")
async def authenticated(*_a, **_k):
return type("U", (), {"id": 1})()
async def accept(_websocket) -> None:
order.append("accept")
monkeypatch.setattr(speech, "get_authenticated_user", authenticated)
monkeypatch.setattr(speech, "accept_authenticated_websocket", accept)
app_module.release_hold.mark_writers_retired()
await speech.stream_asr.__wrapped__(_Socket(), token=None) # type: ignore[attr-defined]
assert order == ["accept", "close:1012"]
class _AsrSocket:
"""Just enough of a WebSocket for ``stream_asr`` up to its first receive."""
from starlette.websockets import WebSocketState as _State
application_state = _State.CONNECTING
def __init__(self) -> None:
self.closed: list[int] = []
async def close(self, code: int = 1000, reason: str = "") -> None:
self.closed.append(code)
self.application_state = self._State.DISCONNECTED
async def test_a_wound_down_asr_session_closes_1012_and_releases_its_token(
monkeypatch,
):
from xagent.web.api import speech
from xagent.web.services import speech_streams
blocked = asyncio.Event()
async def authenticated(*_a, **_k):
return type("U", (), {"id": 1})()
async def accept(_websocket) -> None:
return None
async def receive(_websocket):
blocked.set()
await asyncio.sleep(60)
monkeypatch.setattr(speech, "get_authenticated_user", authenticated)
monkeypatch.setattr(speech, "accept_authenticated_websocket", accept)
monkeypatch.setattr(speech_streams, "_receive_asr_message", receive)
socket = _AsrSocket()
handler = asyncio.create_task(
speech.stream_asr.__wrapped__(socket, token=None) # type: ignore[attr-defined]
)
await blocked.wait()
assert len(app_module.release_hold._socket_writes) == 1
app_module.release_hold.mark_writers_retired()
await handler # the fence cancel does not escape the handler
assert not handler.cancelled()
assert socket.closed == [1012]
assert await app_module.release_hold.drain_socket_writes(1.0)
async def test_the_usage_delivery_of_a_scope_is_a_tracked_worker(monkeypatch):
from xagent.web.tracking import standalone_usage
release = threading.Event()
started = threading.Event()
def blocking_deliver(_self, _details) -> None:
started.set()
release.wait(5)
async def no_gate(*_a, **_k) -> None:
return None
monkeypatch.setattr(standalone_usage, "check_standalone_gate_async", no_gate)
monkeypatch.setattr(
standalone_usage.UsageScope, "_pending", lambda _self: [{"x": 1}]
)
monkeypatch.setattr(standalone_usage.UsageScope, "_deliver", blocking_deliver)
async def metered() -> None:
async with standalone_usage.usage_scope(
1, usage_source="speech_transcribe", model_ids=()
):
pass
exiting = asyncio.create_task(metered())
while not started.is_set():
await asyncio.sleep(0.01)
exiting.cancel()
with suppress(asyncio.CancelledError):
await exiting
assert active_workers() == 1
release.set()
assert await drain_workers(5.0)
# -- the fence fires its interrupts once ----------------------------------------
def test_a_retry_of_the_fence_does_not_interrupt_a_winding_down_write_again():
from xagent.web.services.release_hold import ReleaseHold
hold = ReleaseHold()
interrupted: list[bool] = []
hold.acquire_socket_write(lambda: interrupted.append(True))
hold.mark_writers_retired()
hold.mark_writers_retired()
assert interrupted == [True]
# -- the chokepoint: nothing is dispatched once the fence is up -----------------
async def test_a_fenced_dispatch_only_notifies_the_dispatcher(
retired_hold, monkeypatch
):
from xagent.web.services import task_command_transport as transport
dispatched: list[int] = []
notified: list[bool] = []
async def dispatch_one(_executor, *, command_db_id):
dispatched.append(command_db_id)
return True
monkeypatch.setattr(transport, "dispatch_one_task_command", dispatch_one)
monkeypatch.setattr(transport, "consumes_task_commands", lambda: True)
monkeypatch.setattr(
transport, "notify_task_command_dispatcher", lambda: notified.append(True)
)
await transport.dispatch_task_command_promptly(object(), command_db_id=7) # type: ignore[arg-type]
assert dispatched == []
assert notified == [True]
assert transport.prompt_dispatch_count() == 0
async def test_an_unfenced_dispatch_still_runs_the_command(monkeypatch):
from xagent.web.services import task_command_transport as transport
dispatched: list[int] = []
async def dispatch_one(_executor, *, command_db_id):
dispatched.append(command_db_id)
return True
monkeypatch.setattr(transport, "dispatch_one_task_command", dispatch_one)
monkeypatch.setattr(transport, "consumes_task_commands", lambda: True)
await transport.dispatch_task_command_promptly(object(), command_db_id=7) # type: ignore[arg-type]
assert dispatched == [7]
async def test_the_retire_waits_for_a_prompt_dispatch_that_started_before_the_fence(
steps, monkeypatch
):
from xagent.web.services import task_command_transport as transport
monkeypatch.setattr(app_module, "RETIRE_SETTLE_SECONDS", 0.2)
gate = asyncio.Event()
async def stubborn() -> bool:
# A dispatch that is cancelled but cleans up for a moment.
try:
await asyncio.sleep(60)
except asyncio.CancelledError:
await gate.wait()
raise
return True
task = asyncio.create_task(stubborn())
await asyncio.sleep(0)
transport._prompt_dispatch_tasks.add(task)
try:
with pytest.raises(RuntimeError, match="command dispatches"):
await app_module.retire_writers()
assert not app_module.writers_retired()
gate.set()
monkeypatch.setattr(app_module, "RETIRE_SETTLE_SECONDS", 5)
await app_module.retire_writers()
assert app_module.writers_retired()
assert task.cancelled()
finally:
gate.set()
transport._prompt_dispatch_tasks.discard(task)
# -- every startup-owned writer thread is tracked --------------------------------
async def test_the_task_cleanup_retry_batch_runs_on_tracked_workers(monkeypatch):
from xagent.web.services import task_cleanup_obligations as cleanup
release = threading.Event()
started = threading.Event()
def blocking_candidates(*_a, **_k):
started.set()
release.wait(5)
return []
monkeypatch.setattr(cleanup, "_due_candidates_sync", blocking_candidates)
batch = asyncio.create_task(cleanup.run_cleanup_obligation_batch(lambda: None)) # type: ignore[arg-type,return-value]
while not started.is_set():
await asyncio.sleep(0.01)
batch.cancel()
with suppress(asyncio.CancelledError):
await batch
assert active_workers() == 1
release.set()
assert await drain_workers(5.0)
async def test_the_composio_reconcile_pass_runs_on_a_tracked_worker(monkeypatch):
from xagent import web as _web # noqa: F401
from xagent.web import composio_auth
release = threading.Event()
started = threading.Event()
stop = threading.Event()
def blocking_pass(*, stop_event) -> None:
started.set()
release.wait(5)
monkeypatch.setattr(
composio_auth, "reconcile_orphan_accounts_isolated", blocking_pass
)
loop_task = asyncio.create_task(
composio_auth.run_reconcile_loop(interval_seconds=60, stop_event=stop)
)
while not started.is_set():
await asyncio.sleep(0.01)
stop.set()
loop_task.cancel()
with suppress(asyncio.CancelledError):
await loop_task
assert active_workers() == 1
release.set()
assert await drain_workers(5.0)
async def test_a_trigger_run_start_runs_its_database_work_on_tracked_workers(
monkeypatch,
):
from xagent.web.services import triggers
release = threading.Event()
started = threading.Event()
def blocking_load(_run_id):
started.set()
release.wait(5)
monkeypatch.setattr(triggers, "_load_prepared_trigger_start", blocking_load)
starting = asyncio.create_task(triggers._start_prepared_trigger_run_id(1))
while not started.is_set():
await asyncio.sleep(0.01)
starting.cancel()
with suppress(asyncio.CancelledError):
await starting
assert active_workers() == 1
release.set()
assert await drain_workers(5.0)
async def test_the_retire_sets_the_temp_file_walk_stop_without_waiting_for_it(steps):
stop = threading.Event()
app_module.app.state.temp_file_cleanup_stop = stop
try:
await app_module.retire_writers()
finally:
app_module.app.state.temp_file_cleanup_stop = None
assert stop.is_set()
# -- concurrency and lifecycle ----------------------------------------------------
async def test_two_concurrent_retires_stop_the_writers_once(steps, monkeypatch):
async def slow_stop(*_a, **_k) -> None:
steps.append("stop_orphan_upload_gc_task")
await asyncio.sleep(0.1)
monkeypatch.setattr(app_module, "stop_orphan_upload_gc_task", slow_stop)
await asyncio.gather(app_module.retire_writers(), app_module.retire_writers())
assert steps.count("stop_orphan_upload_gc_task") == 1
assert steps.count("stop_task_lease_recovery_task") == 1
assert app_module.writers_retired()
async def test_a_new_lifespan_clears_the_retired_state(steps, monkeypatch):
await app_module.retire_writers()
assert app_module.writers_retired()
assert app_module.release_hold.socket_writes_fenced
class _Stop(Exception):
pass
def stop_here() -> None:
raise _Stop
monkeypatch.setattr(app_module, "validate_auth_config", stop_here)
with pytest.raises(_Stop):
await app_module.startup_event()
assert not app_module.writers_retired()
assert not app_module.release_hold.socket_writes_fenced
@pytest.fixture
def teardown(monkeypatch: pytest.MonkeyPatch, steps: list[str]) -> list[str]:
"""Stub everything ``shutdown_event`` does after the writers stop."""
from xagent.web.services import task_coordinator_runtime
monkeypatch.setattr(app_module, "flush_langfuse", lambda: None)
async def record_drain() -> None:
steps.append("close_task_coordinators")
async def idle(*_a, **_k) -> None:
return None
monkeypatch.setattr(
task_coordinator_runtime, "close_task_coordinators", record_drain
)
monkeypatch.setattr(
"xagent.web.services.task_execution.background_task_manager.shutdown", idle
)
monkeypatch.setattr(
"xagent.web.services.task_lease_service.wait_for_heartbeat_manager_idle", idle
)
monkeypatch.setattr(app_module, "stop_runtime_performance_monitor", idle)
monkeypatch.setattr(app_module, "stop_app_process_diagnostics", idle)
monkeypatch.setattr("xagent.web.sandbox_manager.get_sandbox_manager", lambda: None)
monkeypatch.setattr(
"xagent.web.services.chrome_mcp_runtime.shutdown_chrome_execution_session_pool",
idle,
)
monkeypatch.setattr(
"xagent.web.services.trace_database.close_trace_database_runtime", idle
)
return steps
async def test_shutdown_stops_the_writers_before_it_drains_the_coordinators(
teardown,
):
await app_module.shutdown_event()
assert teardown.index("stop_task_lease_recovery_task") < teardown.index(
"close_task_coordinators"
)
async def test_shutdown_skips_a_retire_the_host_already_completed(teardown):
await app_module.retire_writers()
stops_before = list(teardown)
await app_module.shutdown_event()
# The stops are idempotent and channel CRUD can restart a bot after the
# host's retire, so shutdown runs them again; only the waits are skipped.
assert teardown[: len(stops_before)] == stops_before
assert CHANNEL_STOP_FLAGS[-1:] == [False] # the channels were stopped again
assert teardown[-1] == "close_task_coordinators"
async def test_shutdown_still_drains_the_coordinators_when_the_retire_times_out(
teardown, monkeypatch
):
monkeypatch.setattr(app_module, "RETIRE_SETTLE_SECONDS", 0.1)
token = app_module.release_hold.acquire_socket_write()
try:
await app_module.shutdown_event()
finally:
token.release()
assert not app_module.writers_retired()
assert "close_task_coordinators" in teardown
# -- builder chat is a socket write ------------------------------------------------
class _BuilderSocket:
"""Delivers one builder frame, then optionally runs a hook before it closes."""
def __init__(self, after_first=None) -> None:
self.frames = 0
self.after_first = after_first
self.state = type("S", (), {})()
async def receive_text(self) -> str:
from starlette.websockets import WebSocketDisconnect
self.frames += 1
if self.frames == 1:
return "{}"
if self.after_first is not None:
await self.after_first()
await asyncio.sleep(0.05)
raise WebSocketDisconnect()
@pytest.fixture
def builder_endpoint(monkeypatch):
from xagent.web.api import websocket as ws
async def authenticated(*_a, **_k):
return type("U", (), {"id": 1, "is_admin": False})()
async def accept(_websocket) -> None:
return None
monkeypatch.setattr(ws, "get_authenticated_user", authenticated)
monkeypatch.setattr(ws, "accept_authenticated_websocket", accept)
return ws
async def test_a_builder_turn_after_the_fence_is_refused_not_started(
builder_endpoint, retired_hold, monkeypatch
):
ws = builder_endpoint
paused: list[bool] = []
started: list[bool] = []
async def send_paused(_websocket, **_k) -> None:
paused.append(True)
async def turn(*_a, **_k) -> None:
started.append(True)
monkeypatch.setattr(ws, "send_execution_paused", send_paused)
monkeypatch.setattr(ws, "handle_builder_chat", turn)
await ws.websocket_builder_chat_endpoint.__wrapped__(_BuilderSocket(), token=None) # type: ignore[attr-defined]
assert paused == [True]
assert started == []
assert not retired_hold._socket_writes
async def test_a_builder_turn_holds_a_token_and_releases_it_when_the_turn_ends(
builder_endpoint, monkeypatch
):
ws = builder_endpoint
seen: list[int] = []
async def turn(*_a, **_k) -> None:
seen.append(len(app_module.release_hold._socket_writes))
monkeypatch.setattr(ws, "handle_builder_chat", turn)
await ws.websocket_builder_chat_endpoint.__wrapped__(_BuilderSocket(), token=None) # type: ignore[attr-defined]
assert seen == [1]
assert not app_module.release_hold._socket_writes
async def test_the_fence_cancels_a_running_builder_turn_and_the_token_follows(
builder_endpoint, monkeypatch
):
ws = builder_endpoint
running = asyncio.Event()
cancelled: list[bool] = []
async def turn(*_a, **_k) -> None:
running.set()
try:
await asyncio.sleep(60)
except asyncio.CancelledError:
cancelled.append(True)
raise
async def fence() -> None:
await running.wait()
app_module.release_hold.mark_writers_retired()
monkeypatch.setattr(ws, "handle_builder_chat", turn)
await ws.websocket_builder_chat_endpoint.__wrapped__( # type: ignore[attr-defined]
_BuilderSocket(after_first=fence), token=None
)
assert cancelled == [True]
assert not app_module.release_hold._socket_writes
# -- entries that write are refused before they write -------------------------------
async def test_the_missing_task_branch_refuses_once_the_fence_rose_after_the_enqueue(
monkeypatch,
):
from xagent.web.api import websocket as ws
paused: list[bool] = []
async def enqueue_then_fence(**_k):
# The enqueue was admitted; the fence goes up before the branch runs.
app_module.release_hold.mark_writers_retired()
return None
async def send_paused(_websocket, **_k) -> None:
paused.append(True)
async def replacement(*_a, **_k) -> None:
raise AssertionError("a replacement task was created after the fence")
monkeypatch.setattr(ws, "_enqueue_websocket_task_command", enqueue_then_fence)
monkeypatch.setattr(ws, "send_execution_paused", send_paused)
monkeypatch.setattr(
ws.command_execution_service, "handle_missing_task_message", replacement
)
await ws.handle_chat_message(object(), 1, {"user": object()}) # type: ignore[arg-type]
assert paused == [True]
assert not app_module.release_hold._socket_writes
assert not app_module.release_hold.snapshot()["responsibilities"]
async def test_a_preview_writes_no_task_row_once_the_fence_is_up(
retired_hold, monkeypatch
):
from xagent.web.api import websocket as ws
paused: list[bool] = []
async def send_paused(_websocket, **_k) -> None:
paused.append(True)
async def create(*_a, **_k) -> int:
raise AssertionError("the hidden preview Task row was written")
class _Socket:
state = type("S", (), {})()
monkeypatch.setattr(ws, "send_execution_paused", send_paused)
monkeypatch.setattr(ws, "_create_preview_task", create)
await ws.handle_build_preview_execution(
_Socket(), # type: ignore[arg-type]
{"message": "hello"},
type("U", (), {"id": 1})(), # type: ignore[arg-type]
)
assert paused == [True]
assert not app_module.release_hold.snapshot()["responsibilities"]
# -- the status a host reads ---------------------------------------------------------
async def test_execution_status_reports_the_fence_and_the_admitted_socket_writes():
from xagent.web.services.release_hold import execution_status
before = execution_status()
token = app_module.release_hold.acquire_socket_write()
try:
during = execution_status()
app_module.release_hold.mark_writers_retired()
fenced = execution_status()
finally:
token.release()
assert (before["socket_writes_fenced"], before["socket_writes"]) == (False, 0)
assert (during["socket_writes_fenced"], during["socket_writes"]) == (False, 1)
assert (fenced["socket_writes_fenced"], fenced["socket_writes"]) == (True, 1)
# Diagnostics only: they never make the process look busy.
assert not before["busy"]
# -- shutdown never waits for the tracked threads -----------------------------------
async def test_shutdown_does_not_wait_for_a_live_tracked_thread(teardown, monkeypatch):
monkeypatch.setattr(app_module, "RETIRE_SETTLE_SECONDS", 30)
release = threading.Event()
started = threading.Event()
def worker() -> None:
started.set()
release.wait(10)
awaiting = asyncio.create_task(run_tracked(worker))
while not started.is_set():
await asyncio.sleep(0.01)
try:
loop = asyncio.get_running_loop()
began = loop.time()
await app_module.shutdown_event()
elapsed = loop.time() - began
# The thread is still counted, and the drain after the retire ran at once.
assert active_workers() == 1
assert "close_task_coordinators" in teardown
assert elapsed < 5
assert not app_module.writers_retired()
# The fence and the stops still happened.
assert app_module.release_hold.socket_writes_fenced
assert "stop_task_lease_recovery_task" in teardown
assert set(CHANNEL_STOP_FLAGS) == {False}
finally:
release.set()
await awaiting
async def test_shutdown_does_not_queue_behind_a_host_retire_that_is_running(
teardown, monkeypatch
):
monkeypatch.setattr(app_module, "RETIRE_SETTLE_SECONDS", 30)
token = app_module.release_hold.acquire_socket_write()
host_retire = asyncio.create_task(app_module.retire_writers())
await asyncio.sleep(0.2)
assert not host_retire.done()
try:
loop = asyncio.get_running_loop()
began = loop.time()
await app_module.shutdown_event()
assert loop.time() - began < 5
assert "close_task_coordinators" in teardown
finally:
token.release()
await host_retire
assert app_module.writers_retired()
async def test_a_host_retire_waits_and_asks_the_channel_stop_to_raise(steps):
await app_module.retire_writers()
assert CHANNEL_STOP_FLAGS == [True]
# -- a failed channel stop ---------------------------------------------------------------
@pytest.fixture
def failing_channel(monkeypatch: pytest.MonkeyPatch) -> list[str]:
from xagent.web.channels.feishu import bot as feishu_bot
from xagent.web.channels.slack import bot as slack_bot
from xagent.web.channels.telegram import bot as telegram_bot
stopped: list[str] = []
class _Channel:
enabled = True
def __init__(self, name: str, fail: bool) -> None:
self.name = name
self.fail = fail
async def stop(self) -> None:
stopped.append(self.name)
if self.fail:
raise RuntimeError(f"{self.name} stop failed")
monkeypatch.setattr(
telegram_bot, "get_telegram_channel", lambda: _Channel("telegram", True)
)
monkeypatch.setattr(
feishu_bot, "get_feishu_channel", lambda: _Channel("feishu", False)
)
monkeypatch.setattr(
slack_bot, "get_slack_channel", lambda: _Channel("slack", False)
)
return stopped
async def test_the_retire_path_raises_a_failed_channel_stop_after_stopping_the_rest(
failing_channel,
):
with pytest.raises(RuntimeError, match="telegram stop failed"):
await app_module._stop_chat_channels(raise_errors=True)
assert failing_channel == ["telegram", "feishu", "slack"]
async def test_the_shutdown_path_logs_a_failed_channel_stop_and_goes_on(
failing_channel,
):
await app_module._stop_chat_channels(raise_errors=False)
assert failing_channel == ["telegram", "feishu", "slack"]
async def test_a_failed_channel_stop_leaves_the_retire_unpublished(
failing_channel, monkeypatch
):
for name in (
"stop_orphan_upload_gc_task",
"stop_composio_reconcile_task",
"stop_composio_missing_account_task",
"stop_retention_purge_task",
"stop_task_cleanup_retry_task",
"stop_uploaded_file_recovery_task",
"stop_task_auto_resume_task",
"stop_task_lease_recovery_task",
):
async def noop(*_a, **_k) -> None:
return None
monkeypatch.setattr(app_module, name, noop)
monkeypatch.setattr(app_module, "_writers_retired", False)
monkeypatch.setattr(app_module, "_retire_writers_lock", None)
with pytest.raises(RuntimeError, match="telegram stop failed"):
await app_module.retire_writers()
assert not app_module.writers_retired()
# -- steering commands after the fence are acknowledged as queued ---------------------------
@pytest.mark.parametrize("name", ["PAUSE", "RESUME"])
@pytest.mark.parametrize("fenced", [False, True])
@pytest.mark.parametrize("consumes", [True, False])
async def test_the_steering_ack_says_queued_once_the_fence_is_up(
monkeypatch, name, fenced, consumes
):
from xagent.web.api import websocket as ws
from xagent.web.services.task_command_transport import TaskCommandKind
class _Enqueued:
created = False
payload_matches = True
client_command_id = "cmd-1"
command_id = 5
async def enqueue(**_k):
return _Enqueued()
sent: list[dict] = []
async def record(message, _websocket):
sent.append(message)
async def dispatch(*_a, **_k) -> None:
return None
monkeypatch.setattr(ws, "_enqueue_websocket_task_command", enqueue)
monkeypatch.setattr(ws.manager, "send_personal_message", record)
monkeypatch.setattr(ws, "dispatch_task_command_promptly", dispatch)
monkeypatch.setattr(ws, "consumes_task_commands", lambda: consumes)
if fenced:
app_module.release_hold.mark_writers_retired()
handler = ws.handle_pause_task if name == "PAUSE" else ws.handle_resume_task
await handler(object(), 9, {"user": object()}) # type: ignore[arg-type]
(ack,) = sent
assert ack["command"] == TaskCommandKind[name].value
assert ack["task_id"] == 9 and ack["command_id"] == "cmd-1"
# A web role that only enqueues (shared execution) is never fenced out of
# applying: the execution workers apply the command as usual.
if fenced and consumes:
assert ack["type"] == "task_command_queued"
assert ack["applied_after_restart"] is True
else:
assert ack["type"] == "task_command_accepted"
assert "applied_after_restart" not in ack
# -- a prompt dispatch is counted from the moment it exists ---------------------------------
async def test_a_dispatch_in_its_inline_wait_is_seen_by_the_drain(monkeypatch):
from xagent.web.services import task_command_transport as transport
started = asyncio.Event()
async def dispatch_one(_executor, *, command_db_id):
started.set()
await asyncio.sleep(60)
return True
monkeypatch.setattr(transport, "dispatch_one_task_command", dispatch_one)
monkeypatch.setattr(transport, "consumes_task_commands", lambda: True)
# Hold the caller inside its inline wait long enough to retire meanwhile.
real_wait_for = asyncio.wait_for
entered = asyncio.Event()
async def slow_wait_for(awaitable, timeout):
entered.set()
return await real_wait_for(awaitable, 5)
monkeypatch.setattr(transport.asyncio, "wait_for", slow_wait_for)
caller = asyncio.create_task(
transport.dispatch_task_command_promptly(object(), command_db_id=3) # type: ignore[arg-type]
)
await entered.wait()
await started.wait()
assert transport.prompt_dispatch_count() == 1
# What the retire does: the dispatch is cancelled and waited for even
# though its inline wait has not ended.
assert await transport.drain_prompt_dispatches(2.0)
# The drain cancelled the dispatch, not the caller: the socket handler
# carries on (the command row is durable) instead of dying.
await caller
assert not caller.cancelled()
assert transport.prompt_dispatch_count() == 0
assert not transport._prompt_dispatch_tasks
async def test_a_dispatch_that_finishes_inline_leaves_nothing_registered(monkeypatch):
from xagent.web.services import task_command_transport as transport
async def dispatch_one(_executor, *, command_db_id):
return True
monkeypatch.setattr(transport, "dispatch_one_task_command", dispatch_one)
monkeypatch.setattr(transport, "consumes_task_commands", lambda: True)
await transport.dispatch_task_command_promptly(object(), command_db_id=3) # type: ignore[arg-type]
assert not transport._prompt_dispatch_tasks
async def test_a_dispatch_that_fails_inline_raises_once_and_leaves_nothing_registered(
monkeypatch,
):
from xagent.web.services import task_command_transport as transport
async def dispatch_one(_executor, *, command_db_id):
raise RuntimeError("dispatch failed")
monkeypatch.setattr(transport, "dispatch_one_task_command", dispatch_one)
monkeypatch.setattr(transport, "consumes_task_commands", lambda: True)
with pytest.raises(RuntimeError, match="dispatch failed"):
await transport.dispatch_task_command_promptly(object(), command_db_id=3) # type: ignore[arg-type]
assert not transport._prompt_dispatch_tasks
async def test_a_slow_dispatch_stays_registered_until_it_finishes(monkeypatch):
from xagent.web.services import task_command_transport as transport
gate = asyncio.Event()
async def dispatch_one(_executor, *, command_db_id):
await gate.wait()
return True
monkeypatch.setattr(transport, "dispatch_one_task_command", dispatch_one)
monkeypatch.setattr(transport, "consumes_task_commands", lambda: True)
await transport.dispatch_task_command_promptly(object(), command_db_id=3) # type: ignore[arg-type]
assert transport.prompt_dispatch_count() == 1
gate.set()
for _ in range(50):
if not transport._prompt_dispatch_tasks:
break
await asyncio.sleep(0.01)
assert not transport._prompt_dispatch_tasks
# -- a fence-cancelled builder turn tells the client -----------------------------------------
async def test_a_fence_cancelled_builder_turn_sends_paused_and_reraises(monkeypatch):
from xagent.web.api import websocket as ws
paused: list[bool] = []
async def send_paused(_websocket, **_k) -> None:
paused.append(True)
async def runtime_inputs(*_a, **_k):
await asyncio.sleep(60)
monkeypatch.setattr(ws, "send_execution_paused", send_paused)
monkeypatch.setattr(ws, "create_ephemeral_tracer", lambda **_k: object())
monkeypatch.setattr(
"xagent.web.services.builder_chat_runtime.load_builder_chat_runtime_inputs",
runtime_inputs,
)
class _Socket:
state = type("S", (), {})()
task = asyncio.create_task(
ws.handle_builder_chat(
_Socket(), # type: ignore[arg-type]
{"message": "hi"},
type("U", (), {"id": 1, "is_admin": False})(), # type: ignore[arg-type]
)
)
await asyncio.sleep(0.1)
app_module.release_hold.mark_writers_retired()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert paused == [True]
async def test_a_genuine_builder_cancel_is_not_reported_as_paused(monkeypatch):
from xagent.web.api import websocket as ws
paused: list[bool] = []
async def send_paused(_websocket, **_k) -> None:
paused.append(True)
monkeypatch.setattr(ws, "send_execution_paused", send_paused)
monkeypatch.setattr(ws, "create_ephemeral_tracer", lambda **_k: object())
async def hang(*_a, **_k):
await asyncio.sleep(60)
monkeypatch.setattr(
"xagent.web.services.builder_chat_runtime.load_builder_chat_runtime_inputs",
hang,
)
class _Socket:
state = type("S", (), {})()
task = asyncio.create_task(
ws.handle_builder_chat(
_Socket(), # type: ignore[arg-type]
{"message": "hi"},
type("U", (), {"id": 1, "is_admin": False})(), # type: ignore[arg-type]
)
)
await asyncio.sleep(0.1)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert paused == []
# -- the preview enqueue is part of the preview's own write ----------------------------------
async def test_a_fence_that_rises_after_the_preview_row_does_not_refuse_the_enqueue(
monkeypatch,
):
from xagent.web.api import websocket as ws
from xagent.web.services.task_command_transport import TaskCommandKind
created: list[int] = []
enqueued: list[int] = []
async def create(*_a, **_k) -> int:
created.append(7)
# The row is written; only now does the host raise the fence.
app_module.release_hold.mark_writers_retired()
return 7
def fake_enqueue(**kwargs):
enqueued.append(kwargs["task_id"])
return object()
async def send_chat(_websocket, task_id, user, _message):
# What handle_chat_message does for the first preview message.
await ws._enqueue_websocket_task_command(
task_id=task_id,
message_data={"user": user},
kind=TaskCommandKind.MESSAGE,
)
class _Socket:
state = type("S", (), {})()
monkeypatch.setattr(ws, "_create_preview_task", create)
monkeypatch.setattr(ws, "_send_preview_chat", send_chat)
monkeypatch.setattr(ws, "_enqueue_websocket_task_command_sync", fake_enqueue)
monkeypatch.setattr(ws.manager, "register_connection", lambda *_a, **_k: None)
await ws.handle_build_preview_execution(
_Socket(), # type: ignore[arg-type]
{"message": "hello"},
type("U", (), {"id": 1, "is_admin": False})(), # type: ignore[arg-type]
)
assert created == [7] and enqueued == [7]
assert not app_module.release_hold._socket_writes
# The hold is scoped to that handler: a later enqueue is fenced again.
from xagent.web.services.client_error_messages import ClientErrorCode
with pytest.raises(ws.ClientVisibleValidationError) as caught:
await ws._enqueue_websocket_task_command(
task_id=1,
message_data={"user": type("U", (), {"id": 1, "is_admin": False})()},
kind=TaskCommandKind.MESSAGE,
)
assert caught.value.error_code is ClientErrorCode.EXECUTION_PAUSED
# -- run_tracked's runner ------------------------------------------------------------------------
async def test_the_runner_of_an_abandoned_call_does_nothing_and_leaves_no_count(
monkeypatch,
):
from xagent.web.services import writer_workers
runners: list = []
async def never_started(runner, *_a, **_k):
runners.append(runner)
await asyncio.Event().wait()
monkeypatch.setattr(writer_workers.asyncio, "to_thread", never_started)
ran: list[bool] = []
task = asyncio.create_task(run_tracked(lambda: ran.append(True)))
await asyncio.sleep(0)
task.cancel()
with suppress(asyncio.CancelledError):
await task
assert active_workers() == 0
# The executor picks the abandoned work item up late.
(runner,) = runners
assert runner() is None
assert ran == []
assert active_workers() == 0
async def test_a_drain_ends_even_when_the_count_went_negative():
from xagent.web.services import writer_workers
writer_workers._active = -1
try:
assert await drain_workers(0.5)
finally:
writer_workers._active = 0
# -- the blocking-call seam core uses -----------------------------------------------------------
async def test_a_core_blocking_call_is_counted_only_inside_the_tracking_scope():
from xagent.core.utils.blocking import run_blocking
from xagent.web.services.writer_workers import tracked_blocking_scope
release = threading.Event()
started = threading.Event()
counts: list[int] = []
def work() -> str:
counts.append(active_workers())
started.set()
release.wait(5)
return "done"
# Outside a scope (an execution, an HTTP route): not counted.
release.set()
assert await run_blocking(work) == "done"
assert counts == [0]
release.clear()
started.clear()
counts.clear()
with tracked_blocking_scope():
awaiting = asyncio.create_task(run_blocking(work))
while not started.is_set():
await asyncio.sleep(0.01)
awaiting.cancel()
with suppress(asyncio.CancelledError):
await awaiting
# The cancelled await does not end the thread, and the count says so.
assert counts == [1]
assert active_workers() == 1
assert not await drain_workers(0.2)
release.set()
assert await drain_workers(5.0)
async def test_a_core_blocking_call_keeps_the_callers_context_and_the_registration_is_idempotent():
import contextvars
from xagent.core.utils.blocking import get_blocking_tracker, run_blocking
from xagent.web.services import writer_workers
var: contextvars.ContextVar[str] = contextvars.ContextVar("v", default="unset")
var.set("caller")
before = get_blocking_tracker()
writer_workers.install_blocking_tracker()
writer_workers.install_blocking_tracker()
assert get_blocking_tracker() is before
assert await run_blocking(var.get) == "caller"
with writer_workers.tracked_blocking_scope():
assert await run_blocking(var.get) == "caller"
async def test_core_blocking_without_a_tracker_is_a_plain_executor_call():
from xagent.core.utils.blocking import (
get_blocking_tracker,
run_blocking,
set_blocking_tracker,
)
previous = get_blocking_tracker()
set_blocking_tracker(None)
try:
assert await run_blocking(lambda a, b=0: a + b, 1, b=2) == 3
assert active_workers() == 0
finally:
set_blocking_tracker(previous)
async def test_a_builder_turn_runs_in_the_tracking_scope(monkeypatch):
from xagent.core.utils.blocking import run_blocking
from xagent.web.api import websocket as ws
seen: list[int] = []
async def turn(*_a, **_k) -> None:
def work() -> None:
seen.append(active_workers())
await run_blocking(work)
monkeypatch.setattr(ws, "handle_builder_chat", turn)
await ws._run_builder_turn(object(), {}, object()) # type: ignore[arg-type]
assert seen == [1]
assert active_workers() == 0
def test_a_tracked_thread_is_counted_from_before_it_starts_until_it_returns():
from xagent.web.services.writer_workers import start_tracked_thread
release = threading.Event()
thread = start_tracked_thread(release.wait, args=(5,), name="t")
assert active_workers() == 1
release.set()
thread.join(5)
assert active_workers() == 0
async def test_a_foreign_cancel_beside_the_fence_cancel_is_not_swallowed(monkeypatch):
from xagent.web.api import speech
from xagent.web.services import speech_streams
blocked = asyncio.Event()
async def authenticated(*_a, **_k):
return type("U", (), {"id": 1})()
async def accept(_websocket) -> None:
return None
async def receive(_websocket):
blocked.set()
await asyncio.sleep(60)
monkeypatch.setattr(speech, "get_authenticated_user", authenticated)
monkeypatch.setattr(speech, "accept_authenticated_websocket", accept)
monkeypatch.setattr(speech_streams, "_receive_asr_message", receive)
socket = _AsrSocket()
handler = asyncio.create_task(
speech.stream_asr.__wrapped__(socket, token=None) # type: ignore[attr-defined]
)
await blocked.wait()
app_module.release_hold.mark_writers_retired() # cancel number one
handler.cancel() # a foreign cancel (server shutdown) lands with it
with suppress(asyncio.CancelledError):
await handler
assert handler.cancelled()
assert await app_module.release_hold.drain_socket_writes(1.0)