Enterprise agent platform for building, deploying, and operating AI teammates.
import asyncio
from types import SimpleNamespace
import pytest
from xagent.web.channels.feishu.bot import FeishuBotInstance, FeishuChannelManager
from xagent.web.models.task import TaskStatus
from xagent.web.services.assistant_history_safety import (
INPUT_REVIEW_REJECTED_MESSAGE_TYPE,
)
from xagent.web.services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE
from xagent.web.services.llm_utils import AutoModelUnavailableError
from xagent.web.services.task_execution_context_service import (
TaskExecutionRecoverySnapshot,
)
from xagent.web.services.task_lease_service import TaskLease
# Bound shutdown waits so a regression fails instead of hanging the CI job.
# Five seconds also leaves enough headroom for a loaded xdist worker; the
# production Feishu disconnect path deliberately yields for 100 ms first.
_TEST_TIMEOUT_SECONDS = 5.0
def make_bot() -> FeishuBotInstance:
bot = object.__new__(FeishuBotInstance)
bot._initialize_batch_control()
bot.user_active_trace_handlers = {}
bot.control_tasks = set()
bot.control_queues = {}
bot.control_locks = {}
bot._accepting = True
bot._ingress_stopped = False
bot._stop_lock = None
bot._stop_loop = None
bot.ws_client = None
bot._ping_task = None
bot.user_message_queues = {}
bot.user_message_tasks = {}
return bot
@pytest.mark.asyncio
@pytest.mark.parametrize("auto_unavailable", [False, True])
async def test_error_after_prepare_settles_preclaimed_task_instead_of_orphaning_it(
monkeypatch: pytest.MonkeyPatch,
auto_unavailable: bool,
) -> None:
bot = object.__new__(FeishuBotInstance)
bot._initialize_batch_control()
bot.user_active_trace_handlers = {}
bot.control_tasks = set()
bot.control_queues = {}
bot.control_locks = {}
bot.channel_id = 1
bot.channel_name = "Feishu prepare failure"
bot.active_tasks = {}
bot.api_client = object()
bot._save_active_tasks = lambda: True
failure = (
AutoModelUnavailableError("private model details")
if auto_unavailable
else RuntimeError("snapshot failed")
)
monkeypatch.setattr(
"xagent.web.channels.feishu.bot.load_task_setup_snapshot_sync",
lambda *_args: (_ for _ in ()).throw(failure),
)
lease = TaskLease(task_id=45, runner_id="runner-a", run_id="run-a")
finalized: list[TaskStatus] = []
class FakeManagedLease:
heartbeat_task = None
def __init__(self) -> None:
self.lease = lease
self.closed = False
async def finalize_result(self, *, status: TaskStatus, **_kwargs) -> bool:
finalized.append(status)
return True
async def close(self) -> bool:
self.closed = True
return True
managed = FakeManagedLease()
async def prepare(**_kwargs): # type: ignore[no-untyped-def]
return SimpleNamespace(
user_id=5,
task_id=45,
is_new_task=True,
managed_lease=managed,
)
sent_messages: list[str] = []
async def send_text(_chat_id: str, text: str) -> str:
sent_messages.append(text)
return "message-id"
monkeypatch.setattr(
"xagent.web.channels.feishu.bot.prepare_channel_task",
prepare,
)
bot._send_text = send_text
message = SimpleNamespace(
event=SimpleNamespace(
message=SimpleNamespace(
chat_id="chat-id",
message_id="message-id",
message_type="text",
content='{"text": "hello"}',
)
)
)
await bot._process_messages_batch("open-id", [message])
assert finalized == [TaskStatus.FAILED]
assert managed.closed is True
assert sent_messages == [
(
CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE
if auto_unavailable
else "Sorry, an error occurred while processing your request."
)
+ " (Task ID: 45)"
]
@pytest.mark.asyncio
async def test_channel_failure_suppresses_stale_error_after_exact_settlement_rejected(
monkeypatch: pytest.MonkeyPatch,
) -> None:
bot = object.__new__(FeishuBotInstance)
bot._initialize_batch_control()
bot.user_active_trace_handlers = {}
bot.control_tasks = set()
bot.control_queues = {}
bot.control_locks = {}
bot.channel_id = 1
bot.channel_name = "Feishu exact settlement"
bot.active_tasks = {}
bot.api_client = object()
bot._save_active_tasks = lambda: True
lease = TaskLease(task_id=45, runner_id="runner-a", run_id="shared-run")
class FakeManagedLease:
heartbeat_task = None
def __init__(self) -> None:
self.lease = lease
self.closed = False
async def close(self) -> bool:
self.closed = True
return True
async def finalize_result(
self,
*,
status: TaskStatus,
**_kwargs,
) -> bool:
settlements.append((self.lease, status))
return False
managed = FakeManagedLease()
settlements: list[tuple[TaskLease, TaskStatus]] = []
class FakeTracer:
def __init__(self) -> None:
self.handlers: list[object] = []
def add_handler(self, handler: object) -> None:
self.handlers.append(handler)
def remove_handler(self, handler: object) -> None:
if handler in self.handlers:
self.handlers.remove(handler)
agent_service = SimpleNamespace(
tracer=FakeTracer(),
set_conversation_history=lambda _messages,
*,
watermark=None,
event_watermark=None: None,
set_execution_context_messages=lambda _messages: None,
set_recovered_skill_context=lambda _context: None,
)
class FakeAgentManager:
async def get_agent_for_task(self, *_args, **_kwargs): # type: ignore[no-untyped-def]
return agent_service
async def execute_task(self, **_kwargs): # type: ignore[no-untyped-def]
raise RuntimeError("channel execution failed")
sent_messages: list[str] = []
async def send_text(_chat_id: str, text: str) -> str:
sent_messages.append(text)
return "loading-message-id"
async def fake_prepare(**_kwargs): # type: ignore[no-untyped-def]
return SimpleNamespace(
user_id=5,
task_id=45,
is_new_task=True,
managed_lease=managed,
)
monkeypatch.setattr(
"xagent.web.channels.feishu.bot.prepare_channel_task",
fake_prepare,
)
monkeypatch.setattr(
"xagent.web.channels.feishu.bot.load_task_setup_snapshot_sync",
lambda *_args: SimpleNamespace(
runtime_user=None,
conversation_history=(),
conversation_watermark=None,
conversation_event_watermark=None,
execution_recovery=TaskExecutionRecoverySnapshot(),
# The turn binds ``task.source`` into the agent context (MCP
# approval gate identity), so the stand-in row carries the
# ``Task.source`` column default a channel-created task gets.
task=SimpleNamespace(source="internal"),
),
)
monkeypatch.setattr(
"xagent.web.channels.feishu.bot.get_agent_manager",
lambda: FakeAgentManager(),
)
async def persist_message(**_kwargs) -> None: # type: ignore[no-untyped-def]
return None
monkeypatch.setattr(
"xagent.web.channels.feishu.bot.persist_channel_user_message",
persist_message,
)
bot._send_text = send_text
message = SimpleNamespace(
event=SimpleNamespace(
message=SimpleNamespace(
chat_id="chat-id",
message_id="message-id",
message_type="text",
content='{"text": "hello"}',
)
)
)
await bot._process_messages_batch("open-id", [message])
assert settlements == [(lease, TaskStatus.FAILED)]
assert managed.closed is True
assert sent_messages == [
"⏳ **Task #45 is processing...**\n_Please wait for the result._"
]
@pytest.mark.parametrize(
(
"execution_result",
"expected_status",
"expected_message_type",
"expected_content",
"expected_error",
"expected_chat_text",
),
[
(
{"success": True, "output": "completed reply"},
TaskStatus.COMPLETED,
"assistant_response",
"completed reply",
None,
"completed reply",
),
(
{
"success": True,
"completion_outcome": "blocked",
"input_review_rejected": True,
"output": "Input declined.",
},
TaskStatus.COMPLETED,
INPUT_REVIEW_REJECTED_MESSAGE_TYPE,
"Input declined.",
None,
"Input declined.",
),
(
{
"success": False,
"status": "waiting_for_user",
"chat_response": {
"message": "Please confirm",
"interactions": [{"label": "Continue?", "options": ["Yes", "No"]}],
},
},
TaskStatus.WAITING_FOR_USER,
"question",
"Please confirm",
None,
"Please confirm\n\n• Continue?\n Options: Yes, No",
),
(
{
"success": False,
"status": "error",
"output": "provider token=secret",
"error": "provider token=secret",
},
TaskStatus.FAILED,
"assistant_response",
"Task execution failed.",
"provider token=secret",
# The direct path appends the task id to the chat text only; the
# persisted transcript above stays exactly "Task execution failed.".
"Task execution failed. (Task ID: 45)",
),
],
)
@pytest.mark.asyncio
async def test_successful_channel_turn_persists_user_before_exact_assistant_settlement(
monkeypatch: pytest.MonkeyPatch,
execution_result: dict,
expected_status: TaskStatus,
expected_message_type: str,
expected_content: str,
expected_error: str | None,
expected_chat_text: str,
) -> None:
bot = object.__new__(FeishuBotInstance)
bot._initialize_batch_control()
bot.user_active_trace_handlers = {}
bot.control_tasks = set()
bot.control_queues = {}
bot.control_locks = {}
bot.channel_id = 1
bot.channel_name = "Feishu history"
bot.active_tasks = {"open-id": "45"}
bot.api_client = object()
bot._save_active_tasks = lambda: True
events: list[str] = []
finalized: list[dict] = []
connector_turn_ids: list[str | None] = []
execution_turn_ids: list[str] = []
persisted_turn_ids: list[str] = []
lease = TaskLease(task_id=45, runner_id="runner-a", run_id="shared-run")
class FakeManagedLease:
heartbeat_task = None
def __init__(self, claimed_lease: TaskLease) -> None:
self.lease = claimed_lease
async def close(self) -> bool:
return True
async def finalize_result(self, **kwargs) -> bool: # type: ignore[no-untyped-def]
events.append("assistant-settlement")
finalized.append(kwargs)
return True
class FakeTracer:
def __init__(self) -> None:
self.handlers: list[object] = []
def add_handler(self, handler: object) -> None:
self.handlers.append(handler)
def remove_handler(self, handler: object) -> None:
if handler in self.handlers:
self.handlers.remove(handler)
agent_service = SimpleNamespace(
tracer=FakeTracer(),
set_conversation_history=lambda _messages,
*,
watermark=None,
event_watermark=None: None,
set_execution_context_messages=lambda _messages: None,
set_recovered_skill_context=lambda _context: None,
)
class FakeAgentManager:
async def get_agent_for_task(self, *_args, **_kwargs): # type: ignore[no-untyped-def]
connector_turn_ids.append(_kwargs.get("connector_runtime_turn_id"))
return agent_service
async def execute_task(self, **_kwargs): # type: ignore[no-untyped-def]
execution_turn_ids.append(_kwargs["context"]["turn_id"])
events.append("execute")
return execution_result
managed = FakeManagedLease(lease)
async def fake_prepare(**_kwargs): # type: ignore[no-untyped-def]
return SimpleNamespace(
user_id=5,
task_id=45,
is_new_task=False,
managed_lease=managed,
)
async def persist_message(**kwargs) -> None: # type: ignore[no-untyped-def]
assert kwargs["task_id"] == 45
assert kwargs["user_id"] == 5
assert kwargs["content"] == "hello"
persisted_turn_ids.append(kwargs["turn_id"])
events.append("user-message")
async def send_text(_chat_id: str, _text: str) -> str:
return "loading-message-id"
chat_texts: list[str] = []
async def update_text(*args, **_kwargs) -> None: # type: ignore[no-untyped-def]
chat_texts.append(args[2])
monkeypatch.setattr(
"xagent.web.channels.feishu.bot.prepare_channel_task",
fake_prepare,
)
monkeypatch.setattr(
"xagent.web.channels.feishu.bot.load_task_setup_snapshot_sync",
lambda *_args: SimpleNamespace(
runtime_user=None,
conversation_history=(),
conversation_watermark=None,
conversation_event_watermark=None,
execution_recovery=TaskExecutionRecoverySnapshot(),
# The turn binds ``task.source`` into the agent context (MCP
# approval gate identity), so the stand-in row carries the
# ``Task.source`` column default a channel-created task gets.
task=SimpleNamespace(source="internal"),
),
)
monkeypatch.setattr(
"xagent.web.channels.feishu.bot.persist_channel_user_message",
persist_message,
)
monkeypatch.setattr(
"xagent.web.channels.feishu.bot.get_agent_manager",
lambda: FakeAgentManager(),
)
bot._send_text = send_text
bot._update_text = update_text
message = SimpleNamespace(
event=SimpleNamespace(
message=SimpleNamespace(
chat_id="chat-id",
message_id="message-id",
message_type="text",
content='{"text": "hello"}',
)
)
)
await bot._process_messages_batch("open-id", [message])
assert events == ["user-message", "execute", "assistant-settlement"]
assert chat_texts == [expected_chat_text]
assert connector_turn_ids == execution_turn_ids == persisted_turn_ids
assert len(connector_turn_ids) == 1
assert connector_turn_ids[0]
assert finalized == [
{
"status": expected_status,
"assistant_content": expected_content,
"turn_id": persisted_turn_ids[0]
if expected_message_type == INPUT_REVIEW_REJECTED_MESSAGE_TYPE
else None,
"interactions": (
[{"label": "Continue?", "options": ["Yes", "No"]}]
if expected_status == TaskStatus.WAITING_FOR_USER
else []
),
"message_type": expected_message_type,
"error_message": expected_error,
"execution_result": execution_result,
}
]
class _FakeFeishuWebSocketClient:
def __init__(
self,
*,
disconnect_started: asyncio.Event | None = None,
allow_disconnect: asyncio.Event | None = None,
disconnect_error: Exception | None = None,
) -> None:
self._auto_reconnect = False
self._reconnect = None
self.disconnect_started = disconnect_started
self.allow_disconnect = allow_disconnect
self.disconnect_error = disconnect_error
self.disconnect_calls = 0
async def _disconnect(self) -> None:
self.disconnect_calls += 1
if self.disconnect_started is not None:
self.disconnect_started.set()
if self.allow_disconnect is not None:
await self.allow_disconnect.wait()
if self.disconnect_error is not None:
raise self.disconnect_error
def _feishu_message(open_id: str = "open-id") -> SimpleNamespace:
return SimpleNamespace(
event=SimpleNamespace(
sender=SimpleNamespace(
sender_id=SimpleNamespace(open_id=open_id),
),
message=SimpleNamespace(create_time=None),
)
)
@pytest.mark.asyncio
async def test_stop_drains_each_feishu_turn_once_before_clearing_runtime_state() -> (
None
):
bot = make_bot()
client = _FakeFeishuWebSocketClient()
bot.ws_client = client
cleanup_started = asyncio.Event()
allow_cleanup = asyncio.Event()
cleanup_finished = asyncio.Event()
async def running_turn() -> None:
try:
await asyncio.Event().wait()
finally:
cleanup_started.set()
await allow_cleanup.wait()
cleanup_finished.set()
turn_task = asyncio.create_task(running_turn())
await asyncio.sleep(0)
bot.user_message_tasks = {"one": turn_task, "two": turn_task}
bot.user_message_queues = {"one": ["queued"]}
stop_task = asyncio.create_task(bot.stop())
try:
await asyncio.wait_for(cleanup_started.wait(), timeout=_TEST_TIMEOUT_SECONDS)
await asyncio.sleep(0)
assert turn_task.cancelling() == 1
assert not stop_task.done()
assert bot.user_message_tasks == {"one": turn_task, "two": turn_task}
assert bot.user_message_queues == {"one": ["queued"]}
allow_cleanup.set()
await asyncio.wait_for(stop_task, timeout=_TEST_TIMEOUT_SECONDS)
assert cleanup_finished.is_set()
assert bot.user_message_tasks == {}
assert bot.user_message_queues == {}
assert client.disconnect_calls == 1
finally:
allow_cleanup.set()
if not turn_task.done():
turn_task.cancel()
await asyncio.gather(turn_task, return_exceptions=True)
if not stop_task.done():
stop_task.cancel()
await asyncio.gather(stop_task, return_exceptions=True)
@pytest.mark.asyncio
async def test_stop_drains_feishu_ping_cleanup() -> None:
bot = make_bot()
bot.ws_client = _FakeFeishuWebSocketClient()
cleanup_started = asyncio.Event()
allow_cleanup = asyncio.Event()
cleanup_finished = asyncio.Event()
async def ping_loop() -> None:
try:
await asyncio.Event().wait()
finally:
cleanup_started.set()
await allow_cleanup.wait()
cleanup_finished.set()
ping_task = asyncio.create_task(ping_loop())
await asyncio.sleep(0)
bot._ping_task = ping_task
stop_task = asyncio.create_task(bot.stop())
try:
await asyncio.wait_for(cleanup_started.wait(), timeout=_TEST_TIMEOUT_SECONDS)
await asyncio.sleep(0)
assert not stop_task.done()
assert not cleanup_finished.is_set()
allow_cleanup.set()
await asyncio.wait_for(stop_task, timeout=_TEST_TIMEOUT_SECONDS)
assert cleanup_finished.is_set()
assert ping_task.done()
assert bot._ping_task is None
finally:
allow_cleanup.set()
if not ping_task.done():
ping_task.cancel()
await asyncio.gather(ping_task, return_exceptions=True)
if not stop_task.done():
stop_task.cancel()
await asyncio.gather(stop_task, return_exceptions=True)
@pytest.mark.asyncio
async def test_stop_drains_feishu_ping_cleanup_after_disconnect_failure() -> None:
bot = make_bot()
bot.ws_client = _FakeFeishuWebSocketClient(
disconnect_error=RuntimeError("disconnect failed")
)
cleanup_started = asyncio.Event()
allow_cleanup = asyncio.Event()
async def ping_loop() -> None:
try:
await asyncio.Event().wait()
finally:
cleanup_started.set()
await allow_cleanup.wait()
ping_task = asyncio.create_task(ping_loop())
await asyncio.sleep(0)
bot._ping_task = ping_task
stop_task = asyncio.create_task(bot.stop())
try:
await asyncio.wait_for(cleanup_started.wait(), timeout=_TEST_TIMEOUT_SECONDS)
assert not stop_task.done()
allow_cleanup.set()
with pytest.raises(RuntimeError, match="disconnect failed"):
await asyncio.wait_for(stop_task, timeout=_TEST_TIMEOUT_SECONDS)
assert ping_task.done()
assert bot._ping_task is None
finally:
allow_cleanup.set()
if not ping_task.done():
ping_task.cancel()
await asyncio.gather(ping_task, return_exceptions=True)
if not stop_task.done():
stop_task.cancel()
await asyncio.gather(stop_task, return_exceptions=True)
@pytest.mark.asyncio
async def test_feishu_stop_cancellation_waits_for_turn_cleanup_before_propagating() -> (
None
):
bot = make_bot()
bot.ws_client = _FakeFeishuWebSocketClient()
cleanup_started = asyncio.Event()
allow_cleanup = asyncio.Event()
async def running_turn() -> None:
try:
await asyncio.Event().wait()
finally:
cleanup_started.set()
await allow_cleanup.wait()
turn_task = asyncio.create_task(running_turn())
await asyncio.sleep(0)
bot.user_message_tasks = {"open-id": turn_task}
bot.user_message_queues = {"open-id": ["queued"]}
stop_task = asyncio.create_task(bot.stop())
try:
await asyncio.wait_for(cleanup_started.wait(), timeout=_TEST_TIMEOUT_SECONDS)
stop_task.cancel()
await asyncio.sleep(0)
assert not stop_task.done()
assert bot.user_message_tasks == {"open-id": turn_task}
assert bot.user_message_queues == {"open-id": ["queued"]}
allow_cleanup.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(stop_task, timeout=_TEST_TIMEOUT_SECONDS)
assert turn_task.done()
assert bot.user_message_tasks == {}
assert bot.user_message_queues == {}
finally:
allow_cleanup.set()
if not turn_task.done():
turn_task.cancel()
await asyncio.gather(turn_task, return_exceptions=True)
if not stop_task.done():
stop_task.cancel()
await asyncio.gather(stop_task, return_exceptions=True)
@pytest.mark.asyncio
async def test_stop_fence_rejects_late_feishu_ingress() -> None:
bot = make_bot()
bot.start_time = 0
disconnect_started = asyncio.Event()
allow_disconnect = asyncio.Event()
bot.ws_client = _FakeFeishuWebSocketClient(
disconnect_started=disconnect_started,
allow_disconnect=allow_disconnect,
)
stop_task = asyncio.create_task(bot.stop())
try:
await asyncio.wait_for(disconnect_started.wait(), timeout=_TEST_TIMEOUT_SECONDS)
bot._handle_message_sync(_feishu_message())
assert bot.user_message_queues == {}
assert bot.user_message_tasks == {}
finally:
allow_disconnect.set()
await asyncio.gather(stop_task, return_exceptions=True)
def test_feishu_stop_is_concurrent_and_cross_loop_idempotent() -> None:
bot = make_bot()
client = _FakeFeishuWebSocketClient()
bot.ws_client = client
async def stop_concurrently() -> None:
await asyncio.gather(bot.stop(), bot.stop())
asyncio.run(stop_concurrently())
asyncio.run(bot.stop())
assert client.disconnect_calls == 1
@pytest.mark.asyncio
async def test_manager_stop_drains_feishu_polling_cleanup_before_removal() -> None:
manager = FeishuChannelManager()
cleanup_started = asyncio.Event()
allow_cleanup = asyncio.Event()
cleanup_finished = asyncio.Event()
async def polling() -> None:
try:
await asyncio.Event().wait()
finally:
cleanup_started.set()
await allow_cleanup.wait()
cleanup_finished.set()
polling_task = asyncio.create_task(polling())
await asyncio.sleep(0)
async def stop_bot() -> None:
return None
bot = SimpleNamespace(polling_task=polling_task, stop=stop_bot)
manager.bots["app-id"] = bot
stop_task = asyncio.create_task(manager._stop_bot_for_appid("app-id"))
try:
await asyncio.wait_for(cleanup_started.wait(), timeout=_TEST_TIMEOUT_SECONDS)
await asyncio.sleep(0)
assert not stop_task.done()
assert "app-id" in manager.bots
assert not cleanup_finished.is_set()
allow_cleanup.set()
await asyncio.wait_for(stop_task, timeout=_TEST_TIMEOUT_SECONDS)
assert cleanup_finished.is_set()
assert polling_task.done()
assert "app-id" not in manager.bots
finally:
allow_cleanup.set()
if not polling_task.done():
polling_task.cancel()
await asyncio.gather(polling_task, return_exceptions=True)
if not stop_task.done():
stop_task.cancel()
await asyncio.gather(stop_task, return_exceptions=True)
@pytest.mark.asyncio
async def test_manager_stop_is_single_flight_per_feishu_bot() -> None:
manager = FeishuChannelManager()
stop_started = asyncio.Event()
allow_stop = asyncio.Event()
cleanup_started = asyncio.Event()
allow_cleanup = asyncio.Event()
stop_calls = 0
async def polling() -> None:
try:
await asyncio.Event().wait()
finally:
cleanup_started.set()
await allow_cleanup.wait()
polling_task = asyncio.create_task(polling())
await asyncio.sleep(0)
async def stop_bot() -> None:
nonlocal stop_calls
stop_calls += 1
stop_started.set()
await allow_stop.wait()
bot = SimpleNamespace(polling_task=polling_task, stop=stop_bot)
manager.bots["app-id"] = bot
stop_tasks = [
asyncio.create_task(manager._stop_bot_for_appid("app-id")),
asyncio.create_task(manager._stop_bot_for_appid("app-id")),
]
try:
await asyncio.wait_for(stop_started.wait(), timeout=_TEST_TIMEOUT_SECONDS)
await asyncio.sleep(0)
assert stop_calls == 1
allow_stop.set()
await asyncio.wait_for(cleanup_started.wait(), timeout=_TEST_TIMEOUT_SECONDS)
assert polling_task.cancelling() == 1
allow_cleanup.set()
await asyncio.wait_for(
asyncio.gather(*stop_tasks), timeout=_TEST_TIMEOUT_SECONDS
)
assert "app-id" not in manager.bots
finally:
allow_stop.set()
allow_cleanup.set()
if not polling_task.done():
polling_task.cancel()
await asyncio.gather(polling_task, return_exceptions=True)
for stop_task in stop_tasks:
if not stop_task.done():
stop_task.cancel()
await asyncio.gather(*stop_tasks, return_exceptions=True)