Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(platform): harden services for container supervision (#13755)

Nicholas Tindlecea289a48bc9
diff --git a/autogpt_platform/backend/backend/copilot/executor/manager.py b/autogpt_platform/backend/backend/copilot/executor/manager.py
index d3c6e87dfb..e0c9e2c947 100644
--- a/autogpt_platform/backend/backend/copilot/executor/manager.py
+++ b/autogpt_platform/backend/backend/copilot/executor/manager.py
@@ -122,7 +122,15 @@ class CoPilotExecutor(AppProcess):
pool_size_gauge.set(self.pool_size)
self._update_metrics()
- start_http_server(settings.config.copilot_executor_port)
+ # Deliberate reuse of pyro_host: despite the legacy name it is the
+ # bind address for every service's internal listener (see
+ # backend.util.service). Metrics follow the same interface as the RPC
+ # server — 0.0.0.0 under docker-compose (PYRO_HOST is set there for
+ # cross-container scraping), loopback in the single-container runtime.
+ start_http_server(
+ settings.config.copilot_executor_port,
+ addr=settings.config.pyro_host,
+ )
self.cancel_thread.start()
self.run_thread.start()
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py
index 14eb4108d8..f20015ff91 100644
--- a/autogpt_platform/backend/backend/executor/manager.py
+++ b/autogpt_platform/backend/backend/executor/manager.py
@@ -1420,7 +1420,15 @@ class ExecutionManager(AppProcess):
pool_size_gauge.set(self.pool_size)
self._update_prompt_metrics()
- start_http_server(settings.config.execution_manager_port)
+ # Deliberate reuse of pyro_host: despite the legacy name it is the
+ # bind address for every service's internal listener (see
+ # backend.util.service). Metrics follow the same interface as the RPC
+ # server — 0.0.0.0 under docker-compose (PYRO_HOST is set there for
+ # cross-container scraping), loopback in the single-container runtime.
+ start_http_server(
+ settings.config.execution_manager_port,
+ addr=settings.config.pyro_host,
+ )
self.cancel_thread.start()
self.run_thread.start()
diff --git a/autogpt_platform/backend/backend/notifications/notifications.py b/autogpt_platform/backend/backend/notifications/notifications.py
index efd1c971a9..87edaba2ae 100644
--- a/autogpt_platform/backend/backend/notifications/notifications.py
+++ b/autogpt_platform/backend/backend/notifications/notifications.py
@@ -1,5 +1,7 @@
import asyncio
import logging
+from concurrent.futures import Future
+from concurrent.futures import TimeoutError as FutureTimeoutError
from datetime import datetime, timedelta, timezone
from typing import Awaitable, Callable
@@ -55,6 +57,20 @@ settings = Settings()
MAX_CONSUMER_RETRY_ATTEMPTS = 3
CONSUMER_RETRY_BACKOFF_SECONDS = 2
+# Upper bound for each stage of _shutdown_service (consumer-task cancellation
+# and RabbitMQ disconnect). A consumer stuck unwinding on a dead broker
+# connection must not hang cleanup forever — the process has to keep making
+# forward progress so its supervisor sees a clean exit. Mirrors the
+# timeout=10 drain in GraphExecutorManager.cleanup.
+SHUTDOWN_TIMEOUT_SECONDS = 10
+
+# Outer deadline for cleanup()'s cross-thread wait on the _shutdown_service
+# barrier. Each barrier stage is individually bounded by
+# SHUTDOWN_TIMEOUT_SECONDS, so this only fires when the event loop stops
+# before ever servicing the barrier — without it, Future.result() would block
+# process exit indefinitely.
+CLEANUP_TIMEOUT_SECONDS = SHUTDOWN_TIMEOUT_SECONDS * 2 + 5
+
NOTIFICATION_EXCHANGE = Exchange(name="notifications", type=ExchangeType.TOPIC)
DEAD_LETTER_EXCHANGE = Exchange(name="dead_letter", type=ExchangeType.TOPIC)
@@ -218,8 +234,11 @@ class NotificationManager(AppService):
def __init__(self):
super().__init__()
self.rabbitmq_config = create_notification_config()
+ self.rabbitmq_service: rabbitmq.AsyncRabbitMQ | None = None
self.running = True
self.email_sender = EmailSender()
+ self._run_service_future: Future[None] | None = None
+ self._run_service_task: asyncio.Task[None] | None = None
@property
def rabbit(self) -> rabbitmq.AsyncRabbitMQ:
@@ -1115,11 +1134,28 @@ class NotificationManager(AppService):
def run_service(self):
# Queue the main _run_service task
- asyncio.run_coroutine_threadsafe(self._run_service(), self.shared_event_loop)
+ # Keep the concurrent Future alive for as long as the manager. asyncio's
+ # event loop only keeps weak references to tasks, so discarding this
+ # handle can garbage-collect the pending RabbitMQ connection task.
+ self._run_service_future = asyncio.run_coroutine_threadsafe(
+ self._run_service_with_task_reference(), self.shared_event_loop
+ )
# Start the main event loop
super().run_service()
+ async def _run_service_with_task_reference(self) -> None:
+ """Retain the asyncio task so cleanup can await its full cancellation."""
+ task = asyncio.current_task()
+ if task is None:
+ raise RuntimeError("Notification service did not start in an asyncio task")
+ self._run_service_task = task
+ try:
+ await self._run_service()
+ finally:
+ if self._run_service_task is task:
+ self._run_service_task = None
+
@continuous_retry()
async def _run_service(self):
logger.info(f"[{self.service_name}] ⏳ Configuring RabbitMQ...")
@@ -1183,13 +1219,97 @@ class NotificationManager(AppService):
await asyncio.gather(*consumer_tasks, return_exceptions=True)
raise
+ async def _shutdown_service(self) -> None:
+ """Stop consumers completely before closing their RabbitMQ connection."""
+ service_future = self._run_service_future
+ service_task = self._run_service_task
+ # Cleanup can race the tracked coroutine's first loop turn. Yield until
+ # it records the actual asyncio task or its scheduling proxy completes.
+ while (
+ service_task is None
+ and service_future is not None
+ and not service_future.done()
+ ):
+ await asyncio.sleep(0)
+ service_task = self._run_service_task
+
+ if service_task is not None and service_task is not asyncio.current_task():
+ if not service_task.done():
+ service_task.cancel()
+ # Bounded wait: a consumer stuck unwinding (e.g. awaiting a broker
+ # reply on a dead connection) must not hang shutdown forever.
+ _, pending = await asyncio.wait(
+ [service_task], timeout=SHUTDOWN_TIMEOUT_SECONDS
+ )
+ if pending:
+ logger.warning(
+ "Notification service task did not finish cancelling within "
+ f"{SHUTDOWN_TIMEOUT_SECONDS}s; continuing shutdown"
+ )
+
+ if self.rabbitmq_service is not None:
+ logger.info("⏳ Disconnecting RabbitMQ...")
+ # asyncio.wait (not wait_for): on timeout, wait_for would cancel
+ # the disconnect and then await that cancellation, so a
+ # cancellation-resistant disconnect could still hang shutdown.
+ # wait() returns at the deadline and abandons the task instead.
+ disconnect_task = asyncio.ensure_future(self.rabbitmq_service.disconnect())
+ _, pending = await asyncio.wait(
+ [disconnect_task], timeout=SHUTDOWN_TIMEOUT_SECONDS
+ )
+ if pending:
+ disconnect_task.cancel()
+ logger.warning(
+ "RabbitMQ disconnect did not complete within "
+ f"{SHUTDOWN_TIMEOUT_SECONDS}s; continuing shutdown"
+ )
+ elif (
+ not disconnect_task.cancelled()
+ and (exc := disconnect_task.exception()) is not None
+ ):
+ logger.warning(f"RabbitMQ disconnect failed during shutdown: {exc}")
+
def cleanup(self):
- """Cleanup service resources"""
+ """Cleanup service resources."""
self.running = False
- logger.info("⏳ Disconnecting RabbitMQ...")
- self.run_and_wait(self.rabbitmq_service.disconnect())
-
- super().cleanup()
+ try:
+ # Use one coroutine as a shutdown barrier. Cancelling only the
+ # concurrent Future marks that proxy done before the underlying
+ # asyncio task and its consumer children have finished unwinding.
+ shutdown = self._shutdown_service()
+ if self.shared_event_loop.is_closed():
+ # The loop thread already stopped and closed the loop (loop
+ # crash or repeated cleanup): every task on it is gone, and
+ # both scheduling paths below would raise "Event loop is
+ # closed" instead of shutting down.
+ shutdown.close()
+ logger.warning(
+ "Event loop is already closed; "
+ "skipping notification service shutdown"
+ )
+ elif self.shared_event_loop.is_running():
+ try:
+ # Each stage of _shutdown_service is already bounded by
+ # SHUTDOWN_TIMEOUT_SECONDS; this outer deadline only fires
+ # when the loop stops before ever servicing the barrier,
+ # which would otherwise block .result() forever.
+ self.run_and_wait(shutdown, timeout=CLEANUP_TIMEOUT_SECONDS)
+ except FutureTimeoutError:
+ logger.warning(
+ "Notification service shutdown did not run within "
+ f"{CLEANUP_TIMEOUT_SECONDS}s; continuing cleanup"
+ )
+ except RuntimeError as e:
+ # The loop closed between the is_closed() check above and
+ # the scheduling call; nothing is left to shut down.
+ shutdown.close()
+ logger.warning(f"Could not run notification service shutdown: {e}")
+ else:
+ self.shared_event_loop.run_until_complete(shutdown)
+ finally:
+ self._run_service_future = None
+ self._run_service_task = None
+ super().cleanup()
class NotificationManagerClient(AppServiceClient):
diff --git a/autogpt_platform/backend/backend/notifications/test_notifications.py b/autogpt_platform/backend/backend/notifications/test_notifications.py
index a9dd733f34..f312a382a9 100644
--- a/autogpt_platform/backend/backend/notifications/test_notifications.py
+++ b/autogpt_platform/backend/backend/notifications/test_notifications.py
@@ -1,5 +1,7 @@
"""Tests for notification error handling in NotificationManager."""
+import asyncio
+import threading
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, Mock, patch
@@ -584,6 +586,351 @@ class TestNotificationErrorHandling:
assert any("sent and removed" in call.lower() for call in info_calls)
+def test_run_service_retains_background_future():
+ """The RabbitMQ task must remain strongly referenced while it is pending."""
+ manager = NotificationManager.__new__(NotificationManager)
+ manager.shared_event_loop = MagicMock()
+ scheduled_future = MagicMock()
+
+ with patch(
+ "backend.notifications.notifications.asyncio.run_coroutine_threadsafe",
+ return_value=scheduled_future,
+ ) as run_coroutine_threadsafe, patch(
+ "backend.notifications.notifications.AppService.run_service"
+ ) as parent_run_service:
+ manager.run_service()
+
+ wrapper_coroutine = run_coroutine_threadsafe.call_args.args[0]
+ wrapper_coroutine.close()
+ assert manager._run_service_future is scheduled_future
+ parent_run_service.assert_called_once_with()
+
+
+def test_cleanup_awaits_background_task_before_disconnect():
+ manager = NotificationManager.__new__(NotificationManager)
+ manager.running = True
+ events = []
+ manager._run_service_future = MagicMock()
+ # Explicitly mark the scheduling proxy done so the wait loop in
+ # _shutdown_service is skipped intentionally, not via a truthy MagicMock.
+ manager._run_service_future.done.return_value = True
+ manager._run_service_task = None
+ manager.shared_event_loop = MagicMock()
+ manager.shared_event_loop.is_closed.return_value = False
+ manager.shared_event_loop.is_running.return_value = True
+ manager.rabbitmq_service = MagicMock()
+ manager.rabbitmq_service.disconnect = AsyncMock(
+ side_effect=lambda: events.append("disconnect")
+ )
+ manager.run_and_wait = MagicMock(
+ side_effect=lambda coro, timeout=None: asyncio.run(coro)
+ )
+
+ with patch(
+ "backend.notifications.notifications.AppService.cleanup",
+ side_effect=lambda: events.append("parent"),
+ ):
+ manager.cleanup()
+
+ assert manager.running is False
+ assert events == ["disconnect", "parent"]
+ manager.rabbitmq_service.disconnect.assert_awaited_once_with()
+ manager.run_and_wait.assert_called_once()
+ assert manager._run_service_future is None
+
+
+def test_shutdown_tolerates_cancelled_disconnect_task():
+ """A disconnect that ends up cancelled must not crash _shutdown_service.
+
+ Task.exception() raises CancelledError (rather than returning it) on a
+ cancelled task, so the error-logging branch must skip cancelled tasks.
+ """
+ manager = NotificationManager.__new__(NotificationManager)
+ manager._run_service_future = None
+ manager._run_service_task = None
+ manager.rabbitmq_service = MagicMock()
+
+ async def cancelled_disconnect():
+ raise asyncio.CancelledError()
+
+ manager.rabbitmq_service.disconnect = cancelled_disconnect
+
+ # Must complete without propagating CancelledError.
+ asyncio.run(manager._shutdown_service())
+
+
+def test_cleanup_handles_shutdown_before_rabbitmq_startup():
+ manager = NotificationManager.__new__(NotificationManager)
+ manager.running = True
+ manager.rabbitmq_service = None
+ manager._run_service_future = None
+ manager._run_service_task = None
+ manager.shared_event_loop = MagicMock()
+ manager.shared_event_loop.is_closed.return_value = False
+ manager.shared_event_loop.is_running.return_value = True
+ manager.run_and_wait = MagicMock(
+ side_effect=lambda coro, timeout=None: asyncio.run(coro)
+ )
+
+ with patch(
+ "backend.notifications.notifications.AppService.cleanup"
+ ) as parent_cleanup:
+ manager.cleanup()
+
+ assert manager.running is False
+ manager.run_and_wait.assert_called_once()
+ parent_cleanup.assert_called_once_with()
+
+
+def test_cleanup_waits_for_consumer_task_cancellation_to_finish():
+ """The event loop must not close while notification tasks are still pending."""
+ manager = NotificationManager.__new__(NotificationManager)
+ manager.running = True
+ manager._run_service_future = None
+ manager._run_service_task = None
+ manager.shared_event_loop = asyncio.new_event_loop()
+ service_started = threading.Event()
+ service_finished = threading.Event()
+ events = []
+
+ async def slow_to_cancel_service():
+ service_started.set()
+ try:
+ await asyncio.Event().wait()
+ finally:
+ # Model async context-manager cleanup that needs several loop turns.
+ for _ in range(5):
+ await asyncio.sleep(0)
+ service_finished.set()
+ events.append("service-finished")
+
+ manager._run_service = slow_to_cancel_service
+ manager.rabbitmq_service = MagicMock()
+ manager.rabbitmq_service.disconnect = AsyncMock(
+ side_effect=lambda: events.append("disconnect")
+ )
+ loop_thread = threading.Thread(
+ target=manager.shared_event_loop.run_forever,
+ daemon=True,
+ )
+ loop_thread.start()
+ scheduled_future = asyncio.run_coroutine_threadsafe(
+ manager._run_service_with_task_reference(), manager.shared_event_loop
+ )
+ manager._run_service_future = scheduled_future
+ assert service_started.wait(timeout=2)
+
+ def stop_parent_loop():
+ manager.shared_event_loop.call_soon_threadsafe(manager.shared_event_loop.stop)
+
+ try:
+ with patch(
+ "backend.notifications.notifications.AppService.cleanup",
+ side_effect=stop_parent_loop,
+ ):
+ manager.cleanup()
+ loop_thread.join(timeout=2)
+
+ assert service_finished.is_set()
+ assert events == ["service-finished", "disconnect"]
+ assert not loop_thread.is_alive()
+ assert scheduled_future.done()
+ assert not asyncio.all_tasks(manager.shared_event_loop)
+ finally:
+ if loop_thread.is_alive():
+ manager.shared_event_loop.call_soon_threadsafe(
+ manager.shared_event_loop.stop
+ )
+ loop_thread.join(timeout=2)
+ manager.shared_event_loop.close()
+
+
+def test_cleanup_gives_up_on_service_task_that_ignores_cancellation():
+ """A consumer stuck unwinding must not hang cleanup past the timeout."""
+ manager = NotificationManager.__new__(NotificationManager)
+ manager.running = True
+ manager.rabbitmq_service = None
+ manager._run_service_future = None
+ manager._run_service_task = None
+ manager.shared_event_loop = asyncio.new_event_loop()
+ service_started = threading.Event()
+
+ async def uncancellable_service():
+ service_started.set()
+ while True:
+ try:
+ await asyncio.sleep(60)
+ except asyncio.CancelledError:
+ # Model a task wedged in unwinding (e.g. awaiting a broker
+ # reply on a dead connection) that never finishes cancelling.
+ continue
+
+ manager._run_service = uncancellable_service
+ loop_thread = threading.Thread(
+ target=manager.shared_event_loop.run_forever,
+ daemon=True,
+ )
+ loop_thread.start()
+ scheduled_future = asyncio.run_coroutine_threadsafe(
+ manager._run_service_with_task_reference(), manager.shared_event_loop
+ )
+ manager._run_service_future = scheduled_future
+ assert service_started.wait(timeout=2)
+
+ def stop_parent_loop():
+ manager.shared_event_loop.call_soon_threadsafe(manager.shared_event_loop.stop)
+
+ try:
+ with patch(
+ "backend.notifications.notifications.SHUTDOWN_TIMEOUT_SECONDS", 0.1
+ ), patch(
+ "backend.notifications.notifications.AppService.cleanup",
+ side_effect=stop_parent_loop,
+ ):
+ manager.cleanup()
+ loop_thread.join(timeout=2)
+
+ assert not loop_thread.is_alive()
+ assert manager._run_service_future is None
+ assert manager._run_service_task is None
+ finally:
+ if loop_thread.is_alive():
+ manager.shared_event_loop.call_soon_threadsafe(
+ manager.shared_event_loop.stop
+ )
+ loop_thread.join(timeout=2)
+ manager.shared_event_loop.close()
+
+
+def test_cleanup_handles_shutdown_before_service_task_first_loop_turn():
+ manager = NotificationManager.__new__(NotificationManager)
+ manager.running = True
+ manager.rabbitmq_service = None
+ manager._run_service_task = None
+ manager.shared_event_loop = asyncio.new_event_loop()
+ service_finished = threading.Event()
+
+ async def slow_to_cancel_service():
+ try:
+ await asyncio.Event().wait()
+ finally:
+ for _ in range(3):
+ await asyncio.sleep(0)
+ service_finished.set()
+
+ manager._run_service = slow_to_cancel_service
+ scheduled_future = asyncio.run_coroutine_threadsafe(
+ manager._run_service_with_task_reference(), manager.shared_event_loop
+ )
+ manager._run_service_future = scheduled_future
+
+ try:
+ with patch("backend.notifications.notifications.AppService.cleanup"):
+ manager.cleanup()
+ manager.shared_event_loop.run_until_complete(asyncio.sleep(0))
+
+ assert service_finished.is_set()
+ assert scheduled_future.done()
+ assert not asyncio.all_tasks(manager.shared_event_loop)
+ finally:
+ manager.shared_event_loop.close()
+
+
+def test_cleanup_skips_shutdown_when_loop_already_closed():
+ """A crashed-and-closed loop must not turn cleanup into a RuntimeError."""
+ manager = NotificationManager.__new__(NotificationManager)
+ manager.running = True
+ manager.rabbitmq_service = MagicMock()
+ manager.rabbitmq_service.disconnect = AsyncMock()
+ manager._run_service_future = MagicMock()
+ manager._run_service_future.done.return_value = True
+ manager._run_service_task = None
+ manager.shared_event_loop = asyncio.new_event_loop()
+ manager.shared_event_loop.close()
+
+ # Unpatched parent cleanup: BaseAppService.cleanup must also survive the
+ # closed loop instead of raising from call_soon_threadsafe(loop.stop).
+ manager.cleanup()
+
+ assert manager.running is False
+ manager.rabbitmq_service.disconnect.assert_not_awaited()
+ assert manager._run_service_future is None
+ assert manager._run_service_task is None
+
+
+def test_cleanup_survives_loop_closing_between_check_and_schedule():
+ """A loop that closes right after the checks must not crash cleanup."""
+ manager = NotificationManager.__new__(NotificationManager)
+ manager.running = True
+ manager.rabbitmq_service = None
+ manager._run_service_future = None
+ manager._run_service_task = None
+ manager.shared_event_loop = MagicMock()
+ manager.shared_event_loop.is_closed.return_value = False
+ manager.shared_event_loop.is_running.return_value = True
+ manager.run_and_wait = MagicMock(side_effect=RuntimeError("Event loop is closed"))
+
+ with patch(
+ "backend.notifications.notifications.AppService.cleanup"
+ ) as parent_cleanup:
+ manager.cleanup()
+
+ parent_cleanup.assert_called_once_with()
+ assert manager._run_service_future is None
+ assert manager._run_service_task is None
+
+
+def test_cleanup_gives_up_when_loop_never_services_the_barrier():
+ """cleanup() must not block forever if the loop never runs the barrier."""
+ manager = NotificationManager.__new__(NotificationManager)
+ manager.running = True
+ manager.rabbitmq_service = None
+ manager._run_service_future = None
+ manager._run_service_task = None
+ manager.shared_event_loop = asyncio.new_event_loop()
+ loop_hogged = threading.Event()
+ release_loop = threading.Event()
+
+ def hog_the_loop():
+ # Keep the loop stuck in one callback so the shutdown barrier
+ # scheduled by cleanup() never gets serviced before the deadline.
+ loop_hogged.set()
+ release_loop.wait(timeout=5)
+
+ loop_thread = threading.Thread(
+ target=manager.shared_event_loop.run_forever,
+ daemon=True,
+ )
+ loop_thread.start()
+ manager.shared_event_loop.call_soon_threadsafe(hog_the_loop)
+ assert loop_hogged.wait(timeout=2)
+
+ def stop_parent_loop():
+ manager.shared_event_loop.call_soon_threadsafe(manager.shared_event_loop.stop)
+
+ try:
+ with patch(
+ "backend.notifications.notifications.CLEANUP_TIMEOUT_SECONDS", 0.1
+ ), patch(
+ "backend.notifications.notifications.AppService.cleanup",
+ side_effect=stop_parent_loop,
+ ):
+ manager.cleanup()
+
+ # cleanup() returned while the loop was still wedged, so the outer
+ # deadline fired instead of blocking on Future.result() forever.
+ assert not release_loop.is_set()
+ assert manager._run_service_future is None
+ assert manager._run_service_task is None
+ finally:
+ release_loop.set()
+ loop_thread.join(timeout=2)
+ # Drain the abandoned barrier task so closing the loop does not leak
+ # a pending task or a never-awaited coroutine.
+ manager.shared_event_loop.run_until_complete(asyncio.sleep(0))
+ manager.shared_event_loop.close()
+
+
class TestConsumerRetryWithBackoff:
"""Verify _process_message_with_retry retries transient failures and only DLQs after exhausting attempts."""
diff --git a/autogpt_platform/backend/backend/util/service.py b/autogpt_platform/backend/backend/util/service.py
index cf9488036a..767aba85ee 100644
--- a/autogpt_platform/backend/backend/util/service.py
+++ b/autogpt_platform/backend/backend/util/service.py
@@ -135,8 +135,12 @@ class BaseAppService(AppProcess, ABC):
logger.info(f"[{self.service_name}] 🛑 Shared event loop stopped")
self.shared_event_loop.close() # ensure held resources are released
- def run_and_wait(self, coro: Coroutine[Any, Any, T]) -> T:
- return asyncio.run_coroutine_threadsafe(coro, self.shared_event_loop).result()
+ def run_and_wait(
+ self, coro: Coroutine[Any, Any, T], timeout: float | None = None
+ ) -> T:
+ return asyncio.run_coroutine_threadsafe(coro, self.shared_event_loop).result(
+ timeout
+ )
def run(self):
self.shared_event_loop = asyncio.new_event_loop()
@@ -152,8 +156,21 @@ class BaseAppService(AppProcess, ABC):
**Note:** if you override this method in a subclass, it must call
`super().cleanup()` *at the end*!
"""
- # Stop the shared event loop to allow resource clean-up
- self.shared_event_loop.call_soon_threadsafe(self.shared_event_loop.stop)
+ # Stop the shared event loop to allow resource clean-up. The loop
+ # thread closes the loop right after run_forever() returns, so a
+ # closed loop here means that thread is already gone (loop crash or
+ # repeated cleanup) and there is nothing left to stop — scheduling
+ # onto it would raise "Event loop is closed" mid-shutdown.
+ if not self.shared_event_loop.is_closed():
+ try:
+ self.shared_event_loop.call_soon_threadsafe(self.shared_event_loop.stop)
+ except RuntimeError:
+ # The loop thread closed the loop between the check above and
+ # the scheduling call; equivalent to the is_closed() case.
+ logger.warning(
+ f"[{self.service_name}] event loop closed before its stop "
+ "could be scheduled; continuing cleanup"
+ )
super().cleanup()