:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(tasks): drop a failed connection before each dispatcher poll (#119441)
Alessandro Pogliaghidd11077185bf
diff --git a/products/tasks/backend/management/commands/run_task_workflow_dispatcher.py b/products/tasks/backend/management/commands/run_task_workflow_dispatcher.pyindex 66258165a54f..d8bb366ac1b1 100644--- a/products/tasks/backend/management/commands/run_task_workflow_dispatcher.py+++ b/products/tasks/backend/management/commands/run_task_workflow_dispatcher.py@@ -143,6 +143,7 @@ class Command(BaseCommand):while not stop.is_set():Path("/tmp/dispatcher-heartbeat").touch()try:+ await sync_to_async(close_old_connections)()await sync_to_async(materialize_due_scheduled_task_runs)(SCHEDULED_RUN_MATERIALIZATION_BATCH_SIZE)if monotonic() - last_metrics_sample >= 15:await sync_to_async(sample_dispatch_metrics)()diff --git a/products/tasks/backend/tests/test_workflow_dispatch.py b/products/tasks/backend/tests/test_workflow_dispatch.pyindex c770540f7f5c..5ed589d18b81 100644--- a/products/tasks/backend/tests/test_workflow_dispatch.py+++ b/products/tasks/backend/tests/test_workflow_dispatch.py@@ -1,3 +1,5 @@+import os+import signalimport asyncioimport threadingfrom datetime import timedelta@@ -6,8 +8,8 @@ from typing import castfrom unittest.mock import AsyncMock, Mock, patchfrom django.conf import settings-from django.db import transaction-from django.test import SimpleTestCase, TestCase+from django.db import OperationalError, transaction+from django.test import SimpleTestCase, TestCase, override_settingsfrom django.utils import timezone as django_timezonefrom asgiref.sync import async_to_sync@@ -1077,3 +1079,41 @@ class TestDispatcherCompletionCallback(SimpleTestCase):self.assertEqual(failed_total() - before, expected_delta)self.assertNotIn(task, in_flight)self.assertNotIn(dispatch.id, in_flight_ids)+++class TestDispatcherPollLoop(SimpleTestCase):+ @override_settings(TASKS_DISPATCHER_POLL_INTERVAL_SECONDS=0)+ def test_poll_recovers_after_connection_loss_in_poll_thread(self) -> None:+ broken_threads: set[int] = set()+ materialize_calls = 0++ def materialize(batch_size: int) -> int:+ nonlocal materialize_calls+ materialize_calls += 1+ if materialize_calls == 1:+ broken_threads.add(threading.get_ident())+ if threading.get_ident() in broken_threads:+ raise OperationalError("the connection is lost")+ return 0++ def close_connections() -> None:+ broken_threads.discard(threading.get_ident())++ def claim(*args: object) -> list[TaskWorkflowDispatch]:+ os.kill(os.getpid(), signal.SIGTERM)+ return []++ module = "products.tasks.backend.management.commands.run_task_workflow_dispatcher"+ with (+ patch(f"{module}.async_connect", new_callable=AsyncMock),+ patch(f"{module}.Path"),+ patch(f"{module}.close_old_connections", side_effect=close_connections),+ patch(f"{module}.materialize_due_scheduled_task_runs", side_effect=materialize),+ patch(f"{module}.sample_dispatch_metrics"),+ patch(f"{module}.claim_dispatches", side_effect=claim) as claim_dispatches,+ patch(f"{module}.release_claims"),+ ):+ asyncio.run(asyncio.wait_for(Command()._run(), timeout=10))++ self.assertEqual(materialize_calls, 2)+ claim_dispatches.assert_called_once()