:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cohorts): don't fail cohort calculation when stats task can't be queued (#119309)
posthog[bot]415387806796
diff --git a/products/cohorts/backend/models/test/test_util.py b/products/cohorts/backend/models/test/test_util.pyindex 6bdc97a6676e..4ecf1b47747e 100644--- a/products/cohorts/backend/models/test/test_util.py+++ b/products/cohorts/backend/models/test/test_util.py@@ -4,9 +4,10 @@ from posthog.test.base import BaseTest, _create_person, flush_persons_and_eventsfrom unittest.mock import MagicMock, patchfrom django.db import DEFAULT_DB_ALIAS, OperationalError-from django.test import SimpleTestCase+from django.test import SimpleTestCase, override_settingsfrom clickhouse_driver.errors import SocketTimeoutError+from kombu.exceptions import OperationalError as KombuOperationalErrorfrom parameterized import parameterizedfrom pydantic import (BaseModel,@@ -28,6 +29,8 @@ from posthog.models.person.sql import PERSON_STATIC_COHORT_TABLEfrom products.cohorts.backend.models.cohort import Cohort, CohortOrEmptyfrom products.cohorts.backend.models.util import (+ COHORT_QUERY_TIMEOUT_SECONDS,+ COHORT_STATS_COLLECTION_DELAY_SECONDS,ERROR_CODE_MESSAGES,CohortErrorCode,_recalculate_cohortpeople_for_team,@@ -39,6 +42,7 @@ from products.cohorts.backend.models.util import (insert_cohort_query_actors_into_ch,parse_error_code,print_cohort_hogql_query,+ run_cohort_query,simplified_cohort_filter_properties,sort_cohorts_topologically,validate_actors_query_for_cohort,@@ -90,6 +94,46 @@ class TestCohortQueryValidation(SimpleTestCase):validate_actors_query_for_cohort(query)+class TestRunCohortQueryStatsScheduling(SimpleTestCase):+ @override_settings(TEST=False, IN_EVAL_TESTING=False)+ @patch("products.cohorts.backend.models.util.capture_exception")+ @patch("posthog.tasks.calculate_cohort.collect_cohort_query_stats")+ def test_broker_failure_does_not_fail_calculation(+ self, mock_stats_task: MagicMock, mock_capture: MagicMock+ ) -> None:+ mock_stats_task.apply_async.side_effect = KombuOperationalError("Timeout connecting to server")+ fn = MagicMock(return_value=42)++ result, _ = run_cohort_query(fn, cohort_id=1, history=MagicMock(id=1), query="SELECT 1")++ self.assertEqual(result, 42)+ fn.assert_called_once()+ self.assertEqual(mock_stats_task.apply_async.call_count, 1)+ mock_capture.assert_called_once()++ @override_settings(TEST=False, IN_EVAL_TESTING=False)+ @patch("products.cohorts.backend.models.util.capture_exception")+ @patch("posthog.tasks.calculate_cohort.collect_cohort_query_stats")+ def test_broker_failure_after_query_returns_result(+ self, mock_stats_task: MagicMock, mock_capture: MagicMock+ ) -> None:+ delayed_task = MagicMock()+ mock_stats_task.apply_async.side_effect = [delayed_task, KombuOperationalError("Timeout connecting to server")]+ delayed_task.revoke.side_effect = KombuOperationalError("Timeout connecting to server")++ result, _ = run_cohort_query(MagicMock(return_value=42), cohort_id=1, history=MagicMock(id=1), query="SELECT 1")++ self.assertEqual(result, 42)+ self.assertEqual(+ [c.kwargs["countdown"] for c in mock_stats_task.apply_async.call_args_list],+ [+ COHORT_QUERY_TIMEOUT_SECONDS + COHORT_STATS_COLLECTION_DELAY_SECONDS,+ COHORT_STATS_COLLECTION_DELAY_SECONDS,+ ],+ )+ self.assertEqual(mock_capture.call_count, 2)++class TestCohortUtils(BaseTest):def test_simplified_cohort_filter_properties_static_cohort(self):_create_person(diff --git a/products/cohorts/backend/models/util.py b/products/cohorts/backend/models/util.pyindex 8a44371297ef..ff2550421626 100644--- a/products/cohorts/backend/models/util.py+++ b/products/cohorts/backend/models/util.py@@ -61,6 +61,9 @@ from products.cohorts.backend.models.dependencies import get_cohort_dependentsfrom products.cohorts.backend.models.sql import GET_COHORT_SIZE_SQL, RECALCULATE_COHORT_BY_IDif TYPE_CHECKING:+ from celery import Task+ from celery.result import AsyncResult+from posthog.personhog_client import ReadConsistency# temporary marker to denote when cohortpeople table started being populated@@ -205,6 +208,18 @@ def save_recovery_bookkeeping(save_fn: Callable[[], None], *, cohort_id: int, tecapture_exception(retry_error, additional_properties={"cohort_id": cohort_id, "team_id": team_id})+def _schedule_cohort_stats_collection(+ task: Task, args: list[Any], *, countdown: int, cohort_id: int+) -> AsyncResult | None:+ """Queue the optional stats task. A broker failure must not fail the cohort calculation."""+ try:+ return task.apply_async(args=args, countdown=countdown)+ except Exception as e:+ logger.warning("cohort_stats_collection_schedule_failed", cohort_id=cohort_id, exc_info=True)+ capture_exception(e, additional_properties={"cohort_id": cohort_id})+ return None++def run_cohort_query(fn, *args, cohort_id: int, history: CohortCalculationHistory | None = None, query: str | None = None, **kwargs):@@ -241,9 +256,11 @@ def run_cohort_query(# Schedule delayed task to collect stats after query_log_archive is synced# Only if we have a history record to update and not in test modeif history and query and not (settings.TEST or settings.IN_EVAL_TESTING):- delayed_task = collect_cohort_query_stats.apply_async(- args=[cohort_tag, cohort_id, start_time.isoformat(), history.id, query],+ delayed_task = _schedule_cohort_stats_collection(+ collect_cohort_query_stats,+ [cohort_tag, cohort_id, start_time.isoformat(), history.id, query],countdown=COHORT_QUERY_TIMEOUT_SECONDS + COHORT_STATS_COLLECTION_DELAY_SECONDS,+ cohort_id=cohort_id,)try:@@ -253,13 +270,18 @@ def run_cohort_query(# If calculation succeeded and we scheduled a delayed task, cancel it and run immediately# This avoids waiting the full timeout when the query completed quicklyif delayed_task and history and query and not settings.TEST:- if delayed_task.state in ["PENDING", "RECEIVED"]:+ try:delayed_task.revoke() # Cancel the delayed task+ except Exception as e:+ logger.warning("cohort_stats_collection_revoke_failed", cohort_id=cohort_id, exc_info=True)+ capture_exception(e, additional_properties={"cohort_id": cohort_id})# Run immediately since the query already completed- collect_cohort_query_stats.apply_async(- args=[cohort_tag, cohort_id, start_time.isoformat(), history.id, query],+ _schedule_cohort_stats_collection(+ collect_cohort_query_stats,+ [cohort_tag, cohort_id, start_time.isoformat(), history.id, query],countdown=COHORT_STATS_COLLECTION_DELAY_SECONDS,+ cohort_id=cohort_id,)return result, end_time