Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cohorts): don't fail cohort calculation when stats task can't be queued (#119309)

posthog[bot]415387806796
diff --git a/products/cohorts/backend/models/test/test_util.py b/products/cohorts/backend/models/test/test_util.py
index 6bdc97a6676e..4ecf1b47747e 100644
--- a/products/cohorts/backend/models/test/test_util.py
+++ b/products/cohorts/backend/models/test/test_util.py
@@ -4,9 +4,10 @@ from posthog.test.base import BaseTest, _create_person, flush_persons_and_events
from unittest.mock import MagicMock, patch
from django.db import DEFAULT_DB_ALIAS, OperationalError
-from django.test import SimpleTestCase
+from django.test import SimpleTestCase, override_settings
from clickhouse_driver.errors import SocketTimeoutError
+from kombu.exceptions import OperationalError as KombuOperationalError
from parameterized import parameterized
from pydantic import (
BaseModel,
@@ -28,6 +29,8 @@ from posthog.models.person.sql import PERSON_STATIC_COHORT_TABLE
from products.cohorts.backend.models.cohort import Cohort, CohortOrEmpty
from products.cohorts.backend.models.util import (
+ COHORT_QUERY_TIMEOUT_SECONDS,
+ COHORT_STATS_COLLECTION_DELAY_SECONDS,
ERROR_CODE_MESSAGES,
CohortErrorCode,
_recalculate_cohortpeople_for_team,
@@ -39,6 +42,7 @@ from products.cohorts.backend.models.util import (
insert_cohort_query_actors_into_ch,
parse_error_code,
print_cohort_hogql_query,
+ run_cohort_query,
simplified_cohort_filter_properties,
sort_cohorts_topologically,
validate_actors_query_for_cohort,
@@ -90,6 +94,46 @@ class TestCohortQueryValidation(SimpleTestCase):
validate_actors_query_for_cohort(query)
+class TestRunCohortQueryStatsScheduling(SimpleTestCase):
+ @override_settings(TEST=False, IN_EVAL_TESTING=False)
+ @patch("products.cohorts.backend.models.util.capture_exception")
+ @patch("posthog.tasks.calculate_cohort.collect_cohort_query_stats")
+ def test_broker_failure_does_not_fail_calculation(
+ self, mock_stats_task: MagicMock, mock_capture: MagicMock
+ ) -> None:
+ mock_stats_task.apply_async.side_effect = KombuOperationalError("Timeout connecting to server")
+ fn = MagicMock(return_value=42)
+
+ result, _ = run_cohort_query(fn, cohort_id=1, history=MagicMock(id=1), query="SELECT 1")
+
+ self.assertEqual(result, 42)
+ fn.assert_called_once()
+ self.assertEqual(mock_stats_task.apply_async.call_count, 1)
+ mock_capture.assert_called_once()
+
+ @override_settings(TEST=False, IN_EVAL_TESTING=False)
+ @patch("products.cohorts.backend.models.util.capture_exception")
+ @patch("posthog.tasks.calculate_cohort.collect_cohort_query_stats")
+ def test_broker_failure_after_query_returns_result(
+ self, mock_stats_task: MagicMock, mock_capture: MagicMock
+ ) -> None:
+ delayed_task = MagicMock()
+ mock_stats_task.apply_async.side_effect = [delayed_task, KombuOperationalError("Timeout connecting to server")]
+ delayed_task.revoke.side_effect = KombuOperationalError("Timeout connecting to server")
+
+ result, _ = run_cohort_query(MagicMock(return_value=42), cohort_id=1, history=MagicMock(id=1), query="SELECT 1")
+
+ self.assertEqual(result, 42)
+ self.assertEqual(
+ [c.kwargs["countdown"] for c in mock_stats_task.apply_async.call_args_list],
+ [
+ COHORT_QUERY_TIMEOUT_SECONDS + COHORT_STATS_COLLECTION_DELAY_SECONDS,
+ COHORT_STATS_COLLECTION_DELAY_SECONDS,
+ ],
+ )
+ self.assertEqual(mock_capture.call_count, 2)
+
+
class TestCohortUtils(BaseTest):
def test_simplified_cohort_filter_properties_static_cohort(self):
_create_person(
diff --git a/products/cohorts/backend/models/util.py b/products/cohorts/backend/models/util.py
index 8a44371297ef..ff2550421626 100644
--- a/products/cohorts/backend/models/util.py
+++ b/products/cohorts/backend/models/util.py
@@ -61,6 +61,9 @@ from products.cohorts.backend.models.dependencies import get_cohort_dependents
from products.cohorts.backend.models.sql import GET_COHORT_SIZE_SQL, RECALCULATE_COHORT_BY_ID
if TYPE_CHECKING:
+ from celery import Task
+ from celery.result import AsyncResult
+
from posthog.personhog_client import ReadConsistency
# temporary marker to denote when cohortpeople table started being populated
@@ -205,6 +208,18 @@ def save_recovery_bookkeeping(save_fn: Callable[[], None], *, cohort_id: int, te
capture_exception(retry_error, additional_properties={"cohort_id": cohort_id, "team_id": team_id})
+def _schedule_cohort_stats_collection(
+ task: Task, args: list[Any], *, countdown: int, cohort_id: int
+) -> AsyncResult | None:
+ """Queue the optional stats task. A broker failure must not fail the cohort calculation."""
+ try:
+ return task.apply_async(args=args, countdown=countdown)
+ except Exception as e:
+ logger.warning("cohort_stats_collection_schedule_failed", cohort_id=cohort_id, exc_info=True)
+ capture_exception(e, additional_properties={"cohort_id": cohort_id})
+ return None
+
+
def run_cohort_query(
fn, *args, cohort_id: int, history: CohortCalculationHistory | None = None, query: str | None = None, **kwargs
):
@@ -241,9 +256,11 @@ def run_cohort_query(
# Schedule delayed task to collect stats after query_log_archive is synced
# Only if we have a history record to update and not in test mode
if history and query and not (settings.TEST or settings.IN_EVAL_TESTING):
- delayed_task = collect_cohort_query_stats.apply_async(
- args=[cohort_tag, cohort_id, start_time.isoformat(), history.id, query],
+ delayed_task = _schedule_cohort_stats_collection(
+ collect_cohort_query_stats,
+ [cohort_tag, cohort_id, start_time.isoformat(), history.id, query],
countdown=COHORT_QUERY_TIMEOUT_SECONDS + COHORT_STATS_COLLECTION_DELAY_SECONDS,
+ cohort_id=cohort_id,
)
try:
@@ -253,13 +270,18 @@ def run_cohort_query(
# If calculation succeeded and we scheduled a delayed task, cancel it and run immediately
# This avoids waiting the full timeout when the query completed quickly
if delayed_task and history and query and not settings.TEST:
- if delayed_task.state in ["PENDING", "RECEIVED"]:
+ try:
delayed_task.revoke() # Cancel the delayed task
+ except Exception as e:
+ logger.warning("cohort_stats_collection_revoke_failed", cohort_id=cohort_id, exc_info=True)
+ capture_exception(e, additional_properties={"cohort_id": cohort_id})
# Run immediately since the query already completed
- collect_cohort_query_stats.apply_async(
- args=[cohort_tag, cohort_id, start_time.isoformat(), history.id, query],
+ _schedule_cohort_stats_collection(
+ collect_cohort_query_stats,
+ [cohort_tag, cohort_id, start_time.isoformat(), history.id, query],
countdown=COHORT_STATS_COLLECTION_DELAY_SECONDS,
+ cohort_id=cohort_id,
)
return result, end_time