:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(celery): push duration and success when a PushGatewayTask run fails (#121699)
Phil Haack8f7b71ffe02a
diff --git a/posthog/metrics.py b/posthog/metrics.pyindex bd2f87a866d1..c19096a1e584 100644--- a/posthog/metrics.py+++ b/posthog/metrics.py@@ -1,4 +1,5 @@# Shared metrics and labels for prometheus metrics+from collections.abc import Iteratorfrom contextlib import contextmanagerfrom django.conf import settings@@ -9,7 +10,7 @@ import structlog# The pushgateway is an internal service — the outbound proxy would reject it.# ProxyHandler({}) tells urllib to ignore proxy env vars.import prometheus_client.exposition as _expo-from prometheus_client import CollectorRegistry, Counter, push_to_gateway+from prometheus_client import CollectorRegistry, Counter, push_to_gateway, pushadd_to_gatewayfrom posthog.exceptions_capture import capture_exception@@ -59,8 +60,31 @@ def _make_handler_no_proxy(url, method, timeout, headers, data, base_handler):_expo._make_handler = _make_handler_no_proxy # ty: ignore[invalid-assignment]+def push_registry(job_name: str, registry: CollectorRegistry, *, replace_group: bool, timeout: float = 30) -> None:+ """+ Push the registry to the PushGateway under job_name. It logs and captures a failed push instead of raising it.++ With replace_group=True the push is a PUT, which deletes every metric in the job's group that the registry does not carry.+ With replace_group=False the push is a POST, which replaces only the metrics the registry carries.+ The timeout applies to each socket operation of the push. The connect and the response can each take that long.+ """+ try:+ if settings.PROM_PUSHGATEWAY_ADDRESS:+ push = push_to_gateway if replace_group else pushadd_to_gateway+ push(settings.PROM_PUSHGATEWAY_ADDRESS, job=job_name, registry=registry, timeout=timeout)+ except Exception as err:+ logger.exception(+ "push_to_gateway",+ target=settings.PROM_PUSHGATEWAY_ADDRESS,+ job=job_name,+ replace_group=replace_group,+ exception=err,+ )+ capture_exception(err)++@contextmanager-def pushed_metrics_registry(job_name: str):+def pushed_metrics_registry(job_name: str) -> Iterator[CollectorRegistry]:"""Return a temporary Prometheus registry that will be pushed to thePushGateway when the context closes.@@ -68,15 +92,13 @@ def pushed_metrics_registry(job_name: str):Parameter job_name: a unique job name to use, all metrics previouslypushed with that name will be deleted.+ If the body raises, the context pushes nothing. The group keeps the metrics of the last push.+ PushGatewayTask and alerts on a stale push_time_seconds depend on this.+NOTE: only use to expose gauges, for use cases where one value perregion makes sense (e.g. instance metrics computed by celery jobs)."""registry = CollectorRegistry()yield registry- try:- if settings.PROM_PUSHGATEWAY_ADDRESS:- push_to_gateway(settings.PROM_PUSHGATEWAY_ADDRESS, job=job_name, registry=registry)- except Exception as err:- logger.exception("push_to_gateway", target=settings.PROM_PUSHGATEWAY_ADDRESS, exception=err)- capture_exception(err)+ push_registry(job_name, registry, replace_group=True)diff --git a/posthog/tasks/test/test_pushgateway_task.py b/posthog/tasks/test/test_pushgateway_task.pyindex 3cb5833f5e8d..0fe7780b88ca 100644--- a/posthog/tasks/test/test_pushgateway_task.py+++ b/posthog/tasks/test/test_pushgateway_task.py@@ -1,138 +1,103 @@from collections.abc import Generator-from typing import Anyimport pytestfrom unittest.mock import MagicMock, patchfrom celery import shared_task+from celery.exceptions import Ignore, Reject, Retry, SoftTimeLimitExceededfrom prometheus_client import CollectorRegistry, Gauge+from pytest_django import Settings-from posthog.tasks.utils import PushGatewayTask+from posthog.tasks.utils import FAILURE_PUSH_TIMEOUT_SECONDS, PushGatewayTask+TASK_GAUGE = "sample_task_rows_processed"-class TestPushGatewayTask:- @pytest.fixture- def mock_registry(self) -> CollectorRegistry:- return CollectorRegistry()-- @pytest.fixture- def mock_push_context(self, mock_registry: CollectorRegistry) -> Generator[MagicMock]:- with patch("posthog.tasks.utils.pushed_metrics_registry") as mock:- mock_context = MagicMock()- mock_context.__enter__ = MagicMock(return_value=mock_registry)- mock_context.__exit__ = MagicMock(return_value=False)- mock.return_value = mock_context- yield mock-- def test_success_metrics_set_on_successful_task(- self, mock_push_context: MagicMock, mock_registry: CollectorRegistry- ) -> None:- @shared_task(bind=True, base=PushGatewayTask, name="test.successful_task")- def successful_task(self: Any) -> str:- return "done"- result = successful_task()+# Celery keeps the first task registered under a name. A second definition with the same name runs the first body.+# The cases pass the error to one task as an argument instead of defining a task for each case.+@shared_task(bind=True, base=PushGatewayTask, name="test.pushgateway.sample_task")+def sample_task(self: PushGatewayTask, error: Exception | None = None) -> str:+ Gauge(TASK_GAUGE, "Rows the task processed", registry=self.metrics_registry).set(42)+ if error is not None:+ raise error+ return "done"- assert result == "done"- success_sample = mock_registry.get_sample_value("posthog_celery_successful_task_success")- assert success_sample == 1- duration_sample = mock_registry.get_sample_value("posthog_celery_successful_task_duration_seconds")- assert duration_sample is not None- assert duration_sample >= 0-- def test_failure_metrics_set_on_failed_task(- self, mock_push_context: MagicMock, mock_registry: CollectorRegistry- ) -> None:- @shared_task(bind=True, base=PushGatewayTask, name="test.failing_task")- def failing_task(self: Any) -> None:- raise ValueError("test error")- with pytest.raises(ValueError, match="test error"):- failing_task()+def _pushed_registry(push: MagicMock) -> CollectorRegistry:+ push.assert_called_once()+ assert push.call_args.kwargs["job"] == "celery_sample_task"+ registry = push.call_args.kwargs["registry"]+ assert isinstance(registry, CollectorRegistry)+ return registry- success_sample = mock_registry.get_sample_value("posthog_celery_failing_task_success")- assert success_sample == 0- duration_sample = mock_registry.get_sample_value("posthog_celery_failing_task_duration_seconds")- assert duration_sample is not None- assert duration_sample >= 0- def test_custom_metrics_can_be_added(self, mock_push_context: MagicMock, mock_registry: CollectorRegistry) -> None:- @shared_task(bind=True, base=PushGatewayTask, name="test.task_with_custom_metric")- def task_with_custom_metric(self: Any) -> str:- custom = Gauge("custom_metric", "Custom", registry=self.metrics_registry)- custom.set(42)- return "done"-- task_with_custom_metric()+class TestPushGatewayTask:+ @pytest.fixture(autouse=True)+ def pushgateway_address(self, settings: Settings) -> None:+ settings.PROM_PUSHGATEWAY_ADDRESS = "http://pushgateway.example.com:9091"- custom_sample = mock_registry.get_sample_value("custom_metric")- assert custom_sample == 42+ @pytest.fixture+ def push_to_gateway(self) -> Generator[MagicMock]:+ with patch("posthog.metrics.push_to_gateway") as mock:+ yield mock- def test_exception_is_reraised(self, mock_push_context: MagicMock, mock_registry: CollectorRegistry) -> None:- @shared_task(bind=True, base=PushGatewayTask, name="test.exception_task")- def exception_task(self: Any) -> None:- raise RuntimeError("specific error")+ @pytest.fixture+ def pushadd_to_gateway(self) -> Generator[MagicMock]:+ with patch("posthog.metrics.pushadd_to_gateway") as mock:+ yield mock- with pytest.raises(RuntimeError, match="specific error"):- exception_task()+ @pytest.fixture+ def run_duration(self) -> Generator[float]:+ with patch("posthog.tasks.utils.time") as mock_time:+ mock_time.monotonic.side_effect = [100.0, 107.5]+ yield 7.5- def test_duration_is_recorded_even_on_failure(- self, mock_push_context: MagicMock, mock_registry: CollectorRegistry+ def test_completed_run_puts_its_metrics_and_returns_despite_a_push_error(+ self, run_duration: float, push_to_gateway: MagicMock, pushadd_to_gateway: MagicMock) -> None:- @shared_task(bind=True, base=PushGatewayTask, name="test.duration_on_failure_task")- def duration_on_failure_task(self: Any) -> None:- raise ValueError("failure")+ push_to_gateway.side_effect = OSError("pushgateway unreachable")- with pytest.raises(ValueError):- duration_on_failure_task()+ result = sample_task()- duration_sample = mock_registry.get_sample_value("posthog_celery_duration_on_failure_task_duration_seconds")- assert duration_sample is not None- assert duration_sample >= 0-- def test_task_name_extracted_correctly(- self, mock_push_context: MagicMock, mock_registry: CollectorRegistry+ assert result == "done"+ registry = _pushed_registry(push_to_gateway)+ assert registry.get_sample_value("posthog_celery_sample_task_success") == 1+ assert registry.get_sample_value("posthog_celery_sample_task_duration_seconds") == run_duration+ assert registry.get_sample_value(TASK_GAUGE) == 42+ pushadd_to_gateway.assert_not_called()++ @pytest.mark.parametrize(+ "error",+ [ValueError("boom"), SoftTimeLimitExceeded()],+ ids=["value_error", "soft_time_limit"],+ )+ def test_failed_run_posts_only_failure_and_duration_and_reraises_despite_a_push_error(+ self, error: Exception, run_duration: float, push_to_gateway: MagicMock, pushadd_to_gateway: MagicMock) -> None:- @shared_task(bind=True, base=PushGatewayTask, name="products.feature_flags.backend.tasks.my_complex_task")- def my_complex_task(self: Any) -> str:- return "done"-- my_complex_task()-- mock_push_context.assert_called_once_with("celery_my_complex_task")-- def test_pushgateway_failure_does_not_fail_task(self) -> None:- """- Verify that failures in the pushgateway (during metrics push) don't cause- the task itself to fail. The real pushed_metrics_registry catches push- exceptions internally, so we simulate that by having __exit__ return False- (normal exit, no exception suppression needed).- """- with patch("posthog.tasks.utils.pushed_metrics_registry") as mock:- mock_context = MagicMock()- mock_context.__enter__ = MagicMock(return_value=CollectorRegistry())- mock_context.__exit__ = MagicMock(return_value=False)- mock.return_value = mock_context-- @shared_task(bind=True, base=PushGatewayTask, name="test.task_with_push_failure")- def task_with_push_failure(self: Any) -> str:- return "completed"-- result = task_with_push_failure()-- assert result == "completed"-- def test_registry_accessible_during_execution(- self, mock_push_context: MagicMock, mock_registry: CollectorRegistry+ pushadd_to_gateway.side_effect = OSError("pushgateway unreachable")++ with pytest.raises(type(error)) as exc_info:+ sample_task(error)++ assert exc_info.value is error+ registry = _pushed_registry(pushadd_to_gateway)+ assert pushadd_to_gateway.call_args.kwargs["timeout"] == FAILURE_PUSH_TIMEOUT_SECONDS+ assert registry.get_sample_value("posthog_celery_sample_task_success") == 0+ assert registry.get_sample_value("posthog_celery_sample_task_duration_seconds") == run_duration+ assert registry.get_sample_value(TASK_GAUGE) is None+ push_to_gateway.assert_not_called()++ @pytest.mark.parametrize(+ "predicate",+ [Retry(), Ignore(), Reject("reason", requeue=False)],+ ids=["retry", "ignore", "reject"],+ )+ def test_task_predicate_is_reraised_and_pushes_nothing(+ self, predicate: Exception, push_to_gateway: MagicMock, pushadd_to_gateway: MagicMock) -> None:- registry_during_execution: list[CollectorRegistry | None] = []-- @shared_task(bind=True, base=PushGatewayTask, name="test.registry_access_task")- def registry_access_task(self: Any) -> str:- registry_during_execution.append(self.metrics_registry)- return "done"-- registry_access_task()+ with pytest.raises(type(predicate)) as exc_info:+ sample_task(predicate)- assert len(registry_during_execution) == 1- assert registry_during_execution[0] is mock_registry+ assert exc_info.value is predicate+ push_to_gateway.assert_not_called()+ pushadd_to_gateway.assert_not_called()diff --git a/posthog/tasks/utils.py b/posthog/tasks/utils.pyindex b08657ce7fe4..59e0bb52c8ca 100644--- a/posthog/tasks/utils.py+++ b/posthog/tasks/utils.py@@ -3,13 +3,18 @@ import threadingfrom typing import Anyfrom celery import Task+from celery.exceptions import TaskPredicatefrom prometheus_client import CollectorRegistry, Gauge# Re-exported for the many existing `from posthog.tasks.utils import CeleryQueue` consumers.# The enum lives in posthog.celery_queues so import-light modules can use it at decorator-eval# time without triggering the posthog.tasks package init (which imports every task module).from posthog.celery_queues import CeleryQueue as CeleryQueue-from posthog.metrics import pushed_metrics_registry+from posthog.metrics import push_registry, pushed_metrics_registry++# A run that hits its soft time limit makes the failure push after that limit.+# The push must end before the hard limit kills the worker. Otherwise Celery loses the task's own exception.+FAILURE_PUSH_TIMEOUT_SECONDS = 5class PushGatewayTask(Task):@@ -19,6 +24,10 @@ class PushGatewayTask(Task):Provides standard duration and success metrics automatically.Tasks can add custom metrics via self.metrics_registry.+ A run that returns replaces the job's group with the duration, success = 1, and the task's custom metrics.+ A run that raises an Exception pushes only the duration and success = 0. The custom metrics keep the values of the last run that returned.+ A run that raises a Celery TaskPredicate (Retry, Ignore, Reject) pushes nothing, because Celery does not count it as a failure.+Usage:@shared_task(bind=True, base=PushGatewayTask, ...)def my_task(self):@@ -39,29 +48,34 @@ class PushGatewayTask(Task):def __call__(self, *args: Any, **kwargs: Any) -> Any:task_name = self.name.split(".")[-1]+ job_name = f"celery_{task_name}"+ start_time = time.monotonic()- with pushed_metrics_registry(f"celery_{task_name}") as registry:- self._local.registry = registry-- duration_gauge = Gauge(- f"posthog_celery_{task_name}_duration_seconds",- f"Duration of {task_name}",- registry=registry,- )- success_gauge = Gauge(- f"posthog_celery_{task_name}_success",- f"Whether {task_name} succeeded (1) or failed (0)",- registry=registry,- )-- start_time = time.monotonic()- try:+ try:+ with pushed_metrics_registry(job_name) as registry:+ self._local.registry = registryresult = self.run(*args, **kwargs)- success_gauge.set(1)+ self._set_run_metrics(registry, task_name, time.monotonic() - start_time, success=True)return result- except Exception:- success_gauge.set(0)- raise- finally:- duration_gauge.set(time.monotonic() - start_time)- self._local.registry = None+ except TaskPredicate:+ raise+ except Exception:+ failure_registry = CollectorRegistry()+ self._set_run_metrics(failure_registry, task_name, time.monotonic() - start_time, success=False)+ push_registry(job_name, failure_registry, replace_group=False, timeout=FAILURE_PUSH_TIMEOUT_SECONDS)+ raise+ finally:+ self._local.registry = None++ @staticmethod+ def _set_run_metrics(registry: CollectorRegistry, task_name: str, duration: float, *, success: bool) -> None:+ Gauge(+ f"posthog_celery_{task_name}_duration_seconds",+ f"Duration of {task_name}",+ registry=registry,+ ).set(duration)+ Gauge(+ f"posthog_celery_{task_name}_success",+ f"Whether {task_name} succeeded (1) or failed (0)",+ registry=registry,+ ).set(1 if success else 0)diff --git a/posthog/test/test_metrics.py b/posthog/test/test_metrics.pyindex db5c7324f425..166b2dc64812 100644--- a/posthog/test/test_metrics.py+++ b/posthog/test/test_metrics.py@@ -1,8 +1,12 @@import threadingimport http.server+import pytest+from unittest.mock import patch+import prometheus_client.exposition as expofrom prometheus_client import CollectorRegistry, Gauge, push_to_gateway+from pytest_django import Settingsfrom posthog.metrics import _make_handler_no_proxy, pushed_metrics_registry@@ -84,3 +88,20 @@ class TestPushgatewayProxyPatch:body = received["body"].decode()assert "test_ctx_metric" in bodyassert "99.0" in body+++class TestPushedMetricsRegistry:+ def test_body_that_raises_propagates_and_pushes_nothing(self, settings: Settings) -> None:+ settings.PROM_PUSHGATEWAY_ADDRESS = "http://pushgateway.example.com:9091"++ with (+ patch("posthog.metrics.push_to_gateway") as put,+ patch("posthog.metrics.pushadd_to_gateway") as post,+ ):+ with pytest.raises(ValueError, match="boom"):+ with pushed_metrics_registry("partial_job") as registry:+ Gauge("partial_metric", "Set before the body raises", registry=registry).set(1)+ raise ValueError("boom")++ put.assert_not_called()+ post.assert_not_called()