Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(celery): push duration and success when a PushGatewayTask run fails (#121699)

Phil Haack8f7b71ffe02a
diff --git a/posthog/metrics.py b/posthog/metrics.py
index bd2f87a866d1..c19096a1e584 100644
--- a/posthog/metrics.py
+++ b/posthog/metrics.py
@@ -1,4 +1,5 @@
# Shared metrics and labels for prometheus metrics
+from collections.abc import Iterator
from contextlib import contextmanager
from django.conf import settings
@@ -9,7 +10,7 @@ import structlog
# The pushgateway is an internal service — the outbound proxy would reject it.
# ProxyHandler({}) tells urllib to ignore proxy env vars.
import prometheus_client.exposition as _expo
-from prometheus_client import CollectorRegistry, Counter, push_to_gateway
+from prometheus_client import CollectorRegistry, Counter, push_to_gateway, pushadd_to_gateway
from posthog.exceptions_capture import capture_exception
@@ -59,8 +60,31 @@ def _make_handler_no_proxy(url, method, timeout, headers, data, base_handler):
_expo._make_handler = _make_handler_no_proxy # ty: ignore[invalid-assignment]
+def push_registry(job_name: str, registry: CollectorRegistry, *, replace_group: bool, timeout: float = 30) -> None:
+ """
+ Push the registry to the PushGateway under job_name. It logs and captures a failed push instead of raising it.
+
+ With replace_group=True the push is a PUT, which deletes every metric in the job's group that the registry does not carry.
+ With replace_group=False the push is a POST, which replaces only the metrics the registry carries.
+ The timeout applies to each socket operation of the push. The connect and the response can each take that long.
+ """
+ try:
+ if settings.PROM_PUSHGATEWAY_ADDRESS:
+ push = push_to_gateway if replace_group else pushadd_to_gateway
+ push(settings.PROM_PUSHGATEWAY_ADDRESS, job=job_name, registry=registry, timeout=timeout)
+ except Exception as err:
+ logger.exception(
+ "push_to_gateway",
+ target=settings.PROM_PUSHGATEWAY_ADDRESS,
+ job=job_name,
+ replace_group=replace_group,
+ exception=err,
+ )
+ capture_exception(err)
+
+
@contextmanager
-def pushed_metrics_registry(job_name: str):
+def pushed_metrics_registry(job_name: str) -> Iterator[CollectorRegistry]:
"""
Return a temporary Prometheus registry that will be pushed to the
PushGateway when the context closes.
@@ -68,15 +92,13 @@ def pushed_metrics_registry(job_name: str):
Parameter job_name: a unique job name to use, all metrics previously
pushed with that name will be deleted.
+ If the body raises, the context pushes nothing. The group keeps the metrics of the last push.
+ PushGatewayTask and alerts on a stale push_time_seconds depend on this.
+
NOTE: only use to expose gauges, for use cases where one value per
region makes sense (e.g. instance metrics computed by celery jobs).
"""
registry = CollectorRegistry()
yield registry
- try:
- if settings.PROM_PUSHGATEWAY_ADDRESS:
- push_to_gateway(settings.PROM_PUSHGATEWAY_ADDRESS, job=job_name, registry=registry)
- except Exception as err:
- logger.exception("push_to_gateway", target=settings.PROM_PUSHGATEWAY_ADDRESS, exception=err)
- capture_exception(err)
+ push_registry(job_name, registry, replace_group=True)
diff --git a/posthog/tasks/test/test_pushgateway_task.py b/posthog/tasks/test/test_pushgateway_task.py
index 3cb5833f5e8d..0fe7780b88ca 100644
--- a/posthog/tasks/test/test_pushgateway_task.py
+++ b/posthog/tasks/test/test_pushgateway_task.py
@@ -1,138 +1,103 @@
from collections.abc import Generator
-from typing import Any
import pytest
from unittest.mock import MagicMock, patch
from celery import shared_task
+from celery.exceptions import Ignore, Reject, Retry, SoftTimeLimitExceeded
from prometheus_client import CollectorRegistry, Gauge
+from pytest_django import Settings
-from posthog.tasks.utils import PushGatewayTask
+from posthog.tasks.utils import FAILURE_PUSH_TIMEOUT_SECONDS, PushGatewayTask
+TASK_GAUGE = "sample_task_rows_processed"
-class TestPushGatewayTask:
- @pytest.fixture
- def mock_registry(self) -> CollectorRegistry:
- return CollectorRegistry()
-
- @pytest.fixture
- def mock_push_context(self, mock_registry: CollectorRegistry) -> Generator[MagicMock]:
- with patch("posthog.tasks.utils.pushed_metrics_registry") as mock:
- mock_context = MagicMock()
- mock_context.__enter__ = MagicMock(return_value=mock_registry)
- mock_context.__exit__ = MagicMock(return_value=False)
- mock.return_value = mock_context
- yield mock
-
- def test_success_metrics_set_on_successful_task(
- self, mock_push_context: MagicMock, mock_registry: CollectorRegistry
- ) -> None:
- @shared_task(bind=True, base=PushGatewayTask, name="test.successful_task")
- def successful_task(self: Any) -> str:
- return "done"
- result = successful_task()
+# Celery keeps the first task registered under a name. A second definition with the same name runs the first body.
+# The cases pass the error to one task as an argument instead of defining a task for each case.
+@shared_task(bind=True, base=PushGatewayTask, name="test.pushgateway.sample_task")
+def sample_task(self: PushGatewayTask, error: Exception | None = None) -> str:
+ Gauge(TASK_GAUGE, "Rows the task processed", registry=self.metrics_registry).set(42)
+ if error is not None:
+ raise error
+ return "done"
- assert result == "done"
- success_sample = mock_registry.get_sample_value("posthog_celery_successful_task_success")
- assert success_sample == 1
- duration_sample = mock_registry.get_sample_value("posthog_celery_successful_task_duration_seconds")
- assert duration_sample is not None
- assert duration_sample >= 0
-
- def test_failure_metrics_set_on_failed_task(
- self, mock_push_context: MagicMock, mock_registry: CollectorRegistry
- ) -> None:
- @shared_task(bind=True, base=PushGatewayTask, name="test.failing_task")
- def failing_task(self: Any) -> None:
- raise ValueError("test error")
- with pytest.raises(ValueError, match="test error"):
- failing_task()
+def _pushed_registry(push: MagicMock) -> CollectorRegistry:
+ push.assert_called_once()
+ assert push.call_args.kwargs["job"] == "celery_sample_task"
+ registry = push.call_args.kwargs["registry"]
+ assert isinstance(registry, CollectorRegistry)
+ return registry
- success_sample = mock_registry.get_sample_value("posthog_celery_failing_task_success")
- assert success_sample == 0
- duration_sample = mock_registry.get_sample_value("posthog_celery_failing_task_duration_seconds")
- assert duration_sample is not None
- assert duration_sample >= 0
- def test_custom_metrics_can_be_added(self, mock_push_context: MagicMock, mock_registry: CollectorRegistry) -> None:
- @shared_task(bind=True, base=PushGatewayTask, name="test.task_with_custom_metric")
- def task_with_custom_metric(self: Any) -> str:
- custom = Gauge("custom_metric", "Custom", registry=self.metrics_registry)
- custom.set(42)
- return "done"
-
- task_with_custom_metric()
+class TestPushGatewayTask:
+ @pytest.fixture(autouse=True)
+ def pushgateway_address(self, settings: Settings) -> None:
+ settings.PROM_PUSHGATEWAY_ADDRESS = "http://pushgateway.example.com:9091"
- custom_sample = mock_registry.get_sample_value("custom_metric")
- assert custom_sample == 42
+ @pytest.fixture
+ def push_to_gateway(self) -> Generator[MagicMock]:
+ with patch("posthog.metrics.push_to_gateway") as mock:
+ yield mock
- def test_exception_is_reraised(self, mock_push_context: MagicMock, mock_registry: CollectorRegistry) -> None:
- @shared_task(bind=True, base=PushGatewayTask, name="test.exception_task")
- def exception_task(self: Any) -> None:
- raise RuntimeError("specific error")
+ @pytest.fixture
+ def pushadd_to_gateway(self) -> Generator[MagicMock]:
+ with patch("posthog.metrics.pushadd_to_gateway") as mock:
+ yield mock
- with pytest.raises(RuntimeError, match="specific error"):
- exception_task()
+ @pytest.fixture
+ def run_duration(self) -> Generator[float]:
+ with patch("posthog.tasks.utils.time") as mock_time:
+ mock_time.monotonic.side_effect = [100.0, 107.5]
+ yield 7.5
- def test_duration_is_recorded_even_on_failure(
- self, mock_push_context: MagicMock, mock_registry: CollectorRegistry
+ def test_completed_run_puts_its_metrics_and_returns_despite_a_push_error(
+ self, run_duration: float, push_to_gateway: MagicMock, pushadd_to_gateway: MagicMock
) -> None:
- @shared_task(bind=True, base=PushGatewayTask, name="test.duration_on_failure_task")
- def duration_on_failure_task(self: Any) -> None:
- raise ValueError("failure")
+ push_to_gateway.side_effect = OSError("pushgateway unreachable")
- with pytest.raises(ValueError):
- duration_on_failure_task()
+ result = sample_task()
- duration_sample = mock_registry.get_sample_value("posthog_celery_duration_on_failure_task_duration_seconds")
- assert duration_sample is not None
- assert duration_sample >= 0
-
- def test_task_name_extracted_correctly(
- self, mock_push_context: MagicMock, mock_registry: CollectorRegistry
+ assert result == "done"
+ registry = _pushed_registry(push_to_gateway)
+ assert registry.get_sample_value("posthog_celery_sample_task_success") == 1
+ assert registry.get_sample_value("posthog_celery_sample_task_duration_seconds") == run_duration
+ assert registry.get_sample_value(TASK_GAUGE) == 42
+ pushadd_to_gateway.assert_not_called()
+
+ @pytest.mark.parametrize(
+ "error",
+ [ValueError("boom"), SoftTimeLimitExceeded()],
+ ids=["value_error", "soft_time_limit"],
+ )
+ def test_failed_run_posts_only_failure_and_duration_and_reraises_despite_a_push_error(
+ self, error: Exception, run_duration: float, push_to_gateway: MagicMock, pushadd_to_gateway: MagicMock
) -> None:
- @shared_task(bind=True, base=PushGatewayTask, name="products.feature_flags.backend.tasks.my_complex_task")
- def my_complex_task(self: Any) -> str:
- return "done"
-
- my_complex_task()
-
- mock_push_context.assert_called_once_with("celery_my_complex_task")
-
- def test_pushgateway_failure_does_not_fail_task(self) -> None:
- """
- Verify that failures in the pushgateway (during metrics push) don't cause
- the task itself to fail. The real pushed_metrics_registry catches push
- exceptions internally, so we simulate that by having __exit__ return False
- (normal exit, no exception suppression needed).
- """
- with patch("posthog.tasks.utils.pushed_metrics_registry") as mock:
- mock_context = MagicMock()
- mock_context.__enter__ = MagicMock(return_value=CollectorRegistry())
- mock_context.__exit__ = MagicMock(return_value=False)
- mock.return_value = mock_context
-
- @shared_task(bind=True, base=PushGatewayTask, name="test.task_with_push_failure")
- def task_with_push_failure(self: Any) -> str:
- return "completed"
-
- result = task_with_push_failure()
-
- assert result == "completed"
-
- def test_registry_accessible_during_execution(
- self, mock_push_context: MagicMock, mock_registry: CollectorRegistry
+ pushadd_to_gateway.side_effect = OSError("pushgateway unreachable")
+
+ with pytest.raises(type(error)) as exc_info:
+ sample_task(error)
+
+ assert exc_info.value is error
+ registry = _pushed_registry(pushadd_to_gateway)
+ assert pushadd_to_gateway.call_args.kwargs["timeout"] == FAILURE_PUSH_TIMEOUT_SECONDS
+ assert registry.get_sample_value("posthog_celery_sample_task_success") == 0
+ assert registry.get_sample_value("posthog_celery_sample_task_duration_seconds") == run_duration
+ assert registry.get_sample_value(TASK_GAUGE) is None
+ push_to_gateway.assert_not_called()
+
+ @pytest.mark.parametrize(
+ "predicate",
+ [Retry(), Ignore(), Reject("reason", requeue=False)],
+ ids=["retry", "ignore", "reject"],
+ )
+ def test_task_predicate_is_reraised_and_pushes_nothing(
+ self, predicate: Exception, push_to_gateway: MagicMock, pushadd_to_gateway: MagicMock
) -> None:
- registry_during_execution: list[CollectorRegistry | None] = []
-
- @shared_task(bind=True, base=PushGatewayTask, name="test.registry_access_task")
- def registry_access_task(self: Any) -> str:
- registry_during_execution.append(self.metrics_registry)
- return "done"
-
- registry_access_task()
+ with pytest.raises(type(predicate)) as exc_info:
+ sample_task(predicate)
- assert len(registry_during_execution) == 1
- assert registry_during_execution[0] is mock_registry
+ assert exc_info.value is predicate
+ push_to_gateway.assert_not_called()
+ pushadd_to_gateway.assert_not_called()
diff --git a/posthog/tasks/utils.py b/posthog/tasks/utils.py
index b08657ce7fe4..59e0bb52c8ca 100644
--- a/posthog/tasks/utils.py
+++ b/posthog/tasks/utils.py
@@ -3,13 +3,18 @@ import threading
from typing import Any
from celery import Task
+from celery.exceptions import TaskPredicate
from prometheus_client import CollectorRegistry, Gauge
# Re-exported for the many existing `from posthog.tasks.utils import CeleryQueue` consumers.
# The enum lives in posthog.celery_queues so import-light modules can use it at decorator-eval
# time without triggering the posthog.tasks package init (which imports every task module).
from posthog.celery_queues import CeleryQueue as CeleryQueue
-from posthog.metrics import pushed_metrics_registry
+from posthog.metrics import push_registry, pushed_metrics_registry
+
+# A run that hits its soft time limit makes the failure push after that limit.
+# The push must end before the hard limit kills the worker. Otherwise Celery loses the task's own exception.
+FAILURE_PUSH_TIMEOUT_SECONDS = 5
class PushGatewayTask(Task):
@@ -19,6 +24,10 @@ class PushGatewayTask(Task):
Provides standard duration and success metrics automatically.
Tasks can add custom metrics via self.metrics_registry.
+ A run that returns replaces the job's group with the duration, success = 1, and the task's custom metrics.
+ A run that raises an Exception pushes only the duration and success = 0. The custom metrics keep the values of the last run that returned.
+ A run that raises a Celery TaskPredicate (Retry, Ignore, Reject) pushes nothing, because Celery does not count it as a failure.
+
Usage:
@shared_task(bind=True, base=PushGatewayTask, ...)
def my_task(self):
@@ -39,29 +48,34 @@ class PushGatewayTask(Task):
def __call__(self, *args: Any, **kwargs: Any) -> Any:
task_name = self.name.split(".")[-1]
+ job_name = f"celery_{task_name}"
+ start_time = time.monotonic()
- with pushed_metrics_registry(f"celery_{task_name}") as registry:
- self._local.registry = registry
-
- duration_gauge = Gauge(
- f"posthog_celery_{task_name}_duration_seconds",
- f"Duration of {task_name}",
- registry=registry,
- )
- success_gauge = Gauge(
- f"posthog_celery_{task_name}_success",
- f"Whether {task_name} succeeded (1) or failed (0)",
- registry=registry,
- )
-
- start_time = time.monotonic()
- try:
+ try:
+ with pushed_metrics_registry(job_name) as registry:
+ self._local.registry = registry
result = self.run(*args, **kwargs)
- success_gauge.set(1)
+ self._set_run_metrics(registry, task_name, time.monotonic() - start_time, success=True)
return result
- except Exception:
- success_gauge.set(0)
- raise
- finally:
- duration_gauge.set(time.monotonic() - start_time)
- self._local.registry = None
+ except TaskPredicate:
+ raise
+ except Exception:
+ failure_registry = CollectorRegistry()
+ self._set_run_metrics(failure_registry, task_name, time.monotonic() - start_time, success=False)
+ push_registry(job_name, failure_registry, replace_group=False, timeout=FAILURE_PUSH_TIMEOUT_SECONDS)
+ raise
+ finally:
+ self._local.registry = None
+
+ @staticmethod
+ def _set_run_metrics(registry: CollectorRegistry, task_name: str, duration: float, *, success: bool) -> None:
+ Gauge(
+ f"posthog_celery_{task_name}_duration_seconds",
+ f"Duration of {task_name}",
+ registry=registry,
+ ).set(duration)
+ Gauge(
+ f"posthog_celery_{task_name}_success",
+ f"Whether {task_name} succeeded (1) or failed (0)",
+ registry=registry,
+ ).set(1 if success else 0)
diff --git a/posthog/test/test_metrics.py b/posthog/test/test_metrics.py
index db5c7324f425..166b2dc64812 100644
--- a/posthog/test/test_metrics.py
+++ b/posthog/test/test_metrics.py
@@ -1,8 +1,12 @@
import threading
import http.server
+import pytest
+from unittest.mock import patch
+
import prometheus_client.exposition as expo
from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
+from pytest_django import Settings
from posthog.metrics import _make_handler_no_proxy, pushed_metrics_registry
@@ -84,3 +88,20 @@ class TestPushgatewayProxyPatch:
body = received["body"].decode()
assert "test_ctx_metric" in body
assert "99.0" in body
+
+
+class TestPushedMetricsRegistry:
+ def test_body_that_raises_propagates_and_pushes_nothing(self, settings: Settings) -> None:
+ settings.PROM_PUSHGATEWAY_ADDRESS = "http://pushgateway.example.com:9091"
+
+ with (
+ patch("posthog.metrics.push_to_gateway") as put,
+ patch("posthog.metrics.pushadd_to_gateway") as post,
+ ):
+ with pytest.raises(ValueError, match="boom"):
+ with pushed_metrics_registry("partial_job") as registry:
+ Gauge("partial_metric", "Set before the body raises", registry=registry).set(1)
+ raise ValueError("boom")
+
+ put.assert_not_called()
+ post.assert_not_called()