:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(health-checks): give every health check run an execution timeout (#127438)
Rafael Audibert6b64e88f2ea4
diff --git a/posthog/admin/admins/health_check_admin.py b/posthog/admin/admins/health_check_admin.pyindex 848c6dae6eea..ec59d248e6dd 100644--- a/posthog/admin/admins/health_check_admin.py+++ b/posthog/admin/admins/health_check_admin.py@@ -106,6 +106,7 @@ def health_check_trigger_view(request, kind: str):dataclasses.asdict(workflow_inputs),id=workflow_id,task_queue=settings.HEALTH_CHECK_TASK_QUEUE,+ execution_timeout=config.execution_timeout,))logger.info(diff --git a/posthog/temporal/health_checks/README.md b/posthog/temporal/health_checks/README.mdindex 6fb9d535882c..6a89fa208346 100644--- a/posthog/temporal/health_checks/README.md+++ b/posthog/temporal/health_checks/README.md@@ -91,6 +91,7 @@ class HealthCheck:schedule: str | None = Nonerollout_percentage: float = 1.0not_processed_threshold: float = 0.1+ execution_timeout: timedelta = timedelta(hours=12)dry_run: bool = Falseactive_since_days: int | None = DEFAULT_ACTIVE_SINCE_DAYS@@ -98,17 +99,18 @@ class HealthCheck:raise NotImplementedError```-| Attribute | Type | Default | Description |-| ------------------------- | ----------------------- | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------- |-| `name` | `str` | required | Unique name used in Temporal workflow and schedule naming |-| `kind` | `str` | required | Issue kind written to the `posthog_healthissue` table. Must be globally unique |-| `owner` | `JobOwners` | required | Team that owns this check (used for alert routing) |-| `policy` | `HealthExecutionPolicy` | `DEFAULT_EXECUTION_POLICY` | Controls batch size and concurrency (see [Execution policies](#execution-policies)) |-| `schedule` | `str \| None` | `None` | Cron expression (UTC). Omit for manual-only checks |-| `rollout_percentage` | `float` | `1.0` | Fraction of teams to include (0–1, e.g. 0.01 = 1%). Deterministic by team ID |-| `not_processed_threshold` | `float` | `0.1` | Fail the workflow if this fraction of teams are skipped or errored |-| `dry_run` | `bool` | `False` | Run detection but skip DB writes (upsert/resolve). Sets the default for scheduled runs; can be overridden per-run in the admin UI |-| `active_since_days` | `int \| None` | `90` | Only process teams whose org has a member with `User.last_login` within this many days. Set to `None` to process all teams |+| Attribute | Type | Default | Description |+| ------------------------- | ----------------------- | -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |+| `name` | `str` | required | Unique name used in Temporal workflow and schedule naming |+| `kind` | `str` | required | Issue kind written to the `posthog_healthissue` table. Must be globally unique |+| `owner` | `JobOwners` | required | Team that owns this check (used for alert routing) |+| `policy` | `HealthExecutionPolicy` | `DEFAULT_EXECUTION_POLICY` | Controls batch size and concurrency (see [Execution policies](#execution-policies)) |+| `schedule` | `str \| None` | `None` | Cron expression (UTC). Omit for manual-only checks |+| `rollout_percentage` | `float` | `1.0` | Fraction of teams to include (0–1, e.g. 0.01 = 1%). Deterministic by team ID |+| `not_processed_threshold` | `float` | `0.1` | Fail the workflow if this fraction of teams are skipped or errored |+| `execution_timeout` | `timedelta` | `12 hours` | Temporal closes the run after this time, including every continue-as-new run. A run stuck on a failed workflow task otherwise stays open, and the schedule skips every start until it closes |+| `dry_run` | `bool` | `False` | Run detection but skip DB writes (upsert/resolve). Sets the default for scheduled runs; can be overridden per-run in the admin UI |+| `active_since_days` | `int \| None` | `90` | Only process teams whose org has a member with `User.last_login` within this many days. Set to `None` to process all teams |## `HealthCheckResult`diff --git a/posthog/temporal/health_checks/framework.py b/posthog/temporal/health_checks/framework.pyindex 0d10bdb6a7ab..519e80b6119e 100644--- a/posthog/temporal/health_checks/framework.py+++ b/posthog/temporal/health_checks/framework.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport inspectfrom dataclasses import dataclass+from datetime import timedeltafrom typing import TYPE_CHECKING, Anyfrom django.conf import settings@@ -11,7 +12,11 @@ from posthog.job_owners import JobOwnersfrom posthog.models.health_issue import HealthIssuefrom posthog.scopes import APIScopeObjectfrom posthog.temporal.health_checks.detectors import DEFAULT_EXECUTION_POLICY, HealthExecutionPolicy-from posthog.temporal.health_checks.models import DEFAULT_ACTIVE_SINCE_DAYS, HealthCheckResult+from posthog.temporal.health_checks.models import (+ DEFAULT_ACTIVE_SINCE_DAYS,+ DEFAULT_EXECUTION_TIMEOUT,+ HealthCheckResult,+)from posthog.temporal.health_checks.registry import _DETECT_FNS, HEALTH_CHECKS, ensure_registry_loadedfrom products.signals.backend.facade.taxonomy import ReportPriority@@ -152,6 +157,7 @@ class HealthCheckRegistration:max_concurrent: introllout_percentage: floatnot_processed_threshold: float+ execution_timeout: timedeltadry_run: boolactive_since_days: int | Noneproduct: Product | None@@ -171,6 +177,11 @@ class HealthCheckRegistration:f"HealthCheckRegistration not_processed_threshold must be between 0 and 1, "f"got {self.not_processed_threshold}")+ # Temporal reads a zero timeout as no timeout, which would let a broken run block the schedule again.+ if self.execution_timeout <= timedelta(0):+ raise ValueError(+ f"HealthCheckRegistration execution_timeout must be positive, got {self.execution_timeout}"+ )# The denied ids are looked up under the resource, so a key alone would gate nobody.if self.access_controlled_object_key is not None and self.access_controlled_resource is None:raise ValueError(@@ -199,6 +210,7 @@ def _register_health_check(cls: type[HealthCheck]) -> None:max_concurrent=cls.policy.max_concurrent,rollout_percentage=cls.rollout_percentage,not_processed_threshold=cls.not_processed_threshold,+ execution_timeout=cls.execution_timeout,dry_run=cls.dry_run,active_since_days=cls.active_since_days,product=cls.product,@@ -221,6 +233,7 @@ class HealthCheck:schedule: str | None = Nonerollout_percentage: float = 1.0not_processed_threshold: float = 0.1+ execution_timeout: timedelta = DEFAULT_EXECUTION_TIMEOUTdry_run: bool = Falseactive_since_days: int | None = DEFAULT_ACTIVE_SINCE_DAYS# False stops the `$health_check_issue_firing` and `$health_check_issue_resolved` events fordiff --git a/posthog/temporal/health_checks/models.py b/posthog/temporal/health_checks/models.pyindex 1b8e78b2a952..b73ce3ccc756 100644--- a/posthog/temporal/health_checks/models.py+++ b/posthog/temporal/health_checks/models.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport dataclassesfrom collections.abc import Callablefrom dataclasses import dataclass+from datetime import timedeltafrom typing import TYPE_CHECKING, Anyfrom posthog.dataclasses import frozen@@ -59,6 +60,8 @@ class BatchResult:BatchDetectFn = Callable[[list[int]], dict[int, list[HealthCheckResult]]]DEFAULT_ACTIVE_SINCE_DAYS: int = 90+# Shorter than a day, so a broken run of a daily check closes before the next scheduled start.+DEFAULT_EXECUTION_TIMEOUT: timedelta = timedelta(hours=12)@frozendiff --git a/posthog/temporal/health_checks/schedule.py b/posthog/temporal/health_checks/schedule.pyindex c56541268a64..86801c8b5eae 100644--- a/posthog/temporal/health_checks/schedule.py+++ b/posthog/temporal/health_checks/schedule.py@@ -36,6 +36,7 @@ async def create_health_check_schedules(client: Client) -> None:dataclasses.asdict(workflow_inputs),id=f"health-check-{config.name}",task_queue=settings.HEALTH_CHECK_TASK_QUEUE,+ execution_timeout=config.execution_timeout,),# Checks look back a day or more, so a start up to 30 minutes after the cron time finds the same# issues. The jitter also stops checks that share a cron time from starting together.diff --git a/posthog/temporal/health_checks/tests/test_framework.py b/posthog/temporal/health_checks/tests/test_framework.pyindex af3d243ad932..589bd3719bac 100644--- a/posthog/temporal/health_checks/tests/test_framework.py+++ b/posthog/temporal/health_checks/tests/test_framework.py@@ -1,3 +1,4 @@+from datetime import timedeltafrom typing import Anyimport pytest@@ -26,6 +27,7 @@ def _registration(**overrides: Any) -> HealthCheckRegistration:"max_concurrent": 1,"rollout_percentage": 1.0,"not_processed_threshold": 0.1,+ "execution_timeout": timedelta(hours=12),"dry_run": False,"active_since_days": 90,"product": None,@@ -59,9 +61,10 @@ def test_accepts_fractional_values(field: str, value: float) -> None:("rollout_percentage", 50.0),("not_processed_threshold", -0.1),("not_processed_threshold", 2.0),+ ("execution_timeout", timedelta(0)),],)-def test_rejects_out_of_range_fractions(field: str, value: float) -> None:+def test_rejects_out_of_range_values(field: str, value: Any) -> None:with pytest.raises(ValueError, match=field):_registration(**{field: value})diff --git a/posthog/temporal/health_checks/tests/test_schedule.py b/posthog/temporal/health_checks/tests/test_schedule.pynew file mode 100644index 000000000000..f692b4a1d133--- /dev/null+++ b/posthog/temporal/health_checks/tests/test_schedule.py@@ -0,0 +1,32 @@+from unittest.mock import AsyncMock, MagicMock, patch++from asgiref.sync import async_to_sync+from temporalio.client import Schedule, ScheduleActionStartWorkflow++from posthog.temporal.health_checks.registry import HEALTH_CHECKS, ensure_registry_loaded+from posthog.temporal.health_checks.schedule import create_health_check_schedules++MODULE = "posthog.temporal.health_checks.schedule"+++def test_every_scheduled_check_starts_with_its_execution_timeout() -> None:+ ensure_registry_loaded()+ scheduled = {config.name: config.execution_timeout for config in HEALTH_CHECKS.values() if config.schedule}++ with (+ patch(f"{MODULE}.a_schedule_exists", AsyncMock(return_value=False)),+ patch(f"{MODULE}.a_create_schedule", AsyncMock()) as create_schedule,+ ):+ async_to_sync(create_health_check_schedules)(MagicMock())++ timeouts: dict[str, object] = {}+ for call in create_schedule.await_args_list:+ schedule_id: str = call.args[1]+ schedule: Schedule = call.args[2]+ assert isinstance(schedule.action, ScheduleActionStartWorkflow)+ timeouts[schedule_id.removeprefix("health-check-").removesuffix("-schedule")] = (+ schedule.action.execution_timeout+ )++ assert scheduled+ assert timeouts == scheduled