Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(health-checks): give every health check run an execution timeout (#127438)

Rafael Audibert6b64e88f2ea4
diff --git a/posthog/admin/admins/health_check_admin.py b/posthog/admin/admins/health_check_admin.py
index 848c6dae6eea..ec59d248e6dd 100644
--- a/posthog/admin/admins/health_check_admin.py
+++ b/posthog/admin/admins/health_check_admin.py
@@ -106,6 +106,7 @@ def health_check_trigger_view(request, kind: str):
dataclasses.asdict(workflow_inputs),
id=workflow_id,
task_queue=settings.HEALTH_CHECK_TASK_QUEUE,
+ execution_timeout=config.execution_timeout,
)
)
logger.info(
diff --git a/posthog/temporal/health_checks/README.md b/posthog/temporal/health_checks/README.md
index 6fb9d535882c..6a89fa208346 100644
--- a/posthog/temporal/health_checks/README.md
+++ b/posthog/temporal/health_checks/README.md
@@ -91,6 +91,7 @@ class HealthCheck:
schedule: str | None = None
rollout_percentage: float = 1.0
not_processed_threshold: float = 0.1
+ execution_timeout: timedelta = timedelta(hours=12)
dry_run: bool = False
active_since_days: int | None = DEFAULT_ACTIVE_SINCE_DAYS
@@ -98,17 +99,18 @@ class HealthCheck:
raise NotImplementedError
```
-| Attribute | Type | Default | Description |
-| ------------------------- | ----------------------- | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------- |
-| `name` | `str` | required | Unique name used in Temporal workflow and schedule naming |
-| `kind` | `str` | required | Issue kind written to the `posthog_healthissue` table. Must be globally unique |
-| `owner` | `JobOwners` | required | Team that owns this check (used for alert routing) |
-| `policy` | `HealthExecutionPolicy` | `DEFAULT_EXECUTION_POLICY` | Controls batch size and concurrency (see [Execution policies](#execution-policies)) |
-| `schedule` | `str \| None` | `None` | Cron expression (UTC). Omit for manual-only checks |
-| `rollout_percentage` | `float` | `1.0` | Fraction of teams to include (0–1, e.g. 0.01 = 1%). Deterministic by team ID |
-| `not_processed_threshold` | `float` | `0.1` | Fail the workflow if this fraction of teams are skipped or errored |
-| `dry_run` | `bool` | `False` | Run detection but skip DB writes (upsert/resolve). Sets the default for scheduled runs; can be overridden per-run in the admin UI |
-| `active_since_days` | `int \| None` | `90` | Only process teams whose org has a member with `User.last_login` within this many days. Set to `None` to process all teams |
+| Attribute | Type | Default | Description |
+| ------------------------- | ----------------------- | -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `name` | `str` | required | Unique name used in Temporal workflow and schedule naming |
+| `kind` | `str` | required | Issue kind written to the `posthog_healthissue` table. Must be globally unique |
+| `owner` | `JobOwners` | required | Team that owns this check (used for alert routing) |
+| `policy` | `HealthExecutionPolicy` | `DEFAULT_EXECUTION_POLICY` | Controls batch size and concurrency (see [Execution policies](#execution-policies)) |
+| `schedule` | `str \| None` | `None` | Cron expression (UTC). Omit for manual-only checks |
+| `rollout_percentage` | `float` | `1.0` | Fraction of teams to include (0–1, e.g. 0.01 = 1%). Deterministic by team ID |
+| `not_processed_threshold` | `float` | `0.1` | Fail the workflow if this fraction of teams are skipped or errored |
+| `execution_timeout` | `timedelta` | `12 hours` | Temporal closes the run after this time, including every continue-as-new run. A run stuck on a failed workflow task otherwise stays open, and the schedule skips every start until it closes |
+| `dry_run` | `bool` | `False` | Run detection but skip DB writes (upsert/resolve). Sets the default for scheduled runs; can be overridden per-run in the admin UI |
+| `active_since_days` | `int \| None` | `90` | Only process teams whose org has a member with `User.last_login` within this many days. Set to `None` to process all teams |
## `HealthCheckResult`
diff --git a/posthog/temporal/health_checks/framework.py b/posthog/temporal/health_checks/framework.py
index 0d10bdb6a7ab..519e80b6119e 100644
--- a/posthog/temporal/health_checks/framework.py
+++ b/posthog/temporal/health_checks/framework.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import inspect
from dataclasses import dataclass
+from datetime import timedelta
from typing import TYPE_CHECKING, Any
from django.conf import settings
@@ -11,7 +12,11 @@ from posthog.job_owners import JobOwners
from posthog.models.health_issue import HealthIssue
from posthog.scopes import APIScopeObject
from posthog.temporal.health_checks.detectors import DEFAULT_EXECUTION_POLICY, HealthExecutionPolicy
-from posthog.temporal.health_checks.models import DEFAULT_ACTIVE_SINCE_DAYS, HealthCheckResult
+from posthog.temporal.health_checks.models import (
+ DEFAULT_ACTIVE_SINCE_DAYS,
+ DEFAULT_EXECUTION_TIMEOUT,
+ HealthCheckResult,
+)
from posthog.temporal.health_checks.registry import _DETECT_FNS, HEALTH_CHECKS, ensure_registry_loaded
from products.signals.backend.facade.taxonomy import ReportPriority
@@ -152,6 +157,7 @@ class HealthCheckRegistration:
max_concurrent: int
rollout_percentage: float
not_processed_threshold: float
+ execution_timeout: timedelta
dry_run: bool
active_since_days: int | None
product: Product | None
@@ -171,6 +177,11 @@ class HealthCheckRegistration:
f"HealthCheckRegistration not_processed_threshold must be between 0 and 1, "
f"got {self.not_processed_threshold}"
)
+ # Temporal reads a zero timeout as no timeout, which would let a broken run block the schedule again.
+ if self.execution_timeout <= timedelta(0):
+ raise ValueError(
+ f"HealthCheckRegistration execution_timeout must be positive, got {self.execution_timeout}"
+ )
# The denied ids are looked up under the resource, so a key alone would gate nobody.
if self.access_controlled_object_key is not None and self.access_controlled_resource is None:
raise ValueError(
@@ -199,6 +210,7 @@ def _register_health_check(cls: type[HealthCheck]) -> None:
max_concurrent=cls.policy.max_concurrent,
rollout_percentage=cls.rollout_percentage,
not_processed_threshold=cls.not_processed_threshold,
+ execution_timeout=cls.execution_timeout,
dry_run=cls.dry_run,
active_since_days=cls.active_since_days,
product=cls.product,
@@ -221,6 +233,7 @@ class HealthCheck:
schedule: str | None = None
rollout_percentage: float = 1.0
not_processed_threshold: float = 0.1
+ execution_timeout: timedelta = DEFAULT_EXECUTION_TIMEOUT
dry_run: bool = False
active_since_days: int | None = DEFAULT_ACTIVE_SINCE_DAYS
# False stops the `$health_check_issue_firing` and `$health_check_issue_resolved` events for
diff --git a/posthog/temporal/health_checks/models.py b/posthog/temporal/health_checks/models.py
index 1b8e78b2a952..b73ce3ccc756 100644
--- a/posthog/temporal/health_checks/models.py
+++ b/posthog/temporal/health_checks/models.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import dataclasses
from collections.abc import Callable
from dataclasses import dataclass
+from datetime import timedelta
from typing import TYPE_CHECKING, Any
from posthog.dataclasses import frozen
@@ -59,6 +60,8 @@ class BatchResult:
BatchDetectFn = Callable[[list[int]], dict[int, list[HealthCheckResult]]]
DEFAULT_ACTIVE_SINCE_DAYS: int = 90
+# Shorter than a day, so a broken run of a daily check closes before the next scheduled start.
+DEFAULT_EXECUTION_TIMEOUT: timedelta = timedelta(hours=12)
@frozen
diff --git a/posthog/temporal/health_checks/schedule.py b/posthog/temporal/health_checks/schedule.py
index c56541268a64..86801c8b5eae 100644
--- a/posthog/temporal/health_checks/schedule.py
+++ b/posthog/temporal/health_checks/schedule.py
@@ -36,6 +36,7 @@ async def create_health_check_schedules(client: Client) -> None:
dataclasses.asdict(workflow_inputs),
id=f"health-check-{config.name}",
task_queue=settings.HEALTH_CHECK_TASK_QUEUE,
+ execution_timeout=config.execution_timeout,
),
# Checks look back a day or more, so a start up to 30 minutes after the cron time finds the same
# issues. The jitter also stops checks that share a cron time from starting together.
diff --git a/posthog/temporal/health_checks/tests/test_framework.py b/posthog/temporal/health_checks/tests/test_framework.py
index af3d243ad932..589bd3719bac 100644
--- a/posthog/temporal/health_checks/tests/test_framework.py
+++ b/posthog/temporal/health_checks/tests/test_framework.py
@@ -1,3 +1,4 @@
+from datetime import timedelta
from typing import Any
import pytest
@@ -26,6 +27,7 @@ def _registration(**overrides: Any) -> HealthCheckRegistration:
"max_concurrent": 1,
"rollout_percentage": 1.0,
"not_processed_threshold": 0.1,
+ "execution_timeout": timedelta(hours=12),
"dry_run": False,
"active_since_days": 90,
"product": None,
@@ -59,9 +61,10 @@ def test_accepts_fractional_values(field: str, value: float) -> None:
("rollout_percentage", 50.0),
("not_processed_threshold", -0.1),
("not_processed_threshold", 2.0),
+ ("execution_timeout", timedelta(0)),
],
)
-def test_rejects_out_of_range_fractions(field: str, value: float) -> None:
+def test_rejects_out_of_range_values(field: str, value: Any) -> None:
with pytest.raises(ValueError, match=field):
_registration(**{field: value})
diff --git a/posthog/temporal/health_checks/tests/test_schedule.py b/posthog/temporal/health_checks/tests/test_schedule.py
new file mode 100644
index 000000000000..f692b4a1d133
--- /dev/null
+++ b/posthog/temporal/health_checks/tests/test_schedule.py
@@ -0,0 +1,32 @@
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from asgiref.sync import async_to_sync
+from temporalio.client import Schedule, ScheduleActionStartWorkflow
+
+from posthog.temporal.health_checks.registry import HEALTH_CHECKS, ensure_registry_loaded
+from posthog.temporal.health_checks.schedule import create_health_check_schedules
+
+MODULE = "posthog.temporal.health_checks.schedule"
+
+
+def test_every_scheduled_check_starts_with_its_execution_timeout() -> None:
+ ensure_registry_loaded()
+ scheduled = {config.name: config.execution_timeout for config in HEALTH_CHECKS.values() if config.schedule}
+
+ with (
+ patch(f"{MODULE}.a_schedule_exists", AsyncMock(return_value=False)),
+ patch(f"{MODULE}.a_create_schedule", AsyncMock()) as create_schedule,
+ ):
+ async_to_sync(create_health_check_schedules)(MagicMock())
+
+ timeouts: dict[str, object] = {}
+ for call in create_schedule.await_args_list:
+ schedule_id: str = call.args[1]
+ schedule: Schedule = call.args[2]
+ assert isinstance(schedule.action, ScheduleActionStartWorkflow)
+ timeouts[schedule_id.removeprefix("health-check-").removesuffix("-schedule")] = (
+ schedule.action.execution_timeout
+ )
+
+ assert scheduled
+ assert timeouts == scheduled