Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(health-checks): fail the workflow on a threshold breach (#129540)

Rafael Audibertac6d339b453c
diff --git a/posthog/temporal/health_checks/models.py b/posthog/temporal/health_checks/models.py
index b73ce3ccc756..57db804e1046 100644
--- a/posthog/temporal/health_checks/models.py
+++ b/posthog/temporal/health_checks/models.py
@@ -6,6 +6,8 @@ from dataclasses import dataclass
from datetime import timedelta
from typing import TYPE_CHECKING, Any
+from temporalio.exceptions import ApplicationError
+
from posthog.dataclasses import frozen
from posthog.models.health_issue import HealthIssue
@@ -110,8 +112,16 @@ class HealthCheckWorkflowInputs:
)
-class HealthCheckThresholdExceeded(Exception):
- pass
+class HealthCheckThresholdExceeded(ApplicationError):
+ """Fails the run when too many teams were skipped or failed.
+
+ The workflow raises it, so it must be an `ApplicationError`. Temporal treats any other exception raised in a
+ workflow as a workflow task failure and retries the task until the run times out. The schedule skips every
+ start while the run stays open, and each retry reports the same breach again.
+ """
+
+ def __init__(self, message: str) -> None:
+ super().__init__(message, type="HealthCheckThresholdExceeded", non_retryable=True)
class HealthCheckBatchSkipped(Exception):
diff --git a/posthog/temporal/health_checks/tests/test_workflows.py b/posthog/temporal/health_checks/tests/test_workflows.py
index 73e2ef866928..154269e08d37 100644
--- a/posthog/temporal/health_checks/tests/test_workflows.py
+++ b/posthog/temporal/health_checks/tests/test_workflows.py
@@ -10,7 +10,8 @@ from django.utils import timezone
from parameterized import parameterized
from temporalio import activity, workflow
-from temporalio.client import WorkflowExecutionStatus
+from temporalio.client import WorkflowExecutionStatus, WorkflowFailureError
+from temporalio.exceptions import ApplicationError
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
@@ -45,12 +46,19 @@ async def fake_run_health_check_batch(team_ids: list[int], kind: str, dry_run: b
return dataclasses.asdict(BatchResult(batch_size=len(team_ids), teams_healthy=len(team_ids)))
[email protected](name="run_health_check_batch")
+async def fake_skip_every_batch(team_ids: list[int], kind: str, dry_run: bool) -> dict:
+ return dataclasses.asdict(BatchResult(batch_size=len(team_ids), teams_skipped=len(team_ids)))
+
+
@activity.defn(name="push_health_check_metrics_activity")
async def fake_push_health_check_metrics(kind: str, totals_dict: dict, success: bool) -> None:
pushed_totals.append(totals_dict)
-async def _run_workflow(served_pages: dict[int, TeamIdPage]) -> tuple[dict, WorkflowExecutionStatus | None]:
+async def _run_workflow(
+ served_pages: dict[int, TeamIdPage], run_batch=fake_run_health_check_batch
+) -> tuple[dict, WorkflowExecutionStatus | None]:
logging.getLogger("temporalio.workflow").setLevel(logging.INFO)
logging.getLogger("temporalio.activity").setLevel(logging.INFO)
pages.clear()
@@ -64,7 +72,7 @@ async def _run_workflow(served_pages: dict[int, TeamIdPage]) -> tuple[dict, Work
env.client,
task_queue=task_queue,
workflows=[HealthCheckWorkflow],
- activities=[fake_get_team_id_page, fake_run_health_check_batch, fake_push_health_check_metrics],
+ activities=[fake_get_team_id_page, run_batch, fake_push_health_check_metrics],
workflow_runner=UnsandboxedWorkflowRunner(),
):
handle = await env.client.start_workflow(
@@ -116,6 +124,17 @@ class TestHealthCheckWorkflow:
assert pushed_totals == []
assert result["stopped_by_kill_switch"] is True
+ @pytest.mark.asyncio
+ async def test_threshold_breach_fails_the_run_instead_of_retrying_until_timeout(self):
+ with pytest.raises(WorkflowFailureError) as exc_info:
+ await _run_workflow(PAGES, run_batch=fake_skip_every_batch)
+
+ cause = exc_info.value.cause
+ assert isinstance(cause, ApplicationError)
+ assert cause.type == "HealthCheckThresholdExceeded"
+ assert "6/6 teams not processed" in cause.message
+ assert len(pushed_totals) == 1
+
@pytest.mark.django_db
def test_team_id_pages_visit_each_recently_active_team_once(monkeypatch: pytest.MonkeyPatch) -> None: