:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(health-checks): fail the workflow on a threshold breach (#129540)
Rafael Audibertac6d339b453c
diff --git a/posthog/temporal/health_checks/models.py b/posthog/temporal/health_checks/models.pyindex b73ce3ccc756..57db804e1046 100644--- a/posthog/temporal/health_checks/models.py+++ b/posthog/temporal/health_checks/models.py@@ -6,6 +6,8 @@ from dataclasses import dataclassfrom datetime import timedeltafrom typing import TYPE_CHECKING, Any+from temporalio.exceptions import ApplicationError+from posthog.dataclasses import frozenfrom posthog.models.health_issue import HealthIssue@@ -110,8 +112,16 @@ class HealthCheckWorkflowInputs:)-class HealthCheckThresholdExceeded(Exception):- pass+class HealthCheckThresholdExceeded(ApplicationError):+ """Fails the run when too many teams were skipped or failed.++ The workflow raises it, so it must be an `ApplicationError`. Temporal treats any other exception raised in a+ workflow as a workflow task failure and retries the task until the run times out. The schedule skips every+ start while the run stays open, and each retry reports the same breach again.+ """++ def __init__(self, message: str) -> None:+ super().__init__(message, type="HealthCheckThresholdExceeded", non_retryable=True)class HealthCheckBatchSkipped(Exception):diff --git a/posthog/temporal/health_checks/tests/test_workflows.py b/posthog/temporal/health_checks/tests/test_workflows.pyindex 73e2ef866928..154269e08d37 100644--- a/posthog/temporal/health_checks/tests/test_workflows.py+++ b/posthog/temporal/health_checks/tests/test_workflows.py@@ -10,7 +10,8 @@ from django.utils import timezonefrom parameterized import parameterizedfrom temporalio import activity, workflow-from temporalio.client import WorkflowExecutionStatus+from temporalio.client import WorkflowExecutionStatus, WorkflowFailureError+from temporalio.exceptions import ApplicationErrorfrom temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import UnsandboxedWorkflowRunner, Worker@@ -45,12 +46,19 @@ async def fake_run_health_check_batch(team_ids: list[int], kind: str, dry_run: breturn dataclasses.asdict(BatchResult(batch_size=len(team_ids), teams_healthy=len(team_ids)))[email protected](name="run_health_check_batch")+async def fake_skip_every_batch(team_ids: list[int], kind: str, dry_run: bool) -> dict:+ return dataclasses.asdict(BatchResult(batch_size=len(team_ids), teams_skipped=len(team_ids)))++@activity.defn(name="push_health_check_metrics_activity")async def fake_push_health_check_metrics(kind: str, totals_dict: dict, success: bool) -> None:pushed_totals.append(totals_dict)-async def _run_workflow(served_pages: dict[int, TeamIdPage]) -> tuple[dict, WorkflowExecutionStatus | None]:+async def _run_workflow(+ served_pages: dict[int, TeamIdPage], run_batch=fake_run_health_check_batch+) -> tuple[dict, WorkflowExecutionStatus | None]:logging.getLogger("temporalio.workflow").setLevel(logging.INFO)logging.getLogger("temporalio.activity").setLevel(logging.INFO)pages.clear()@@ -64,7 +72,7 @@ async def _run_workflow(served_pages: dict[int, TeamIdPage]) -> tuple[dict, Workenv.client,task_queue=task_queue,workflows=[HealthCheckWorkflow],- activities=[fake_get_team_id_page, fake_run_health_check_batch, fake_push_health_check_metrics],+ activities=[fake_get_team_id_page, run_batch, fake_push_health_check_metrics],workflow_runner=UnsandboxedWorkflowRunner(),):handle = await env.client.start_workflow(@@ -116,6 +124,17 @@ class TestHealthCheckWorkflow:assert pushed_totals == []assert result["stopped_by_kill_switch"] is True+ @pytest.mark.asyncio+ async def test_threshold_breach_fails_the_run_instead_of_retrying_until_timeout(self):+ with pytest.raises(WorkflowFailureError) as exc_info:+ await _run_workflow(PAGES, run_batch=fake_skip_every_batch)++ cause = exc_info.value.cause+ assert isinstance(cause, ApplicationError)+ assert cause.type == "HealthCheckThresholdExceeded"+ assert "6/6 teams not processed" in cause.message+ assert len(pushed_totals) == 1+@pytest.mark.django_dbdef test_team_id_pages_visit_each_recently_active_team_once(monkeypatch: pytest.MonkeyPatch) -> None: