:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(health): page team ids and continue as new in health check runs (#126047)
Rafael Audibert9ca492ce0ea8
diff --git a/posthog/temporal/health_checks/README.md b/posthog/temporal/health_checks/README.mdindex aadb825ea8a8..150b64eb3c0e 100644--- a/posthog/temporal/health_checks/README.md+++ b/posthog/temporal/health_checks/README.md@@ -154,6 +154,8 @@ On each check run, for every team in the batch:Teams are split into batches of `batch_size` IDs. Up to `max_concurrent` batches run concurrently, controlled by an `asyncio.Semaphore` in the Temporal workflow. Each batch activity has a retry policy (3 attempts, 30 s initial interval, exponential backoff up to 5 min).+The workflow reads team IDs one page at a time, in ID order, and continues as new between pages when its history grows. A small `batch_size` therefore makes a run longer, but it cannot make Temporal end the run early.+Set the `policy` class attribute on your `HealthCheck` subclass to choose one.## Configuration guide@@ -228,7 +230,8 @@ Default ClickHouse settings: `max_execution_time=30`, `max_threads=2`.## ClickHouse kill switchHealth checks are automatically skipped when the ClickHouse kill switch is active (LIGHT or FULL).-The `get_team_id_batches` activity returns an empty list, and the workflow completes with `total_teams: 0`.+The `get_team_id_page` activity returns a page marked `stopped_by_kill_switch`, also in the middle of a run.+The workflow then stops and sends no metrics, so the last-run timestamp still shows the last run that checked every team.## Admin UIdiff --git a/posthog/temporal/health_checks/__init__.py b/posthog/temporal/health_checks/__init__.pyindex 2150e6225abb..afa28dbee70e 100644--- a/posthog/temporal/health_checks/__init__.py+++ b/posthog/temporal/health_checks/__init__.py@@ -1,18 +1,20 @@from posthog.temporal.health_checks.activities import (get_team_id_batches,+ get_team_id_page,push_health_check_metrics_activity,run_health_check_batch,)from posthog.temporal.health_checks.workflows import HealthCheckWorkflowWORKFLOWS = [HealthCheckWorkflow]-ACTIVITIES = [get_team_id_batches, run_health_check_batch, push_health_check_metrics_activity]+ACTIVITIES = [get_team_id_batches, get_team_id_page, run_health_check_batch, push_health_check_metrics_activity]__all__ = ["WORKFLOWS","ACTIVITIES","HealthCheckWorkflow","get_team_id_batches",+ "get_team_id_page","run_health_check_batch","push_health_check_metrics_activity",]diff --git a/posthog/temporal/health_checks/activities.py b/posthog/temporal/health_checks/activities.pyindex edec3e3c12ea..af9dc418c312 100644--- a/posthog/temporal/health_checks/activities.py+++ b/posthog/temporal/health_checks/activities.py@@ -1,9 +1,10 @@import dataclassesfrom datetime import timedeltafrom itertools import batched+from typing import TYPE_CHECKINGfrom django.db import close_old_connections-from django.db.models import Exists, OuterRef+from django.db.models import Exists, OuterRef, QuerySetfrom django.utils import timezoneimport structlog@@ -14,18 +15,46 @@ from posthog.clickhouse.query_tagging import Feature, tag_queriesfrom posthog.models.organization import OrganizationMembershipfrom posthog.sync import database_sync_to_asyncfrom posthog.temporal.common.rollout import filter_ids_for_rollout-from posthog.temporal.health_checks.models import BatchResult, HealthCheckWorkflowInputs+from posthog.temporal.health_checks.models import BatchResult, HealthCheckWorkflowInputs, TeamIdPagefrom posthog.temporal.health_checks.observability import push_health_check_metricsfrom posthog.temporal.health_checks.processing import _process_batch_detectionfrom posthog.temporal.health_checks.registry import ensure_registry_loaded, get_detect_fn, get_product+if TYPE_CHECKING:+ from posthog.models.team import Team+logger = structlog.get_logger(__name__)+# A page holds this many batches at most, so one page adds a bounded number of events to the workflow history.+BATCHES_PER_TEAM_ID_PAGE = 200+# Keeps the page payload far below Temporal's 2 MiB limit whatever the batch size.+MAX_TEAM_IDS_PER_PAGE = 20_000-@database_sync_to_async-def _get_team_id_batches_sync(inputs: HealthCheckWorkflowInputs) -> list[list[int]]:++def team_id_page_size(batch_size: int) -> int:+ return max(batch_size, min(batch_size * BATCHES_PER_TEAM_ID_PAGE, MAX_TEAM_IDS_PER_PAGE))+++def _scheduled_teams(inputs: HealthCheckWorkflowInputs) -> "QuerySet[Team]":+ """Teams a scheduled run visits: every team, or only those with a member who logged in recently."""from posthog.models.team import Team+ qs = Team.objects.exclude(id=0)+ if inputs.active_since_days is not None and inputs.active_since_days > 0:+ cutoff = timezone.now() - timedelta(days=inputs.active_since_days)+ qs = qs.filter(+ Exists(+ OrganizationMembership.objects.filter(+ organization_id=OuterRef("organization_id"),+ user__last_login__gte=cutoff,+ )+ )+ )+ return qs+++@database_sync_to_async+def _get_team_id_batches_sync(inputs: HealthCheckWorkflowInputs) -> list[list[int]]:# Temporal activities run in a thread pool where DB connections can go stale# between executions. close_old_connections() ensures we get a fresh connection.close_old_connections()@@ -43,25 +72,8 @@ def _get_team_id_batches_sync(inputs: HealthCheckWorkflowInputs) -> list[list[inteam_ids = inputs.team_idslogger.info("processing configured teams", count=len(team_ids))else:- qs = Team.objects.exclude(id=0)- cutoff = None- if inputs.active_since_days is not None and inputs.active_since_days > 0:- cutoff = timezone.now() - timedelta(days=inputs.active_since_days)- qs = qs.filter(- Exists(- OrganizationMembership.objects.filter(- organization_id=OuterRef("organization_id"),- user__last_login__gte=cutoff,- )- )- )- team_ids = list(qs.values_list("id", flat=True))- logger.info(- "team query complete",- count=len(team_ids),- active_since_days=inputs.active_since_days,- cutoff=cutoff.isoformat() if cutoff else None,- )+ team_ids = list(_scheduled_teams(inputs).values_list("id", flat=True))+ logger.info("team query complete", count=len(team_ids), active_since_days=inputs.active_since_days)if inputs.rollout_percentage < 1.0:team_ids = filter_ids_for_rollout(team_ids, inputs.rollout_percentage)@@ -78,6 +90,56 @@ async def get_team_id_batches(inputs: HealthCheckWorkflowInputs) -> list[list[inreturn await _get_team_id_batches_sync(inputs)+def _next_team_ids(inputs: HealthCheckWorkflowInputs, after_team_id: int, page_size: int) -> list[int]:+ if inputs.team_ids:+ return sorted(team_id for team_id in set(inputs.team_ids) if team_id > after_team_id)[:page_size]++ teams = _scheduled_teams(inputs).filter(id__gt=after_team_id).order_by("id")+ return list(teams.values_list("id", flat=True)[:page_size])+++def build_team_id_page(inputs: HealthCheckWorkflowInputs, after_team_id: int) -> TeamIdPage:+ """The next scheduled teams after `after_team_id`, in id order. `next_after_team_id` is None on the last page."""+ kill_switch_level = get_kill_switch_level()+ if kill_switch_level != KillSwitchLevel.OFF:+ logger.info(+ "skipping health check due to clickhouse kill switch",+ kind=inputs.kind,+ kill_switch_level=kill_switch_level.value,+ )+ return TeamIdPage(team_ids=[], next_after_team_id=None, stopped_by_kill_switch=True)++ page_size = team_id_page_size(inputs.batch_size)+ team_ids = _next_team_ids(inputs, after_team_id, page_size)+ next_after_team_id = team_ids[-1] if len(team_ids) == page_size else None++ # The rollout keeps the same share of every page. The pages are stable while the team set is stable,+ # so the selected teams are stable across runs too.+ if inputs.rollout_percentage < 1.0:+ team_ids = filter_ids_for_rollout(team_ids, inputs.rollout_percentage)++ logger.info(+ "team id page",+ kind=inputs.kind,+ after_team_id=after_team_id,+ count=len(team_ids),+ next_after_team_id=next_after_team_id,+ )+ return TeamIdPage(team_ids=team_ids, next_after_team_id=next_after_team_id)+++async def get_team_id_page(inputs: HealthCheckWorkflowInputs, after_team_id: int) -> TeamIdPage:+ return await _get_team_id_page_sync(inputs, after_team_id)+++@database_sync_to_async+def _get_team_id_page_sync(inputs: HealthCheckWorkflowInputs, after_team_id: int) -> TeamIdPage:+ # See comment in _get_team_id_batches_sync for why this is needed.+ close_old_connections()+ return build_team_id_page(inputs, after_team_id)++@database_sync_to_asyncdef _run_health_check_batch_sync(team_ids: list[int],diff --git a/posthog/temporal/health_checks/models.py b/posthog/temporal/health_checks/models.pyindex 3f5aa0cdf8e9..49174bafdfde 100644--- a/posthog/temporal/health_checks/models.py+++ b/posthog/temporal/health_checks/models.py@@ -5,6 +5,7 @@ from collections.abc import Callablefrom dataclasses import dataclassfrom typing import TYPE_CHECKING, Any+from posthog.dataclasses import frozenfrom posthog.models.health_issue import HealthIssueif TYPE_CHECKING:@@ -60,7 +61,24 @@ BatchDetectFn = Callable[[list[int]], dict[int, list[HealthCheckResult]]]DEFAULT_ACTIVE_SINCE_DAYS: int = 90-@dataclass+@frozen+class HealthCheckRunProgress:+ """How far one scheduled run got. It crosses continue-as-new, so the run reports and judges every team."""++ after_team_id: int = 0+ totals: BatchResult = dataclasses.field(default_factory=BatchResult)+ batches: int = 0+ failed_batches: int = 0+++@frozen+class TeamIdPage:+ team_ids: list[int]+ next_after_team_id: int | None+ stopped_by_kill_switch: bool = False+++@dataclass(frozen=True)class HealthCheckWorkflowInputs:name: strkind: str@@ -72,6 +90,7 @@ class HealthCheckWorkflowInputs:dry_run: bool = Falseactive_since_days: int | None = DEFAULT_ACTIVE_SINCE_DAYSowner: str = ""+ progress: HealthCheckRunProgress | None = None@classmethoddef from_config(cls, config: HealthCheckRegistration) -> HealthCheckWorkflowInputs:diff --git a/posthog/temporal/health_checks/tests/test_workflows.py b/posthog/temporal/health_checks/tests/test_workflows.pynew file mode 100644index 000000000000..73e2ef866928--- /dev/null+++ b/posthog/temporal/health_checks/tests/test_workflows.py@@ -0,0 +1,140 @@+import uuid+import logging+import dataclasses+from datetime import timedelta++import pytest+from unittest.mock import patch++from django.utils import timezone++from parameterized import parameterized+from temporalio import activity, workflow+from temporalio.client import WorkflowExecutionStatus+from temporalio.testing import WorkflowEnvironment+from temporalio.worker import UnsandboxedWorkflowRunner, Worker++from posthog.models import Organization, Team, User+from posthog.temporal.health_checks import (+ activities as health_check_activities,+ workflows as health_check_workflows,+)+from posthog.temporal.health_checks.activities import build_team_id_page+from posthog.temporal.health_checks.models import BatchResult, HealthCheckWorkflowInputs, TeamIdPage+from posthog.temporal.health_checks.workflows import HealthCheckWorkflow++FIRST_PAGE = TeamIdPage(team_ids=[1, 2, 3], next_after_team_id=3)+PAGES = {+ 0: FIRST_PAGE,+ 3: TeamIdPage(team_ids=[4, 5], next_after_team_id=5),+ 5: TeamIdPage(team_ids=[6], next_after_team_id=None),+}+pages: dict[int, TeamIdPage] = {}+processed_team_ids: list[int] = []+pushed_totals: list[dict] = []++[email protected](name="get_team_id_page")+async def fake_get_team_id_page(inputs: HealthCheckWorkflowInputs, after_team_id: int) -> TeamIdPage:+ return pages[after_team_id]++[email protected](name="run_health_check_batch")+async def fake_run_health_check_batch(team_ids: list[int], kind: str, dry_run: bool) -> dict:+ processed_team_ids.extend(team_ids)+ return dataclasses.asdict(BatchResult(batch_size=len(team_ids), teams_healthy=len(team_ids)))++[email protected](name="push_health_check_metrics_activity")+async def fake_push_health_check_metrics(kind: str, totals_dict: dict, success: bool) -> None:+ pushed_totals.append(totals_dict)+++async def _run_workflow(served_pages: dict[int, TeamIdPage]) -> tuple[dict, WorkflowExecutionStatus | None]:+ logging.getLogger("temporalio.workflow").setLevel(logging.INFO)+ logging.getLogger("temporalio.activity").setLevel(logging.INFO)+ pages.clear()+ pages.update(served_pages)+ processed_team_ids.clear()+ pushed_totals.clear()+ task_queue = str(uuid.uuid4())++ async with await WorkflowEnvironment.start_time_skipping() as env:+ async with Worker(+ env.client,+ task_queue=task_queue,+ workflows=[HealthCheckWorkflow],+ activities=[fake_get_team_id_page, fake_run_health_check_batch, fake_push_health_check_metrics],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ handle = await env.client.start_workflow(+ HealthCheckWorkflow.run,+ HealthCheckWorkflowInputs(name="check", kind="check", batch_size=2, max_concurrent=2),+ id=str(uuid.uuid4()),+ task_queue=task_queue,+ execution_timeout=timedelta(minutes=5),+ )+ result = await handle.result()+ first_run = await env.client.get_workflow_handle(handle.id, run_id=handle.first_execution_run_id).describe()+ return result, first_run.status+++class TestHealthCheckWorkflow:+ @parameterized.expand(+ [+ ("single_run", False, 10_000, False),+ ("server_suggests_continue_as_new", True, 10_000, True),+ ("history_reaches_the_cap", False, 20, True),+ ]+ )+ @pytest.mark.asyncio+ async def test_every_team_runs_once_and_the_totals_cover_the_whole_run(+ self, _name, continue_as_new_suggested, max_history_events, expect_continued+ ):+ with (+ patch.object(workflow.Info, "is_continue_as_new_suggested", return_value=continue_as_new_suggested),+ patch.object(health_check_workflows, "MAX_HISTORY_EVENTS_PER_RUN", max_history_events),+ ):+ result, first_run_status = await _run_workflow(PAGES)++ assert sorted(processed_team_ids) == [1, 2, 3, 4, 5, 6]++ assert len(pushed_totals) == 1+ assert pushed_totals[0]["batch_size"] == 6+ assert pushed_totals[0]["teams_healthy"] == 6+ assert result["total_teams"] == 6+ assert result["batches"] == 4+ assert (first_run_status == WorkflowExecutionStatus.CONTINUED_AS_NEW) is expect_continued++ @pytest.mark.asyncio+ async def test_kill_switch_stops_the_run_without_reporting_it_complete(self):+ stopped = TeamIdPage(team_ids=[], next_after_team_id=None, stopped_by_kill_switch=True)++ result, _ = await _run_workflow({0: FIRST_PAGE, 3: stopped})++ assert sorted(processed_team_ids) == [1, 2, 3]+ assert pushed_totals == []+ assert result["stopped_by_kill_switch"] is True+++def test_team_id_pages_visit_each_recently_active_team_once(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(health_check_activities, "BATCHES_PER_TEAM_ID_PAGE", 2)+ active_org = Organization.objects.create(name="active")+ inactive_org = Organization.objects.create(name="inactive")+ for org, last_login in ((active_org, timezone.now()), (inactive_org, timezone.now() - timedelta(days=60))):+ user = User.objects.create_and_join(org, f"{org.name}@example.com", None)+ User.objects.filter(id=user.id).update(last_login=last_login)+ active_team_ids = [Team.objects.create(organization=active_org).id for _ in range(2)]+ Team.objects.create(organization=inactive_org)+ active_team_ids += [Team.objects.create(organization=active_org).id for _ in range(3)]+ inputs = HealthCheckWorkflowInputs(name="check", kind="check", batch_size=1, active_since_days=30)++ visited: list[int] = []+ after_team_id: int | None = 0+ while after_team_id is not None:+ page = build_team_id_page(inputs, after_team_id)+ visited += page.team_ids+ after_team_id = page.next_after_team_id++ assert visited == active_team_idsdiff --git a/posthog/temporal/health_checks/workflows.py b/posthog/temporal/health_checks/workflows.pyindex d63b381a284b..727154cc54f8 100644--- a/posthog/temporal/health_checks/workflows.py+++ b/posthog/temporal/health_checks/workflows.py@@ -11,11 +11,13 @@ from posthog.temporal.common.base import PostHogWorkflowwith temporalio.workflow.unsafe.imports_passed_through():from posthog.temporal.health_checks.activities import (get_team_id_batches,+ get_team_id_page,push_health_check_metrics_activity,run_health_check_batch,)from posthog.temporal.health_checks.models import (BatchResult,+ HealthCheckRunProgress,HealthCheckThresholdExceeded,HealthCheckWorkflowInputs,)@@ -27,15 +29,58 @@ ACTIVITY_RETRY_POLICY = RetryPolicy(maximum_interval=timedelta(minutes=5),)+PAGED_RUN_PATCH = "health-check-paged-run-2026-10"+# Temporal terminates a workflow at 51,200 history events, and a run that ends that way sends no metrics+# and raises nothing. A run continues as new well below the limit, even when the server never suggests it.+MAX_HISTORY_EVENTS_PER_RUN = 10_000+@temporalio.workflow.defn(name="health-check-workflow")class HealthCheckWorkflow(PostHogWorkflow):+ """Run one health check over every scheduled team.++ Team ids arrive one page at a time, and the run continues as new between pages when its history+ grows. So the history and every payload stay bounded whatever the team count and the batch size.+ `inputs.progress` carries the totals across, and only the last run reports them.+ """+@staticmethoddef parse_inputs(inputs: list[str]) -> HealthCheckWorkflowInputs:return HealthCheckWorkflowInputs(**json.loads(inputs[0]))@temporalio.workflow.runasync def run(self, inputs: HealthCheckWorkflowInputs) -> dict:+ if not temporalio.workflow.patched(PAGED_RUN_PATCH):+ return await self._run_unpaged(inputs)++ progress = inputs.progress or HealthCheckRunProgress()+ semaphore = asyncio.Semaphore(inputs.max_concurrent)+ while True:+ page = await temporalio.workflow.execute_activity(+ get_team_id_page,+ args=[inputs, progress.after_team_id],+ start_to_close_timeout=timedelta(minutes=5),+ retry_policy=ACTIVITY_RETRY_POLICY,+ )+ if page.stopped_by_kill_switch:+ # The run sends no metrics, so monitoring still shows the last run that checked every team.+ return {"kind": inputs.kind, "total_teams": progress.totals.batch_size, "stopped_by_kill_switch": True}+ batches = [+ page.team_ids[start : start + inputs.batch_size]+ for start in range(0, len(page.team_ids), inputs.batch_size)+ ]+ progress = await self._run_batches(inputs, batches, progress, semaphore)+ if page.next_after_team_id is None:+ break+ progress = dataclasses.replace(progress, after_team_id=page.next_after_team_id)+ info = temporalio.workflow.info()+ if info.is_continue_as_new_suggested() or info.get_current_history_length() >= MAX_HISTORY_EVENTS_PER_RUN:+ temporalio.workflow.continue_as_new(dataclasses.replace(inputs, progress=progress))++ return await self._finish(inputs, progress)++ async def _run_unpaged(self, inputs: HealthCheckWorkflowInputs) -> dict:+ """The run before PAGED_RUN_PATCH, kept so open runs replay. Delete it and `get_team_id_batches` after one deploy."""batches = await temporalio.workflow.execute_activity(get_team_id_batches,inputs,@@ -51,7 +96,16 @@ class HealthCheckWorkflow(PostHogWorkflow):}semaphore = asyncio.Semaphore(inputs.max_concurrent)+ progress = await self._run_batches(inputs, batches, HealthCheckRunProgress(), semaphore)+ return await self._finish(inputs, progress)+ async def _run_batches(+ self,+ inputs: HealthCheckWorkflowInputs,+ batches: list[list[int]],+ progress: HealthCheckRunProgress,+ semaphore: asyncio.Semaphore,+ ) -> HealthCheckRunProgress:async def _run_batch(batch: list[int]) -> dict:async with semaphore:return await temporalio.workflow.execute_activity(@@ -63,16 +117,24 @@ class HealthCheckWorkflow(PostHogWorkflow):results = await asyncio.gather(*[_run_batch(b) for b in batches], return_exceptions=True)- totals = BatchResult()+ totals = dataclasses.replace(progress.totals)failed_batches = 0for batch, r in zip(batches, results):if isinstance(r, BaseException):failed_batches += 1totals += BatchResult(batch_size=len(batch), teams_failed=len(batch))else:- batch_result = BatchResult(**r)- totals += batch_result+ totals += BatchResult(**r)++ return dataclasses.replace(+ progress,+ totals=totals,+ batches=progress.batches + len(batches),+ failed_batches=progress.failed_batches + failed_batches,+ )+ async def _finish(self, inputs: HealthCheckWorkflowInputs, progress: HealthCheckRunProgress) -> dict:+ totals = progress.totalsthreshold_exceeded = totals.batch_size > 0 and totals.not_processed_rate > inputs.not_processed_thresholdawait temporalio.workflow.execute_activity(push_health_check_metrics_activity,@@ -92,8 +154,8 @@ class HealthCheckWorkflow(PostHogWorkflow):return {"kind": inputs.kind,"total_teams": totals.batch_size,- "batches": len(batches),- "failed_batches": failed_batches,+ "batches": progress.batches,+ "failed_batches": progress.failed_batches,"issues_upserted": totals.issues_upserted,"issues_resolved": totals.issues_resolved,"teams_with_issues": totals.teams_with_issues,