Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(health): page team ids and continue as new in health check runs (#126047)

Rafael Audibert9ca492ce0ea8
diff --git a/posthog/temporal/health_checks/README.md b/posthog/temporal/health_checks/README.md
index aadb825ea8a8..150b64eb3c0e 100644
--- a/posthog/temporal/health_checks/README.md
+++ b/posthog/temporal/health_checks/README.md
@@ -154,6 +154,8 @@ On each check run, for every team in the batch:
Teams are split into batches of `batch_size` IDs. Up to `max_concurrent` batches run concurrently, controlled by an `asyncio.Semaphore` in the Temporal workflow. Each batch activity has a retry policy (3 attempts, 30 s initial interval, exponential backoff up to 5 min).
+The workflow reads team IDs one page at a time, in ID order, and continues as new between pages when its history grows. A small `batch_size` therefore makes a run longer, but it cannot make Temporal end the run early.
+
Set the `policy` class attribute on your `HealthCheck` subclass to choose one.
## Configuration guide
@@ -228,7 +230,8 @@ Default ClickHouse settings: `max_execution_time=30`, `max_threads=2`.
## ClickHouse kill switch
Health checks are automatically skipped when the ClickHouse kill switch is active (LIGHT or FULL).
-The `get_team_id_batches` activity returns an empty list, and the workflow completes with `total_teams: 0`.
+The `get_team_id_page` activity returns a page marked `stopped_by_kill_switch`, also in the middle of a run.
+The workflow then stops and sends no metrics, so the last-run timestamp still shows the last run that checked every team.
## Admin UI
diff --git a/posthog/temporal/health_checks/__init__.py b/posthog/temporal/health_checks/__init__.py
index 2150e6225abb..afa28dbee70e 100644
--- a/posthog/temporal/health_checks/__init__.py
+++ b/posthog/temporal/health_checks/__init__.py
@@ -1,18 +1,20 @@
from posthog.temporal.health_checks.activities import (
get_team_id_batches,
+ get_team_id_page,
push_health_check_metrics_activity,
run_health_check_batch,
)
from posthog.temporal.health_checks.workflows import HealthCheckWorkflow
WORKFLOWS = [HealthCheckWorkflow]
-ACTIVITIES = [get_team_id_batches, run_health_check_batch, push_health_check_metrics_activity]
+ACTIVITIES = [get_team_id_batches, get_team_id_page, run_health_check_batch, push_health_check_metrics_activity]
__all__ = [
"WORKFLOWS",
"ACTIVITIES",
"HealthCheckWorkflow",
"get_team_id_batches",
+ "get_team_id_page",
"run_health_check_batch",
"push_health_check_metrics_activity",
]
diff --git a/posthog/temporal/health_checks/activities.py b/posthog/temporal/health_checks/activities.py
index edec3e3c12ea..af9dc418c312 100644
--- a/posthog/temporal/health_checks/activities.py
+++ b/posthog/temporal/health_checks/activities.py
@@ -1,9 +1,10 @@
import dataclasses
from datetime import timedelta
from itertools import batched
+from typing import TYPE_CHECKING
from django.db import close_old_connections
-from django.db.models import Exists, OuterRef
+from django.db.models import Exists, OuterRef, QuerySet
from django.utils import timezone
import structlog
@@ -14,18 +15,46 @@ from posthog.clickhouse.query_tagging import Feature, tag_queries
from posthog.models.organization import OrganizationMembership
from posthog.sync import database_sync_to_async
from posthog.temporal.common.rollout import filter_ids_for_rollout
-from posthog.temporal.health_checks.models import BatchResult, HealthCheckWorkflowInputs
+from posthog.temporal.health_checks.models import BatchResult, HealthCheckWorkflowInputs, TeamIdPage
from posthog.temporal.health_checks.observability import push_health_check_metrics
from posthog.temporal.health_checks.processing import _process_batch_detection
from posthog.temporal.health_checks.registry import ensure_registry_loaded, get_detect_fn, get_product
+if TYPE_CHECKING:
+ from posthog.models.team import Team
+
logger = structlog.get_logger(__name__)
+# A page holds this many batches at most, so one page adds a bounded number of events to the workflow history.
+BATCHES_PER_TEAM_ID_PAGE = 200
+# Keeps the page payload far below Temporal's 2 MiB limit whatever the batch size.
+MAX_TEAM_IDS_PER_PAGE = 20_000
-@database_sync_to_async
-def _get_team_id_batches_sync(inputs: HealthCheckWorkflowInputs) -> list[list[int]]:
+
+def team_id_page_size(batch_size: int) -> int:
+ return max(batch_size, min(batch_size * BATCHES_PER_TEAM_ID_PAGE, MAX_TEAM_IDS_PER_PAGE))
+
+
+def _scheduled_teams(inputs: HealthCheckWorkflowInputs) -> "QuerySet[Team]":
+ """Teams a scheduled run visits: every team, or only those with a member who logged in recently."""
from posthog.models.team import Team
+ qs = Team.objects.exclude(id=0)
+ if inputs.active_since_days is not None and inputs.active_since_days > 0:
+ cutoff = timezone.now() - timedelta(days=inputs.active_since_days)
+ qs = qs.filter(
+ Exists(
+ OrganizationMembership.objects.filter(
+ organization_id=OuterRef("organization_id"),
+ user__last_login__gte=cutoff,
+ )
+ )
+ )
+ return qs
+
+
+@database_sync_to_async
+def _get_team_id_batches_sync(inputs: HealthCheckWorkflowInputs) -> list[list[int]]:
# Temporal activities run in a thread pool where DB connections can go stale
# between executions. close_old_connections() ensures we get a fresh connection.
close_old_connections()
@@ -43,25 +72,8 @@ def _get_team_id_batches_sync(inputs: HealthCheckWorkflowInputs) -> list[list[in
team_ids = inputs.team_ids
logger.info("processing configured teams", count=len(team_ids))
else:
- qs = Team.objects.exclude(id=0)
- cutoff = None
- if inputs.active_since_days is not None and inputs.active_since_days > 0:
- cutoff = timezone.now() - timedelta(days=inputs.active_since_days)
- qs = qs.filter(
- Exists(
- OrganizationMembership.objects.filter(
- organization_id=OuterRef("organization_id"),
- user__last_login__gte=cutoff,
- )
- )
- )
- team_ids = list(qs.values_list("id", flat=True))
- logger.info(
- "team query complete",
- count=len(team_ids),
- active_since_days=inputs.active_since_days,
- cutoff=cutoff.isoformat() if cutoff else None,
- )
+ team_ids = list(_scheduled_teams(inputs).values_list("id", flat=True))
+ logger.info("team query complete", count=len(team_ids), active_since_days=inputs.active_since_days)
if inputs.rollout_percentage < 1.0:
team_ids = filter_ids_for_rollout(team_ids, inputs.rollout_percentage)
@@ -78,6 +90,56 @@ async def get_team_id_batches(inputs: HealthCheckWorkflowInputs) -> list[list[in
return await _get_team_id_batches_sync(inputs)
+def _next_team_ids(inputs: HealthCheckWorkflowInputs, after_team_id: int, page_size: int) -> list[int]:
+ if inputs.team_ids:
+ return sorted(team_id for team_id in set(inputs.team_ids) if team_id > after_team_id)[:page_size]
+
+ teams = _scheduled_teams(inputs).filter(id__gt=after_team_id).order_by("id")
+ return list(teams.values_list("id", flat=True)[:page_size])
+
+
+def build_team_id_page(inputs: HealthCheckWorkflowInputs, after_team_id: int) -> TeamIdPage:
+ """The next scheduled teams after `after_team_id`, in id order. `next_after_team_id` is None on the last page."""
+ kill_switch_level = get_kill_switch_level()
+ if kill_switch_level != KillSwitchLevel.OFF:
+ logger.info(
+ "skipping health check due to clickhouse kill switch",
+ kind=inputs.kind,
+ kill_switch_level=kill_switch_level.value,
+ )
+ return TeamIdPage(team_ids=[], next_after_team_id=None, stopped_by_kill_switch=True)
+
+ page_size = team_id_page_size(inputs.batch_size)
+ team_ids = _next_team_ids(inputs, after_team_id, page_size)
+ next_after_team_id = team_ids[-1] if len(team_ids) == page_size else None
+
+ # The rollout keeps the same share of every page. The pages are stable while the team set is stable,
+ # so the selected teams are stable across runs too.
+ if inputs.rollout_percentage < 1.0:
+ team_ids = filter_ids_for_rollout(team_ids, inputs.rollout_percentage)
+
+ logger.info(
+ "team id page",
+ kind=inputs.kind,
+ after_team_id=after_team_id,
+ count=len(team_ids),
+ next_after_team_id=next_after_team_id,
+ )
+ return TeamIdPage(team_ids=team_ids, next_after_team_id=next_after_team_id)
+
+
+async def get_team_id_page(inputs: HealthCheckWorkflowInputs, after_team_id: int) -> TeamIdPage:
+ return await _get_team_id_page_sync(inputs, after_team_id)
+
+
+@database_sync_to_async
+def _get_team_id_page_sync(inputs: HealthCheckWorkflowInputs, after_team_id: int) -> TeamIdPage:
+ # See comment in _get_team_id_batches_sync for why this is needed.
+ close_old_connections()
+ return build_team_id_page(inputs, after_team_id)
+
+
@database_sync_to_async
def _run_health_check_batch_sync(
team_ids: list[int],
diff --git a/posthog/temporal/health_checks/models.py b/posthog/temporal/health_checks/models.py
index 3f5aa0cdf8e9..49174bafdfde 100644
--- a/posthog/temporal/health_checks/models.py
+++ b/posthog/temporal/health_checks/models.py
@@ -5,6 +5,7 @@ from collections.abc import Callable
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
+from posthog.dataclasses import frozen
from posthog.models.health_issue import HealthIssue
if TYPE_CHECKING:
@@ -60,7 +61,24 @@ BatchDetectFn = Callable[[list[int]], dict[int, list[HealthCheckResult]]]
DEFAULT_ACTIVE_SINCE_DAYS: int = 90
-@dataclass
+@frozen
+class HealthCheckRunProgress:
+ """How far one scheduled run got. It crosses continue-as-new, so the run reports and judges every team."""
+
+ after_team_id: int = 0
+ totals: BatchResult = dataclasses.field(default_factory=BatchResult)
+ batches: int = 0
+ failed_batches: int = 0
+
+
+@frozen
+class TeamIdPage:
+ team_ids: list[int]
+ next_after_team_id: int | None
+ stopped_by_kill_switch: bool = False
+
+
+@dataclass(frozen=True)
class HealthCheckWorkflowInputs:
name: str
kind: str
@@ -72,6 +90,7 @@ class HealthCheckWorkflowInputs:
dry_run: bool = False
active_since_days: int | None = DEFAULT_ACTIVE_SINCE_DAYS
owner: str = ""
+ progress: HealthCheckRunProgress | None = None
@classmethod
def from_config(cls, config: HealthCheckRegistration) -> HealthCheckWorkflowInputs:
diff --git a/posthog/temporal/health_checks/tests/test_workflows.py b/posthog/temporal/health_checks/tests/test_workflows.py
new file mode 100644
index 000000000000..73e2ef866928
--- /dev/null
+++ b/posthog/temporal/health_checks/tests/test_workflows.py
@@ -0,0 +1,140 @@
+import uuid
+import logging
+import dataclasses
+from datetime import timedelta
+
+import pytest
+from unittest.mock import patch
+
+from django.utils import timezone
+
+from parameterized import parameterized
+from temporalio import activity, workflow
+from temporalio.client import WorkflowExecutionStatus
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import UnsandboxedWorkflowRunner, Worker
+
+from posthog.models import Organization, Team, User
+from posthog.temporal.health_checks import (
+ activities as health_check_activities,
+ workflows as health_check_workflows,
+)
+from posthog.temporal.health_checks.activities import build_team_id_page
+from posthog.temporal.health_checks.models import BatchResult, HealthCheckWorkflowInputs, TeamIdPage
+from posthog.temporal.health_checks.workflows import HealthCheckWorkflow
+
+FIRST_PAGE = TeamIdPage(team_ids=[1, 2, 3], next_after_team_id=3)
+PAGES = {
+ 0: FIRST_PAGE,
+ 3: TeamIdPage(team_ids=[4, 5], next_after_team_id=5),
+ 5: TeamIdPage(team_ids=[6], next_after_team_id=None),
+}
+pages: dict[int, TeamIdPage] = {}
+processed_team_ids: list[int] = []
+pushed_totals: list[dict] = []
+
+
[email protected](name="get_team_id_page")
+async def fake_get_team_id_page(inputs: HealthCheckWorkflowInputs, after_team_id: int) -> TeamIdPage:
+ return pages[after_team_id]
+
+
[email protected](name="run_health_check_batch")
+async def fake_run_health_check_batch(team_ids: list[int], kind: str, dry_run: bool) -> dict:
+ processed_team_ids.extend(team_ids)
+ return dataclasses.asdict(BatchResult(batch_size=len(team_ids), teams_healthy=len(team_ids)))
+
+
[email protected](name="push_health_check_metrics_activity")
+async def fake_push_health_check_metrics(kind: str, totals_dict: dict, success: bool) -> None:
+ pushed_totals.append(totals_dict)
+
+
+async def _run_workflow(served_pages: dict[int, TeamIdPage]) -> tuple[dict, WorkflowExecutionStatus | None]:
+ logging.getLogger("temporalio.workflow").setLevel(logging.INFO)
+ logging.getLogger("temporalio.activity").setLevel(logging.INFO)
+ pages.clear()
+ pages.update(served_pages)
+ processed_team_ids.clear()
+ pushed_totals.clear()
+ task_queue = str(uuid.uuid4())
+
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with Worker(
+ env.client,
+ task_queue=task_queue,
+ workflows=[HealthCheckWorkflow],
+ activities=[fake_get_team_id_page, fake_run_health_check_batch, fake_push_health_check_metrics],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ handle = await env.client.start_workflow(
+ HealthCheckWorkflow.run,
+ HealthCheckWorkflowInputs(name="check", kind="check", batch_size=2, max_concurrent=2),
+ id=str(uuid.uuid4()),
+ task_queue=task_queue,
+ execution_timeout=timedelta(minutes=5),
+ )
+ result = await handle.result()
+ first_run = await env.client.get_workflow_handle(handle.id, run_id=handle.first_execution_run_id).describe()
+ return result, first_run.status
+
+
+class TestHealthCheckWorkflow:
+ @parameterized.expand(
+ [
+ ("single_run", False, 10_000, False),
+ ("server_suggests_continue_as_new", True, 10_000, True),
+ ("history_reaches_the_cap", False, 20, True),
+ ]
+ )
+ @pytest.mark.asyncio
+ async def test_every_team_runs_once_and_the_totals_cover_the_whole_run(
+ self, _name, continue_as_new_suggested, max_history_events, expect_continued
+ ):
+ with (
+ patch.object(workflow.Info, "is_continue_as_new_suggested", return_value=continue_as_new_suggested),
+ patch.object(health_check_workflows, "MAX_HISTORY_EVENTS_PER_RUN", max_history_events),
+ ):
+ result, first_run_status = await _run_workflow(PAGES)
+
+ assert sorted(processed_team_ids) == [1, 2, 3, 4, 5, 6]
+
+ assert len(pushed_totals) == 1
+ assert pushed_totals[0]["batch_size"] == 6
+ assert pushed_totals[0]["teams_healthy"] == 6
+ assert result["total_teams"] == 6
+ assert result["batches"] == 4
+ assert (first_run_status == WorkflowExecutionStatus.CONTINUED_AS_NEW) is expect_continued
+
+ @pytest.mark.asyncio
+ async def test_kill_switch_stops_the_run_without_reporting_it_complete(self):
+ stopped = TeamIdPage(team_ids=[], next_after_team_id=None, stopped_by_kill_switch=True)
+
+ result, _ = await _run_workflow({0: FIRST_PAGE, 3: stopped})
+
+ assert sorted(processed_team_ids) == [1, 2, 3]
+ assert pushed_totals == []
+ assert result["stopped_by_kill_switch"] is True
+
+
+def test_team_id_pages_visit_each_recently_active_team_once(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(health_check_activities, "BATCHES_PER_TEAM_ID_PAGE", 2)
+ active_org = Organization.objects.create(name="active")
+ inactive_org = Organization.objects.create(name="inactive")
+ for org, last_login in ((active_org, timezone.now()), (inactive_org, timezone.now() - timedelta(days=60))):
+ user = User.objects.create_and_join(org, f"{org.name}@example.com", None)
+ User.objects.filter(id=user.id).update(last_login=last_login)
+ active_team_ids = [Team.objects.create(organization=active_org).id for _ in range(2)]
+ Team.objects.create(organization=inactive_org)
+ active_team_ids += [Team.objects.create(organization=active_org).id for _ in range(3)]
+ inputs = HealthCheckWorkflowInputs(name="check", kind="check", batch_size=1, active_since_days=30)
+
+ visited: list[int] = []
+ after_team_id: int | None = 0
+ while after_team_id is not None:
+ page = build_team_id_page(inputs, after_team_id)
+ visited += page.team_ids
+ after_team_id = page.next_after_team_id
+
+ assert visited == active_team_ids
diff --git a/posthog/temporal/health_checks/workflows.py b/posthog/temporal/health_checks/workflows.py
index d63b381a284b..727154cc54f8 100644
--- a/posthog/temporal/health_checks/workflows.py
+++ b/posthog/temporal/health_checks/workflows.py
@@ -11,11 +11,13 @@ from posthog.temporal.common.base import PostHogWorkflow
with temporalio.workflow.unsafe.imports_passed_through():
from posthog.temporal.health_checks.activities import (
get_team_id_batches,
+ get_team_id_page,
push_health_check_metrics_activity,
run_health_check_batch,
)
from posthog.temporal.health_checks.models import (
BatchResult,
+ HealthCheckRunProgress,
HealthCheckThresholdExceeded,
HealthCheckWorkflowInputs,
)
@@ -27,15 +29,58 @@ ACTIVITY_RETRY_POLICY = RetryPolicy(
maximum_interval=timedelta(minutes=5),
)
+PAGED_RUN_PATCH = "health-check-paged-run-2026-10"
+# Temporal terminates a workflow at 51,200 history events, and a run that ends that way sends no metrics
+# and raises nothing. A run continues as new well below the limit, even when the server never suggests it.
+MAX_HISTORY_EVENTS_PER_RUN = 10_000
+
@temporalio.workflow.defn(name="health-check-workflow")
class HealthCheckWorkflow(PostHogWorkflow):
+ """Run one health check over every scheduled team.
+
+ Team ids arrive one page at a time, and the run continues as new between pages when its history
+ grows. So the history and every payload stay bounded whatever the team count and the batch size.
+ `inputs.progress` carries the totals across, and only the last run reports them.
+ """
+
@staticmethod
def parse_inputs(inputs: list[str]) -> HealthCheckWorkflowInputs:
return HealthCheckWorkflowInputs(**json.loads(inputs[0]))
@temporalio.workflow.run
async def run(self, inputs: HealthCheckWorkflowInputs) -> dict:
+ if not temporalio.workflow.patched(PAGED_RUN_PATCH):
+ return await self._run_unpaged(inputs)
+
+ progress = inputs.progress or HealthCheckRunProgress()
+ semaphore = asyncio.Semaphore(inputs.max_concurrent)
+ while True:
+ page = await temporalio.workflow.execute_activity(
+ get_team_id_page,
+ args=[inputs, progress.after_team_id],
+ start_to_close_timeout=timedelta(minutes=5),
+ retry_policy=ACTIVITY_RETRY_POLICY,
+ )
+ if page.stopped_by_kill_switch:
+ # The run sends no metrics, so monitoring still shows the last run that checked every team.
+ return {"kind": inputs.kind, "total_teams": progress.totals.batch_size, "stopped_by_kill_switch": True}
+ batches = [
+ page.team_ids[start : start + inputs.batch_size]
+ for start in range(0, len(page.team_ids), inputs.batch_size)
+ ]
+ progress = await self._run_batches(inputs, batches, progress, semaphore)
+ if page.next_after_team_id is None:
+ break
+ progress = dataclasses.replace(progress, after_team_id=page.next_after_team_id)
+ info = temporalio.workflow.info()
+ if info.is_continue_as_new_suggested() or info.get_current_history_length() >= MAX_HISTORY_EVENTS_PER_RUN:
+ temporalio.workflow.continue_as_new(dataclasses.replace(inputs, progress=progress))
+
+ return await self._finish(inputs, progress)
+
+ async def _run_unpaged(self, inputs: HealthCheckWorkflowInputs) -> dict:
+ """The run before PAGED_RUN_PATCH, kept so open runs replay. Delete it and `get_team_id_batches` after one deploy."""
batches = await temporalio.workflow.execute_activity(
get_team_id_batches,
inputs,
@@ -51,7 +96,16 @@ class HealthCheckWorkflow(PostHogWorkflow):
}
semaphore = asyncio.Semaphore(inputs.max_concurrent)
+ progress = await self._run_batches(inputs, batches, HealthCheckRunProgress(), semaphore)
+ return await self._finish(inputs, progress)
+ async def _run_batches(
+ self,
+ inputs: HealthCheckWorkflowInputs,
+ batches: list[list[int]],
+ progress: HealthCheckRunProgress,
+ semaphore: asyncio.Semaphore,
+ ) -> HealthCheckRunProgress:
async def _run_batch(batch: list[int]) -> dict:
async with semaphore:
return await temporalio.workflow.execute_activity(
@@ -63,16 +117,24 @@ class HealthCheckWorkflow(PostHogWorkflow):
results = await asyncio.gather(*[_run_batch(b) for b in batches], return_exceptions=True)
- totals = BatchResult()
+ totals = dataclasses.replace(progress.totals)
failed_batches = 0
for batch, r in zip(batches, results):
if isinstance(r, BaseException):
failed_batches += 1
totals += BatchResult(batch_size=len(batch), teams_failed=len(batch))
else:
- batch_result = BatchResult(**r)
- totals += batch_result
+ totals += BatchResult(**r)
+
+ return dataclasses.replace(
+ progress,
+ totals=totals,
+ batches=progress.batches + len(batches),
+ failed_batches=progress.failed_batches + failed_batches,
+ )
+ async def _finish(self, inputs: HealthCheckWorkflowInputs, progress: HealthCheckRunProgress) -> dict:
+ totals = progress.totals
threshold_exceeded = totals.batch_size > 0 and totals.not_processed_rate > inputs.not_processed_threshold
await temporalio.workflow.execute_activity(
push_health_check_metrics_activity,
@@ -92,8 +154,8 @@ class HealthCheckWorkflow(PostHogWorkflow):
return {
"kind": inputs.kind,
"total_teams": totals.batch_size,
- "batches": len(batches),
- "failed_batches": failed_batches,
+ "batches": progress.batches,
+ "failed_batches": progress.failed_batches,
"issues_upserted": totals.issues_upserted,
"issues_resolved": totals.issues_resolved,
"teams_with_issues": totals.teams_with_issues,