Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(growth): fill a daily sdk versions rollup in dagster (#126050)

Rafael Audibert6ca51906d16d
diff --git a/.github/workflows/ci-dagster.yml b/.github/workflows/ci-dagster.yml
index e7040bc8ff48..787b4cd2a826 100644
--- a/.github/workflows/ci-dagster.yml
+++ b/.github/workflows/ci-dagster.yml
@@ -158,6 +158,7 @@ jobs:
# --- DAG code itself ---
- 'posthog/dags/**'
- 'products/*/dags/**'
+ - 'posthog/temporal/health_checks/rollups/**' # daily rollup fills run in Dagster
# --- posthog submodules that DAGs actually import ---
# (verified by tracing imports of all 148 dagster Python files)
- 'posthog/clickhouse/**'
@@ -282,6 +283,7 @@ jobs:
dagster_direct:
- 'posthog/dags/**'
- 'products/*/dags/**'
+ - 'posthog/temporal/health_checks/rollups/**' # daily rollup fills run in Dagster
- .github/workflows/ci-dagster.yml
- .github/actions/setup-uv/**
- tools/dagster_test_selection.py
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yaml
index 7efcccb62f9f..3670124d8d35 100644
--- a/.semgrep/rules/security/idor-team-scoped-models.yaml
+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml
@@ -389,6 +389,7 @@ rules:
|HeatmapCaptureRequest
|HeatmapScreenshotHistory
|ScheduledChange
+ |SdkVersionsDaily
|SessionGroupSummary
|SessionRecording
|SessionRecordingPlaylist
@@ -881,6 +882,7 @@ rules:
|HeatmapCaptureRequest
|HeatmapScreenshotHistory
|ScheduledChange
+ |SdkVersionsDaily
|SessionGroupSummary
|SessionRecording
|SessionRecordingPlaylist
@@ -1236,6 +1238,7 @@ rules:
|SandboxEnvironment
|SavedHeatmap
|ScoreDefinition
+ |SdkVersionsDaily
|SessionRecording
|SessionRecordingEvent
|SessionRecordingPlaylist
diff --git a/posthog/api/test/test_team_deletion.py b/posthog/api/test/test_team_deletion.py
index 1d79a8d36910..144e1f71e9d4 100644
--- a/posthog/api/test/test_team_deletion.py
+++ b/posthog/api/test/test_team_deletion.py
@@ -1,4 +1,5 @@
import json
+import datetime as dt
import time_machine
from posthog.test.base import NonAtomicBaseTest
@@ -18,6 +19,7 @@ from posthog.temporal.tests.delete_teams.inline import execute_deletion_workflow
from products.batch_exports.backend.facade.temporal import ACTIVITIES, WORKFLOWS
from products.early_access_features.backend.models import EarlyAccessFeature
+from products.growth.backend.models import SdkVersionsDaily
class TestTeamDeletionSideEffects(NonAtomicBaseTest):
@@ -170,11 +172,22 @@ class TestTeamDeletionSideEffects(NonAtomicBaseTest):
EarlyAccessFeature.objects.create(
team=team, name="Test flag", description="A fancy new flag.", stage="beta", feature_flag=flag
)
+ for team_id in (team.id, self.team.id):
+ SdkVersionsDaily.objects.unscoped().create(
+ team_id=team_id,
+ day=dt.date(2026, 10, 22),
+ lib="web",
+ lib_version="1.0.0",
+ event_count=1,
+ max_timestamp=dt.datetime(2026, 10, 22, tzinfo=dt.UTC),
+ )
with execute_deletion_workflows_inline():
response = self.client.delete(f"/api/environments/{team.id}")
self.assertEqual(response.status_code, 204)
self.assertFalse(Team.objects.filter(id=team.id).exists())
+ self.assertFalse(SdkVersionsDaily.objects.for_team(team.id).exists())
+ self.assertTrue(SdkVersionsDaily.objects.for_team(self.team.id).exists())
# Verify personhog RPCs were called for persons-DB cleanup
fake = get_active_fake()
diff --git a/posthog/dags/daily_rollups.py b/posthog/dags/daily_rollups.py
new file mode 100644
index 000000000000..6d2ec21068aa
--- /dev/null
+++ b/posthog/dags/daily_rollups.py
@@ -0,0 +1,105 @@
+import datetime as dt
+
+import dagster
+import pydantic
+
+from posthog.dags.common import JobOwners
+from posthog.dataclasses import frozen
+from posthog.temporal.health_checks.rollups.framework import DailyRollup
+from posthog.temporal.health_checks.rollups.runner import (
+ RollupDayRefused,
+ clean_up,
+ plan_requested_days,
+ plan_scheduled_days,
+ published_days_in_retention,
+ run_rollup_day,
+)
+
+DAILY_ROLLUP_CRON = "0 1 * * *"
+# A first run fills every day that retention keeps, one ClickHouse day scan after another.
+MAX_RUNTIME_SECONDS = 6 * 60 * 60
+# A retry plans again from the published days, so it refills the refresh day that a transient error failed.
+# The next scheduled run does not, because it refreshes a later day.
+FILL_RETRY_POLICY = dagster.RetryPolicy(max_retries=2, delay=15 * 60)
+
+
+class DailyRollupConfig(dagster.Config):
+ days: list[str] = pydantic.Field(
+ default_factory=list,
+ description=(
+ "UTC days to fill, as YYYY-MM-DD. Leave empty to publish yesterday, refresh the day before, "
+ "and fill every missing day that retention keeps."
+ ),
+ )
+ force: bool = pydantic.Field(
+ default=False,
+ description="Publish a day even when it has fewer than half the rows of the recent median.",
+ )
+
+
+@frozen
+class DailyRollupDefinitions:
+ job: dagster.JobDefinition
+ schedule: dagster.ScheduleDefinition
+
+
+def _today() -> dt.date:
+ return dt.datetime.now(dt.UTC).date()
+
+
+def build_daily_rollup_definitions(rollup: type[DailyRollup], *, owner: JobOwners) -> DailyRollupDefinitions:
+ """A daily job that fills one rollup table from ClickHouse, and its schedule.
+
+ Every planned day runs even when an earlier one fails, and clean-up runs last. The op then fails
+ if any day failed, which leaves that day as it was. Dagster retries it unless every failed day was
+ refused, because a refused day fails the same way again.
+ """
+ table = rollup.table()
+
+ @dagster.op(name=f"fill_{table}", retry_policy=FILL_RETRY_POLICY)
+ def fill_rollup(context: dagster.OpExecutionContext, config: DailyRollupConfig) -> None:
+ today = _today()
+ if config.days:
+ plan = plan_requested_days([dt.date.fromisoformat(day) for day in config.days], today=today)
+ else:
+ plan = plan_scheduled_days(rollup, today=today, published=published_days_in_retention(rollup, today=today))
+ context.log.info(f"Filling {table} for {', '.join(f'{step.day} ({step.mode})' for step in plan)}")
+
+ failures: list[str] = []
+ only_refusals = True
+ for step in plan:
+ try:
+ result = run_rollup_day(rollup, step, force=config.force)
+ except Exception as error:
+ context.log.exception(f"Filling {table} for {step.day} failed")
+ failures.append(f"{step.day}: {error}")
+ only_refusals = only_refusals and isinstance(error, RollupDayRefused)
+ continue
+ context.log.info(f"{table} {result.day}: {result.outcome}, {result.rows:,} rows, {result.events:,} events")
+
+ deleted_rows = clean_up(rollup, today=today)
+ context.log.info(f"Clean-up removed {deleted_rows:,} rows of {table}")
+ if failures:
+ raise dagster.Failure(
+ description=f"{len(failures)} of {len(plan)} days of {table} failed. "
+ "Launch the job with `days` to fill them again, and `force` to publish a refused day. "
+ + "; ".join(failures),
+ allow_retries=not only_refusals,
+ )
+
+ @dagster.job(
+ name=f"{table}_job",
+ description=f"Fills the {table} daily rollup from ClickHouse events.",
+ tags={"owner": owner.value, "dagster/max_runtime": MAX_RUNTIME_SECONDS},
+ )
+ def fill_rollup_job() -> None:
+ fill_rollup()
+
+ schedule = dagster.ScheduleDefinition(
+ name=f"{table}_schedule",
+ job=fill_rollup_job,
+ cron_schedule=DAILY_ROLLUP_CRON,
+ execution_timezone="UTC",
+ default_status=dagster.DefaultScheduleStatus.RUNNING,
+ )
+ return DailyRollupDefinitions(job=fill_rollup_job, schedule=schedule)
diff --git a/posthog/dags/locations/growth.py b/posthog/dags/locations/growth.py
index 30ab3ec506d6..db9d7c17f52f 100644
--- a/posthog/dags/locations/growth.py
+++ b/posthog/dags/locations/growth.py
@@ -7,6 +7,7 @@ from products.growth.dags import (
identity_matching,
oauth,
product_push_campaigns,
+ sdk_versions_rollup,
team_production_event_activation,
)
@@ -18,6 +19,7 @@ jobs = [
team_production_event_activation.detect_first_team_production_event_job,
product_push_campaigns.product_push_campaigns_job,
ai_enrichment.ai_enrichment_job,
+ sdk_versions_rollup.sdk_versions_daily_rollup.job,
# Manual-run only: growth names the organizations, product, and window per run.
custom_product_push_campaigns.custom_product_push_campaigns_job,
]
@@ -27,6 +29,7 @@ schedules = [
team_production_event_activation.detect_first_team_production_event_schedule,
product_push_campaigns.product_push_campaigns_schedule,
ai_enrichment.ai_enrichment_schedule,
+ sdk_versions_rollup.sdk_versions_daily_rollup.schedule,
]
# Identity matching processes internal PostHog data that only exists on Cloud US (team 2),
# so the job is not registered on Cloud EU.
diff --git a/posthog/models/team/util.py b/posthog/models/team/util.py
index 6d3f70ee2d7f..7d16f69f6854 100644
--- a/posthog/models/team/util.py
+++ b/posthog/models/team/util.py
@@ -89,6 +89,7 @@ def _delete_misc_small_tables_for_teams(team_ids: list[int]) -> None:
from products.data_modeling.backend.facade.models import Edge, Node
from products.early_access_features.backend.models import EarlyAccessFeature
+ from products.growth.backend.facade import api as growth_facade
from products.today.backend.facade import api as today_facade
error_tracking_fingerprint = apps.get_model("error_tracking", "ErrorTrackingIssueFingerprintV2")
@@ -105,6 +106,7 @@ def _delete_misc_small_tables_for_teams(team_ids: list[int]) -> None:
_raw_delete_batch(error_tracking_fingerprint.objects.filter(team_id__in=team_ids))
# Highlights carry team_id as a plain integer, so nothing cascades to them.
today_facade.delete_highlights_for_teams(team_ids)
+ growth_facade.delete_sdk_version_rollups_for_teams(team_ids)
# FeatureFlagHashKeyOverride references Person, so it must go before persons are deleted.
_delete_hash_key_overrides_for_teams(team_ids)
_delete_llm_evaluations_for_teams(team_ids)
diff --git a/posthog/temporal/health_checks/README.md b/posthog/temporal/health_checks/README.md
index 150b64eb3c0e..24a0836e2216 100644
--- a/posthog/temporal/health_checks/README.md
+++ b/posthog/temporal/health_checks/README.md
@@ -227,6 +227,42 @@ rows = execute_clickhouse_health_team_query(
Default ClickHouse settings: `max_execution_time=30`, `max_threads=2`.
+## Daily rollups
+
+Checks that would scan days of raw `events` read a daily rollup on Postgres instead.
+A Dagster job per rollup reads one UTC day from ClickHouse and replaces that day in a small Postgres table in one transaction.
+
+| Table | Product | Kept | Dagster job |
+| --------------------------- | ------- | ------- | ------------------------------- |
+| `growth_sdk_versions_daily` | growth | 10 days | `growth_sdk_versions_daily_job` |
+
+### How a day is published
+
+The job runs at 01:00 UTC. It publishes yesterday, and refills the day before only if its rows or events moved by more than 1%, which picks up late events.
+It also publishes every day that retention keeps and that has no rows yet.
+So the first run after a deploy fills the whole retention window, and a day lost to an outage comes back on the next run.
+
+A filled day is refused, and the published day stays as it was, when:
+
+- it has no rows, which is what a ClickHouse outage looks like
+- it has more than 2M rows
+- it has fewer than half the rows of the median of the last 7 published days, unless the run is forced
+
+Rows for teams that no longer exist are dropped before writing, and team deletion removes a team's rows.
+Every run also deletes the rows of teams that no longer exist, which a fill can write while a team deletion runs.
+A refused or failed day does not stop the other days or the clean-up, and the run fails at the end, which alerts the owning team.
+Dagster retries a failed run twice, 15 minutes apart, unless every failed day was refused.
+
+To fill a day by hand, launch the job in Dagster with `days: ["2026-10-01", ...]`, and add `force: true` to publish a day the median rule refused.
+
+### Adding a rollup
+
+Subclass `DailyRollup` (`rollups/framework.py`) next to the product's model, with `columns`, `fill_sql()` and `retention_days`.
+The fill drops rows whose strings exceed a column's `max_length` or hold a NUL byte, but filter client-controlled values in `fill_sql()` too, so the dropped count stays near zero.
+A fill reads every team's events, so read event properties through their materialized columns where they exist.
+Build its job and schedule with `build_daily_rollup_definitions` (`posthog/dags/daily_rollups.py`) in the product's `dags/`, and register both in the product's Dagster location.
+The model needs a composite primary key over the natural key, an index on `day`, no foreign keys, and `EnvironmentScopedManager`, and the product facade must delete its rows on team deletion.
+
## ClickHouse kill switch
Health checks are automatically skipped when the ClickHouse kill switch is active (LIGHT or FULL).
diff --git a/posthog/temporal/health_checks/rollups/__init__.py b/posthog/temporal/health_checks/rollups/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/posthog/temporal/health_checks/rollups/fill.py b/posthog/temporal/health_checks/rollups/fill.py
new file mode 100644
index 000000000000..fb1e56b8976d
--- /dev/null
+++ b/posthog/temporal/health_checks/rollups/fill.py
@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+import datetime as dt
+import tempfile
+from collections.abc import (
+ Iterable,
+ Iterator,
+ Sequence,
+ Set as AbstractSet,
+)
+from typing import IO, Any
+
+import structlog
+
+from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.query_tagging import Feature, tags_context
+from posthog.clickhouse.workload import Workload
+from posthog.dataclasses import frozen
+from posthog.temporal.health_checks.rollups.framework import DailyRollup
+from posthog.utils import ensure_utc
+
+logger = structlog.get_logger(__name__)
+
+TEAM_ID_RANGE_COUNT = 16
+
+SPOOL_MAX_BYTES = 64 * 1024 * 1024
+
+FILL_QUERY_SETTINGS = {"max_execution_time": 1800, "max_threads": 8}
+
+
+class DayOverRowCap(Exception):
+ """The day has more rows than the rollup allows, so nothing of it may be published."""
+
+
+@frozen
+class TeamIdRange:
+ start: int
+ end: int
+
+
+@frozen(frozen=False)
+class DayFill:
+ """One filled day, written in COPY text format into `buffer` and ready to publish.
+
+ Close it once published, or once it is refused, to free the spooled file.
+ """
+
+ day: dt.date
+ row_count: int
+ event_count: int
+ buffer: IO[bytes]
+
+ def chunks(self, size: int = 1024 * 1024) -> Iterator[bytes]:
+ self.buffer.seek(0)
+ while chunk := self.buffer.read(size):
+ yield chunk
+
+ def close(self) -> None:
+ self.buffer.close()
+
+
+def team_id_ranges(team_ids: Iterable[int], count: int = TEAM_ID_RANGE_COUNT) -> list[TeamIdRange]:
+ """Split the team ids into at most `count` ranges that hold about the same number of teams.
+
+ Team ids are dense at the top, where active teams are, so equal-width ranges would put most
+ teams in the last few ranges.
+ """
+ ids = sorted(set(team_ids))
+ if not ids:
+ return []
+ count = max(1, min(count, len(ids)))
+ starts = [ids[(len(ids) * index) // count] for index in range(count)]
+ ends = [*starts[1:], ids[-1] + 1]
+ return [TeamIdRange(start=start, end=end) for start, end in zip(starts, ends)]
+
+
+def _copy_text(value: Any) -> str:
+ """Format one value for the Postgres COPY text format."""
+ if value is None:
+ return "\\N"
+ if isinstance(value, bool):
+ return "t" if value else "f"
+ if isinstance(value, dt.datetime):
+ return ensure_utc(value).isoformat()
+ if isinstance(value, dt.date):
+ return value.isoformat()
+ text = str(value)
+ if "\x00" in text:
+ raise ValueError("COPY text cannot hold a NUL byte")
+ return text.replace("\\", "\\\\").replace("\t", "\\t").replace("\n", "\\n").replace("\r", "\\r")
+
+
+def _row_columns(rollup: type[DailyRollup]) -> tuple[str, ...]:
+ return (rollup.columns[0], *rollup.columns[2:])
+
+
+@frozen
+class _StringLimit:
+ position: int
+ max_length: int
+
+
+def _string_limits(rollup: type[DailyRollup]) -> list[_StringLimit]:
+ """The row position and max_length of every length-limited column."""
+ limits = []
+ for position, column in enumerate(_row_columns(rollup)):
+ max_length = getattr(rollup.model._meta.get_field(column), "max_length", None)
+ if max_length is not None:
+ limits.append(_StringLimit(position=position, max_length=max_length))
+ return limits
+
+
+def _storable(row: Sequence[Any], string_limits: list[_StringLimit]) -> bool:
+ for limit in string_limits:
+ value = row[limit.position]
+ if isinstance(value, str) and (len(value) > limit.max_length or "\x00" in value):
+ return False
+ return True
+
+
+def _fetch_rows(rollup: type[DailyRollup], day: dt.date, team_ids: AbstractSet[int]) -> Iterator[Sequence[Any]]:
+ params = {
+ **rollup.fill_params(),
+ "day_start": f"{day.isoformat()} 00:00:00",
+ "day_end": f"{(day + dt.timedelta(days=1)).isoformat()} 00:00:00",
+ }
+ sql = rollup.fill_sql()
+ # ClickHouse stops a range at the cap, so one oversized range cannot fill the worker's memory.
+ settings = {
+ **FILL_QUERY_SETTINGS,
+ "max_result_rows": rollup.max_rows_per_day + 1,
+ "result_overflow_mode": "break",
+ }
+ for team_range in team_id_ranges(team_ids):
+ with tags_context(product=rollup.product, feature=Feature.HEALTH_CHECK, name=f"rollup_{rollup.table()}"):
+ rows = sync_execute(
+ sql,
+ {**params, "team_id_start": team_range.start, "team_id_end": team_range.end},
+ settings=settings,
+ workload=Workload.OFFLINE,
+ )
+ if len(rows) > rollup.max_rows_per_day:
+ raise DayOverRowCap(f"one team id range has more than {rollup.max_rows_per_day:,} rows")
+ yield from rows
+
+
+def fill_day(rollup: type[DailyRollup], day: dt.date, live_team_ids: AbstractSet[int]) -> DayFill:
+ """Read one UTC day from ClickHouse, one team id range at a time, into a COPY buffer.
+
+ Rows for teams that no longer exist and rows that do not fit the table are dropped here,
+ before anything reaches Postgres. A day over the row cap raises DayOverRowCap.
+ """
+ event_count_position = _row_columns(rollup).index("event_count")
+ string_limits = _string_limits(rollup)
+
+ buffer: IO[bytes] = tempfile.SpooledTemporaryFile(max_size=SPOOL_MAX_BYTES, mode="w+b")
+ row_count = 0
+ event_count = 0
+ dropped_unknown_team = 0
+ dropped_unstorable = 0
+ try:
+ for row in _fetch_rows(rollup, day, live_team_ids):
+ if row[0] not in live_team_ids:
+ dropped_unknown_team += 1
+ continue
+ if not _storable(row, string_limits):
+ dropped_unstorable += 1
+ continue
+ values = (row[0], day, *row[1:])
+ buffer.write(("\t".join(_copy_text(value) for value in values) + "\n").encode())
+ row_count += 1
+ event_count += int(row[event_count_position])
+ if row_count > rollup.max_rows_per_day:
+ raise DayOverRowCap(f"the day has more than {rollup.max_rows_per_day:,} rows")
+ except BaseException:
+ buffer.close()
+ raise
+
+ logger.info(
+ "daily rollup filled",
+ table=rollup.table(),
+ day=day.isoformat(),
+ rows=row_count,
+ events=event_count,
+ dropped_unknown_team=dropped_unknown_team,
+ dropped_unstorable=dropped_unstorable,
+ )
+ return DayFill(day=day, row_count=row_count, event_count=event_count, buffer=buffer)
diff --git a/posthog/temporal/health_checks/rollups/framework.py b/posthog/temporal/health_checks/rollups/framework.py
new file mode 100644
index 000000000000..5443f3849e5a
--- /dev/null
+++ b/posthog/temporal/health_checks/rollups/framework.py
@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from posthog.clickhouse.query_tagging import Product
+
+if TYPE_CHECKING:
+ from django.db import models
+
+DEFAULT_MAX_ROWS_PER_DAY = 2_000_000
+
+
+class DailyRollup:
+ """One Postgres table that holds one row per natural key per UTC day, filled from ClickHouse.
+
+ A Dagster job built with `build_daily_rollup_job` fills a day by running `fill_sql()` once per
+ team id range, and replaces the day in Postgres in one transaction.
+
+ `fill_sql()` returns the table's columns in `columns` order, without `day`, which the framework
+ adds. It must filter on `%(team_id_start)s <= team_id < %(team_id_end)s` and on
+ `%(day_start)s <= timestamp < %(day_end)s`. `columns` must include `event_count`.
+ """
+
+ model: ClassVar[type[models.Model]]
+ product: ClassVar[Product | None] = None
+ columns: ClassVar[tuple[str, ...]]
+ retention_days: ClassVar[int]
+ max_rows_per_day: ClassVar[int] = DEFAULT_MAX_ROWS_PER_DAY
+
+ def __init_subclass__(cls, **kwargs: object) -> None:
+ super().__init_subclass__(**kwargs)
+ if "model" not in cls.__dict__:
+ return
+ if cls.columns[:2] != ("team_id", "day"):
+ raise ValueError(f"{cls.__name__}.columns must start with team_id and day")
+ if "event_count" not in cls.columns:
+ raise ValueError(f"{cls.__name__}.columns must include event_count")
+
+ @classmethod
+ def table(cls) -> str:
+ return cls.model._meta.db_table
+
+ @classmethod
+ def fill_sql(cls) -> str:
+ raise NotImplementedError
+
+ @classmethod
+ def fill_params(cls) -> Mapping[str, Any]:
+ """Extra ClickHouse parameters for `fill_sql()`, such as an allowlist."""
+ return {}
diff --git a/posthog/temporal/health_checks/rollups/freshness.py b/posthog/temporal/health_checks/rollups/freshness.py
new file mode 100644
index 000000000000..9f31d0b7fd22
--- /dev/null
+++ b/posthog/temporal/health_checks/rollups/freshness.py
@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import datetime as dt
+from collections.abc import Collection
+from typing import Any
+
+from django.core.cache import cache
+from django.db import connections, models, router
+
+import structlog
+
+from posthog.dataclasses import frozen
+
+logger = structlog.get_logger(__name__)
+
+PUBLISHED_DAYS_CACHE_SECONDS = 5 * 60
+MAX_LATEST_DAY_AGE_DAYS = 2
+MAX_PUBLISHED_DAYS_LISTED = 64
+
+
+@frozen
+class RollupWindow:
+ start: dt.date
+ end: dt.date
+
+ def __post_init__(self) -> None:
+ if self.start > self.end:
+ raise ValueError(f"RollupWindow start {self.start} is after end {self.end}")
+
+ @property
+ def live_since(self) -> dt.datetime:
+ """The UTC instant after the window, from which live queries cover the gap to now."""
+ return dt.datetime.combine(self.end + dt.timedelta(days=1), dt.time.min, tzinfo=dt.UTC)
+
+
+def window_rows(model: type[models.Model], team_ids: Collection[int], window: RollupWindow) -> models.QuerySet[Any]:
+ """The rollup rows of these environments inside the window. Name the model in READ_REPLICA_OPT_IN to read the replica."""
+ return model.objects.unscoped().filter( # type: ignore[attr-defined]
+ team_id__in=team_ids, day__range=(window.start, window.end)
+ )
+
+
+def _published_days_cache_key(table: str) -> str:
+ return f"daily_rollup:published_days:{table}"
+
+
+def _query_published_days(table: str, using: str) -> list[dt.date]:
+ query = f"""
+ WITH RECURSIVE published(day) AS (
+ (SELECT day FROM {table} ORDER BY day DESC LIMIT 1)
+ UNION ALL
+ SELECT (SELECT t.day FROM {table} t WHERE t.day < published.day ORDER BY t.day DESC LIMIT 1)
+ FROM published
+ WHERE published.day IS NOT NULL
+ )
+ SELECT day FROM published WHERE day IS NOT NULL LIMIT %s
+ """
+ with connections[using].cursor() as cursor:
+ cursor.execute(query, [MAX_PUBLISHED_DAYS_LISTED])
+ return [row[0] for row in cursor.fetchall()]
+
+
+def published_days(model: type[models.Model]) -> list[dt.date]:
+ """Published days of a rollup table, newest first, cached for a few minutes."""
+ table = model._meta.db_table
+ cache_key = _published_days_cache_key(table)
+ cached = cache.get(cache_key)
+ if cached is not None:
+ return [dt.date.fromisoformat(day) for day in cached]
+ days = _query_published_days(table, router.db_for_read(model))
+ cache.set(cache_key, [day.isoformat() for day in days], PUBLISHED_DAYS_CACHE_SECONDS)
+ return days
+
+
+def invalidate_published_days(model: type[models.Model]) -> None:
+ cache.delete(_published_days_cache_key(model._meta.db_table))
+
+
+def window_from_published_days(
+ published: list[dt.date], days: int, *, today: dt.date, table: str = ""
+) -> RollupWindow | None:
+ """The `days`-day window ending at the newest published day, or None when readers must not use it.
+
+ None when nothing is published, when the newest day is too old, or when any day of the window
+ is missing. A reader that sums a window with a gap would undercount, and a check would read
+ the undercount as missing events.
+ """
+ if days <= 0:
+ raise ValueError(f"days must be > 0, got {days}")
+ if not published:
+ logger.warning("daily rollup has no published days", table=table)
+ return None
+ latest = max(published)
+ if latest < today - dt.timedelta(days=MAX_LATEST_DAY_AGE_DAYS):
+ logger.warning("daily rollup is stale", table=table, latest_day=latest.isoformat(), today=today.isoformat())
+ return None
+ window = RollupWindow(start=latest - dt.timedelta(days=days - 1), end=latest)
+ published_set = set(published)
+ missing = [
+ day for day in (window.start + dt.timedelta(days=offset) for offset in range(days)) if day not in published_set
+ ]
+ if missing:
+ logger.warning(
+ "daily rollup window has missing days",
+ table=table,
+ window_days=days,
+ missing_days=[day.isoformat() for day in missing[:10]],
+ missing_count=len(missing),
+ )
+ return None
+ return window
+
+
+def rollup_window(model: type[models.Model], days: int) -> RollupWindow | None:
+ return window_from_published_days(
+ published_days(model), days, today=dt.datetime.now(dt.UTC).date(), table=model._meta.db_table
+ )
diff --git a/posthog/temporal/health_checks/rollups/runner.py b/posthog/temporal/health_checks/rollups/runner.py
new file mode 100644
index 000000000000..4ae4d667f45e
--- /dev/null
+++ b/posthog/temporal/health_checks/rollups/runner.py
@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import datetime as dt
+from collections.abc import Collection
+from typing import Literal
+
+import structlog
+
+from posthog.dataclasses import frozen
+from posthog.models.team import Team
+from posthog.temporal.health_checks.rollups.fill import DayOverRowCap, fill_day
+from posthog.temporal.health_checks.rollups.framework import DailyRollup
+from posthog.temporal.health_checks.rollups.freshness import invalidate_published_days
+from posthog.temporal.health_checks.rollups.store import (
+ day_stats,
+ delete_days_before,
+ delete_rows_of_deleted_teams,
+ publish_day,
+)
+from posthog.temporal.health_checks.rollups.validation import day_moved, refusal_reason
+
+logger = structlog.get_logger(__name__)
+
+MEDIAN_HISTORY_DAYS = 7
+
+RollupDayMode = Literal["publish", "refresh"]
+RollupDayOutcome = Literal["published", "unchanged"]
+
+
+@frozen
+class RollupDay:
+ day: dt.date
+ mode: RollupDayMode
+
+
+@frozen
+class RollupDayResult:
+ day: dt.date
+ outcome: RollupDayOutcome
+ rows: int
+ events: int
+
+
+class RollupDayRefused(Exception):
+ """The filled day looks wrong, so the published day stays as it was."""
+
+
+def plan_scheduled_days(
+ rollup: type[DailyRollup], *, today: dt.date, published: Collection[dt.date]
+) -> list[RollupDay]:
+ """The days one scheduled run fills, newest first.
+
+ It publishes yesterday and refreshes the day before, to pick up late events. It also publishes
+ every missing day that retention keeps, so the first run after a deploy and the run after an
+ outage fill the gap with no manual backfill.
+ """
+ yesterday = today - dt.timedelta(days=1)
+ plan = [RollupDay(day=yesterday, mode="publish"), RollupDay(day=yesterday - dt.timedelta(days=1), mode="refresh")]
+ oldest_kept = today - dt.timedelta(days=rollup.retention_days)
+ day = yesterday - dt.timedelta(days=2)
+ while day >= oldest_kept:
+ if day not in published:
+ plan.append(RollupDay(day=day, mode="publish"))
+ day -= dt.timedelta(days=1)
+ return plan
+
+
+def plan_requested_days(days: Collection[dt.date], *, today: dt.date) -> list[RollupDay]:
+ future = sorted(day for day in days if day >= today)
+ if future:
+ raise ValueError(f"Cannot fill a day that has not ended: {', '.join(day.isoformat() for day in future)}")
+ return [RollupDay(day=day, mode="publish") for day in sorted(set(days))]
+
+
+def published_days_in_retention(rollup: type[DailyRollup], *, today: dt.date) -> set[dt.date]:
+ kept = [today - dt.timedelta(days=offset) for offset in range(1, rollup.retention_days + 1)]
+ return set(day_stats(rollup, kept))
+
+
+def _live_team_ids() -> set[int]:
+ return set(Team.objects.values_list("id", flat=True))
+
+
+def run_rollup_day(rollup: type[DailyRollup], step: RollupDay, *, force: bool = False) -> RollupDayResult:
+ """Fill one day of one rollup and publish it unless it is refused or unchanged.
+
+ Every slow part, the ClickHouse reads, runs before the Postgres transaction opens.
+ """
+ try:
+ fill = fill_day(rollup, step.day, _live_team_ids())
+ except DayOverRowCap as error:
+ logger.warning("daily rollup day refused", table=rollup.table(), day=step.day.isoformat(), reason=str(error))
+ raise RollupDayRefused(f"Refused to publish {rollup.table()} for {step.day.isoformat()}: {error}") from error
+ try:
+ if step.mode == "refresh" and not day_moved(day_stats(rollup, [step.day]).get(step.day), fill):
+ logger.info(
+ "daily rollup day unchanged", table=rollup.table(), day=step.day.isoformat(), rows=fill.row_count
+ )
+ return RollupDayResult(day=step.day, outcome="unchanged", rows=fill.row_count, events=fill.event_count)
+
+ history_days = [step.day - dt.timedelta(days=offset) for offset in range(1, MEDIAN_HISTORY_DAYS + 1)]
+ stats = day_stats(rollup, history_days)
+ history = [stats[history_day] for history_day in history_days if history_day in stats]
+ reason = refusal_reason(rollup, fill, history, force=force)
+ if reason is not None:
+ logger.warning("daily rollup day refused", table=rollup.table(), day=step.day.isoformat(), reason=reason)
+ raise RollupDayRefused(f"Refused to publish {rollup.table()} for {step.day.isoformat()}: {reason}")
+
+ publish_day(rollup, fill)
+ invalidate_published_days(rollup.model)
+ return RollupDayResult(day=step.day, outcome="published", rows=fill.row_count, events=fill.event_count)
+ finally:
+ fill.close()
+
+
+def clean_up(rollup: type[DailyRollup], *, today: dt.date) -> int:
+ """Delete the days retention no longer keeps and the rows of deleted environments."""
+ deleted = delete_days_before(rollup, today - dt.timedelta(days=rollup.retention_days))
+ deleted += delete_rows_of_deleted_teams(rollup)
+ if deleted:
+ invalidate_published_days(rollup.model)
+ return deleted
diff --git a/posthog/temporal/health_checks/rollups/store.py b/posthog/temporal/health_checks/rollups/store.py
new file mode 100644
index 000000000000..ae7595581d47
--- /dev/null
+++ b/posthog/temporal/health_checks/rollups/store.py
@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import datetime as dt
+from collections.abc import Sequence
+
+from django.db import connections, models, transaction
+from django.db.models import Exists, OuterRef
+
+import structlog
+
+from posthog.dataclasses import frozen
+from posthog.models.team import Team
+from posthog.temporal.health_checks.rollups.fill import DayFill
+from posthog.temporal.health_checks.rollups.framework import DailyRollup
+
+logger = structlog.get_logger(__name__)
+
+WRITE_DATABASE = "default"
+DELETE_BATCH_SIZE = 10_000
+
+PUBLISH_STATEMENT_TIMEOUT = "10min"
+PUBLISH_LOCK_TIMEOUT = "30s"
+
+
+@frozen
+class DayStats:
+ row_count: int
+ event_count: int
+
+
+def _advisory_lock_key(rollup: type[DailyRollup]) -> str:
+ return f"daily_rollup:{rollup.table()}"
+
+
+def publish_day(rollup: type[DailyRollup], fill: DayFill) -> None:
+ """Replace one day of the rollup with the filled rows, in one transaction.
+
+ Readers on the replica see the old day or the new one, never a mix. The advisory lock is
+ transaction-scoped, so it works behind PgBouncer and releases on commit or rollback.
+ """
+ table = rollup.table()
+ column_list = ", ".join(rollup.columns)
+ delete_sql = f"DELETE FROM {table} WHERE day = %s"
+ copy_sql = f"COPY {table} ({column_list}) FROM STDIN"
+
+ with transaction.atomic(using=WRITE_DATABASE), connections[WRITE_DATABASE].cursor() as cursor:
+ cursor.execute(f"SET LOCAL statement_timeout = '{PUBLISH_STATEMENT_TIMEOUT}'")
+ cursor.execute(f"SET LOCAL lock_timeout = '{PUBLISH_LOCK_TIMEOUT}'")
+ cursor.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", [_advisory_lock_key(rollup)])
+ cursor.execute(delete_sql, [fill.day])
+ replaced_rows = cursor.rowcount
+ with cursor.cursor.copy(copy_sql) as copy:
+ for chunk in fill.chunks():
+ copy.write(chunk)
+
+ logger.info(
+ "daily rollup published",
+ table=table,
+ day=fill.day.isoformat(),
+ rows=fill.row_count,
+ events=fill.event_count,
+ replaced_rows=replaced_rows,
+ )
+
+
+def day_stats(rollup: type[DailyRollup], days: Sequence[dt.date]) -> dict[dt.date, DayStats]:
+ """Row and event counts per published day. A day with no rows is absent from the result."""
+ if not days:
+ return {}
+ rows = (
+ rollup.model.objects.unscoped() # type: ignore[attr-defined]
+ .using(WRITE_DATABASE)
+ .filter(day__in=list(days))
+ .values("day")
+ .annotate(row_count=models.Count("*"), event_count=models.Sum("event_count"))
+ .order_by("day")
+ )
+ return {row["day"]: DayStats(row_count=row["row_count"], event_count=row["event_count"] or 0) for row in rows}
+
+
+def _delete_in_batches(table: str, condition: str, params: list[dt.date | list[int]]) -> int:
+ delete_sql = f"DELETE FROM {table} WHERE ctid = ANY(ARRAY(SELECT ctid FROM {table} WHERE {condition} LIMIT %s))"
+ deleted = 0
+ while True:
+ with connections[WRITE_DATABASE].cursor() as cursor:
+ cursor.execute(delete_sql, [*params, DELETE_BATCH_SIZE])
+ batch = cursor.rowcount
+ deleted += batch
+ if batch < DELETE_BATCH_SIZE:
+ return deleted
+
+
+def delete_days_before(rollup: type[DailyRollup], cutoff: dt.date) -> int:
+ deleted = _delete_in_batches(rollup.table(), "day < %s", [cutoff])
+ logger.info("daily rollup retention", table=rollup.table(), cutoff=cutoff.isoformat(), deleted_rows=deleted)
+ return deleted
+
+
+def delete_rollup_rows_for_teams(model: type[models.Model], team_ids: Sequence[int]) -> int:
+ """Delete every row of the given environments. Team deletion calls this, because no foreign key cascades."""
+ if not team_ids:
+ return 0
+ return _delete_in_batches(model._meta.db_table, "team_id = ANY(%s)", [list(team_ids)])
+
+
+def delete_rows_of_deleted_teams(rollup: type[DailyRollup]) -> int:
+ """Delete the rows of environments that no longer exist.
+
+ A fill that publishes while a team deletion runs can write rows after the deletion removed them.
+ """
+ deleted_team_ids = list(
+ rollup.model.objects.unscoped() # type: ignore[attr-defined]
+ .using(WRITE_DATABASE)
+ .filter(~Exists(Team.objects.filter(id=OuterRef("team_id"))))
+ .values_list("team_id", flat=True)
+ .distinct()
+ )
+ return delete_rollup_rows_for_teams(rollup.model, deleted_team_ids)
diff --git a/posthog/temporal/health_checks/rollups/tests/__init__.py b/posthog/temporal/health_checks/rollups/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/posthog/temporal/health_checks/rollups/tests/test_rollup_rules.py b/posthog/temporal/health_checks/rollups/tests/test_rollup_rules.py
new file mode 100644
index 000000000000..b163d8cf3d9a
--- /dev/null
+++ b/posthog/temporal/health_checks/rollups/tests/test_rollup_rules.py
@@ -0,0 +1,155 @@
+import io
+import datetime as dt
+
+import pytest
+
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from posthog.temporal.health_checks.rollups.fill import DayFill, _copy_text, team_id_ranges
+from posthog.temporal.health_checks.rollups.freshness import window_from_published_days
+from posthog.temporal.health_checks.rollups.runner import plan_requested_days, plan_scheduled_days
+from posthog.temporal.health_checks.rollups.store import DayStats
+from posthog.temporal.health_checks.rollups.validation import day_moved, refusal_reason
+
+from products.growth.backend.sdk_version_rollup import SdkVersionsDailyRollup
+
+TODAY = dt.date(2026, 10, 29)
+
+
+def _fill(row_count: int, event_count: int = 1000) -> DayFill:
+ return DayFill(day=TODAY, row_count=row_count, event_count=event_count, buffer=io.BytesIO())
+
+
+def _history(*row_counts: int) -> list[DayStats]:
+ return [DayStats(row_count=rows, event_count=rows * 10) for rows in row_counts]
+
+
+def _days_back(*offsets: int) -> list[dt.date]:
+ return [TODAY - dt.timedelta(days=offset) for offset in offsets]
+
+
+class TestRollupRules(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("empty", [], 16, 0),
+ ("fewer_teams_than_ranges", [5, 9, 2], 16, 3),
+ ("many_teams", list(range(1, 1001)), 16, 16),
+ ("sparse_ids_with_duplicates", [1, 1, 50, 9_000, 9_001, 500_000], 4, 4),
+ ]
+ )
+ def test_team_id_ranges_cover_every_team_exactly_once(
+ self, _name: str, team_ids: list[int], count: int, expected_ranges: int
+ ) -> None:
+ ranges = team_id_ranges(team_ids, count)
+
+ assert len(ranges) == expected_ranges
+ for team_id in set(team_ids):
+ assert sum(r.start <= team_id < r.end for r in ranges) == 1
+
+ @parameterized.expand(
+ [
+ ("null", None, "\\N"),
+ ("true", True, "t"),
+ ("false", False, "f"),
+ ("int", 42, "42"),
+ ("date", dt.date(2026, 10, 22), "2026-10-22"),
+ ("naive_datetime_is_utc", dt.datetime(2026, 10, 22, 23, 59, 41), "2026-10-22T23:59:41+00:00"),
+ ("tab_newline_backslash", "1.0\t\n\r\\x", "1.0\\t\\n\\r\\\\x"),
+ ]
+ )
+ def test_copy_text_formats_values_for_postgres(self, _name: str, value: object, expected: str) -> None:
+ assert _copy_text(value) == expected
+
+ def test_copy_text_rejects_nul(self) -> None:
+ with pytest.raises(ValueError):
+ _copy_text("1.0\x00")
+
+ @parameterized.expand(
+ [
+ ("empty_day", 0, _history(100, 100, 100), False, "no rows"),
+ ("empty_day_even_when_forced", 0, _history(100, 100, 100), True, "no rows"),
+ ("under_half_the_median", 40, _history(100, 90, 110), False, "median"),
+ ("under_half_the_median_but_forced", 40, _history(100, 90, 110), True, None),
+ ("normal_day", 95, _history(100, 90, 110), False, None),
+ ("too_little_history_for_a_median", 1, _history(100, 100), False, None),
+ ]
+ )
+ def test_refusal_reason(
+ self, _name: str, row_count: int, history: list[DayStats], force: bool, expected: str | None
+ ) -> None:
+ reason = refusal_reason(SdkVersionsDailyRollup, _fill(row_count), history, force=force)
+
+ if expected is None:
+ assert reason is None
+ else:
+ assert reason is not None and expected in reason
+
+ @parameterized.expand(
+ [
+ ("never_published", None, 100, 1000, True),
+ ("identical", DayStats(row_count=100, event_count=1000), 100, 1000, False),
+ ("within_one_percent", DayStats(row_count=100, event_count=1000), 101, 1010, False),
+ ("late_events", DayStats(row_count=100, event_count=1000), 100, 1011, True),
+ ("new_rows", DayStats(row_count=100, event_count=1000), 102, 1000, True),
+ ]
+ )
+ def test_day_moved(self, _name: str, published: DayStats | None, rows: int, events: int, expected: bool) -> None:
+ assert day_moved(published, _fill(rows, events)) is expected
+
+ @parameterized.expand(
+ [
+ ("full_window", _days_back(*range(1, 8)), 7, (TODAY - dt.timedelta(days=7), TODAY - dt.timedelta(days=1))),
+ (
+ "latest_is_two_days_old",
+ _days_back(*range(2, 9)),
+ 7,
+ (TODAY - dt.timedelta(days=8), TODAY - dt.timedelta(days=2)),
+ ),
+ ("latest_is_three_days_old", _days_back(*range(3, 10)), 7, None),
+ ("gap_inside_window", _days_back(1, 2, 3, 5, 6, 7, 8), 7, None),
+ ("first_days_after_deploy", _days_back(1, 2, 3), 30, None),
+ ("nothing_published", [], 7, None),
+ ]
+ )
+ def test_window_from_published_days(
+ self, _name: str, published: list[dt.date], days: int, expected: tuple[dt.date, dt.date] | None
+ ) -> None:
+ window = window_from_published_days(published, days, today=TODAY)
+
+ assert (None if window is None else (window.start, window.end)) == expected
+
+ @parameterized.expand(
+ [
+ ("every_day_published", _days_back(*range(1, 11)), [(1, "publish"), (2, "refresh")]),
+ (
+ "gaps_inside_retention",
+ _days_back(1, 2, 3, 5, 6, 7, 8, 9),
+ [(1, "publish"), (2, "refresh"), (4, "publish"), (10, "publish")],
+ ),
+ (
+ "first_run_after_deploy",
+ [],
+ [(1, "publish"), (2, "refresh"), *[(offset, "publish") for offset in range(3, 11)]],
+ ),
+ ]
+ )
+ def test_scheduled_plan_refreshes_recent_days_and_fills_every_missing_day_retention_keeps(
+ self, _name: str, published: list[dt.date], expected: list[tuple[int, str]]
+ ) -> None:
+ plan = plan_scheduled_days(SdkVersionsDailyRollup, today=TODAY, published=published)
+
+ assert [(step.day, step.mode) for step in plan] == [
+ (TODAY - dt.timedelta(days=offset), mode) for offset, mode in expected
+ ]
+
+ def test_requested_plan_publishes_each_day_once_and_refuses_today(self) -> None:
+ plan = plan_requested_days(_days_back(9, 10, 9), today=TODAY)
+ assert [(step.day, step.mode) for step in plan] == [
+ (dt.date(2026, 10, 19), "publish"),
+ (dt.date(2026, 10, 20), "publish"),
+ ]
+
+ with pytest.raises(ValueError):
+ plan_requested_days([TODAY], today=TODAY)
diff --git a/posthog/temporal/health_checks/rollups/tests/test_rollup_store.py b/posthog/temporal/health_checks/rollups/tests/test_rollup_store.py
new file mode 100644
index 000000000000..4bb3ba9eae37
--- /dev/null
+++ b/posthog/temporal/health_checks/rollups/tests/test_rollup_store.py
@@ -0,0 +1,169 @@
+import datetime as dt
+
+import pytest
+import time_machine
+from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events, materialized
+from unittest.mock import patch
+
+from django.core.cache import cache
+
+import dagster
+
+from posthog.dags.daily_rollups import DailyRollupConfig
+from posthog.models.team import Team
+from posthog.temporal.health_checks.rollups import store
+from posthog.temporal.health_checks.rollups.runner import RollupDay, RollupDayRefused, run_rollup_day
+from posthog.temporal.health_checks.rollups.store import delete_days_before, delete_rollup_rows_for_teams
+
+from products.growth.backend.models import SdkVersionsDaily
+from products.growth.backend.sdk_version_rollup import SdkVersionsDailyRollup
+from products.growth.dags.sdk_versions_rollup import sdk_versions_daily_rollup
+
+YESTERDAY = dt.date(2026, 10, 28)
+NOON_YESTERDAY = dt.datetime.combine(YESTERDAY, dt.time(12), tzinfo=dt.UTC)
+
+
+def _sdk_row(team_id: int, day: dt.date, event_count: int = 1) -> SdkVersionsDaily:
+ return SdkVersionsDaily(
+ team_id=team_id,
+ day=day,
+ lib="web",
+ lib_version="1.0.0",
+ event_count=event_count,
+ max_timestamp=dt.datetime.combine(day, dt.time(12), tzinfo=dt.UTC),
+ )
+
+
+@time_machine.travel("2026-10-29T12:00:00Z", tick=False)
+class TestRollupFillAndPublish(ClickhouseTestMixin, BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ cache.clear()
+
+ def _capture_sdk_events(self, team: Team, events: list[tuple[str, str]], timestamp: dt.datetime) -> None:
+ for lib, version in events:
+ _create_event(
+ team=team,
+ event="$pageview",
+ distinct_id="user",
+ timestamp=timestamp,
+ properties={"$lib": lib, "$lib_version": version},
+ )
+
+ def test_publish_replaces_one_day_and_drops_deleted_teams(self) -> None:
+ deleted_team = Team.objects.create(organization=self.organization)
+ with materialized("events", "$lib"), materialized("events", "$lib_version"):
+ self._capture_sdk_events(self.team, [("web", "1.0.0")] * 3, NOON_YESTERDAY)
+ self._capture_sdk_events(deleted_team, [("web", "1.0.0")], NOON_YESTERDAY)
+ flush_persons_and_events()
+ Team.objects.filter(pk=deleted_team.pk).delete()
+ other_day = _sdk_row(self.team.id, YESTERDAY - dt.timedelta(days=5), event_count=7)
+ SdkVersionsDaily.objects.unscoped().bulk_create([other_day])
+
+ run_rollup_day(SdkVersionsDailyRollup, RollupDay(day=YESTERDAY, mode="publish"))
+ run_rollup_day(SdkVersionsDailyRollup, RollupDay(day=YESTERDAY, mode="publish"))
+
+ rows = list(SdkVersionsDaily.objects.unscoped().values_list("team_id", "day", "event_count").order_by("day"))
+ assert rows == [(self.team.id, other_day.day, 7), (self.team.id, YESTERDAY, 3)]
+
+ def test_refused_day_keeps_the_published_rows(self) -> None:
+ SdkVersionsDaily.objects.unscoped().bulk_create([_sdk_row(self.team.id, YESTERDAY, event_count=9)])
+
+ with materialized("events", "$lib"), materialized("events", "$lib_version"):
+ with pytest.raises(RollupDayRefused, match="no rows"):
+ run_rollup_day(SdkVersionsDailyRollup, RollupDay(day=YESTERDAY, mode="publish"))
+
+ assert list(SdkVersionsDaily.objects.for_team(self.team.id).values_list("event_count", flat=True)) == [9]
+
+ def test_day_over_the_row_cap_is_refused_even_when_forced(self) -> None:
+ with materialized("events", "$lib"), materialized("events", "$lib_version"):
+ self._capture_sdk_events(self.team, [("web", "1.0.0"), ("web", "1.1.0"), ("web", "1.2.0")], NOON_YESTERDAY)
+ flush_persons_and_events()
+
+ with (
+ patch.object(SdkVersionsDailyRollup, "max_rows_per_day", 1),
+ pytest.raises(RollupDayRefused, match="more than 1 rows"),
+ ):
+ run_rollup_day(SdkVersionsDailyRollup, RollupDay(day=YESTERDAY, mode="publish"), force=True)
+
+ assert not SdkVersionsDaily.objects.for_team(self.team.id).exists()
+
+ def test_sdk_fill_keeps_known_sdks_with_storable_versions(self) -> None:
+ events = [
+ ("web", "1.200.0"),
+ ("web", "1.200.0"),
+ ("posthog-java", ""),
+ ("posthog-python", ""),
+ ("unknown-sdk", "1.0.0"),
+ ("posthog-python", "9" * 65),
+ ("posthog-node", "4.0.0-beta.1"),
+ ]
+ with materialized("events", "$lib"), materialized("events", "$lib_version"):
+ self._capture_sdk_events(self.team, events, NOON_YESTERDAY)
+ flush_persons_and_events()
+
+ run_rollup_day(SdkVersionsDailyRollup, RollupDay(day=YESTERDAY, mode="publish"))
+
+ rows = set(SdkVersionsDaily.objects.for_team(self.team.id).values_list("lib", "lib_version", "event_count"))
+ assert rows == {("web", "1.200.0", 2), ("posthog-java", "", 1), ("posthog-node", "4.0.0-beta.1", 1)}
+
+ @patch.object(store, "DELETE_BATCH_SIZE", 2)
+ def test_batched_deletes_remove_every_matching_row(self) -> None:
+ other_team_id = self.team.id + 1
+ SdkVersionsDaily.objects.unscoped().bulk_create(
+ [_sdk_row(self.team.id, YESTERDAY - dt.timedelta(days=offset)) for offset in range(15)]
+ + [_sdk_row(other_team_id, YESTERDAY)]
+ )
+
+ assert delete_days_before(SdkVersionsDailyRollup, YESTERDAY - dt.timedelta(days=9)) == 5
+ assert SdkVersionsDaily.objects.for_team(self.team.id).count() == 10
+
+ assert delete_rollup_rows_for_teams(SdkVersionsDaily, [self.team.id]) == 10
+ assert SdkVersionsDaily.objects.unscoped().count() == 1
+
+ def test_scheduled_job_publishes_the_good_days_and_cleans_up_before_failing_on_a_refused_day(self) -> None:
+ expired_day = YESTERDAY - dt.timedelta(days=20)
+ deleted_team_id = self.team.id + 1000
+ SdkVersionsDaily.objects.unscoped().bulk_create(
+ [_sdk_row(self.team.id, expired_day), _sdk_row(deleted_team_id, YESTERDAY - dt.timedelta(days=4))]
+ )
+ with materialized("events", "$lib"), materialized("events", "$lib_version"):
+ self._capture_sdk_events(self.team, [("web", "1.0.0")], NOON_YESTERDAY)
+ self._capture_sdk_events(self.team, [("web", "1.0.0")] * 2, NOON_YESTERDAY - dt.timedelta(days=3))
+ flush_persons_and_events()
+
+ result = sdk_versions_daily_rollup.job.execute_in_process(raise_on_error=False)
+
+ assert not result.success
+ failure = result.failure_data_for_node(f"fill_{SdkVersionsDailyRollup.table()}")
+ assert failure is not None and "Launch the job with `days`" in str(failure.error)
+
+ published = set(SdkVersionsDaily.objects.unscoped().values_list("team_id", "day", "event_count"))
+ assert published == {(self.team.id, YESTERDAY, 1), (self.team.id, YESTERDAY - dt.timedelta(days=3), 2)}
+
+ def test_requested_day_with_force_publishes_under_the_median(self) -> None:
+ SdkVersionsDaily.objects.unscoped().bulk_create(
+ [
+ _sdk_row(team_id, YESTERDAY - dt.timedelta(days=offset))
+ for offset in range(2, 6)
+ for team_id in range(self.team.id, self.team.id + 10)
+ ]
+ )
+ with materialized("events", "$lib"), materialized("events", "$lib_version"):
+ self._capture_sdk_events(self.team, [("web", "1.0.0")], NOON_YESTERDAY)
+ flush_persons_and_events()
+ op = f"fill_{SdkVersionsDailyRollup.table()}"
+ requested = DailyRollupConfig(days=[YESTERDAY.isoformat()])
+
+ refused = sdk_versions_daily_rollup.job.execute_in_process(
+ run_config=dagster.RunConfig(ops={op: requested}), raise_on_error=False
+ )
+ forced = sdk_versions_daily_rollup.job.execute_in_process(
+ run_config=dagster.RunConfig(ops={op: DailyRollupConfig(days=requested.days, force=True)})
+ )
+
+ assert not refused.success
+ assert forced.success
+ assert list(
+ SdkVersionsDaily.objects.for_team(self.team.id).filter(day=YESTERDAY).values_list("event_count", flat=True)
+ ) == [1]
diff --git a/posthog/temporal/health_checks/rollups/validation.py b/posthog/temporal/health_checks/rollups/validation.py
new file mode 100644
index 000000000000..0470e13015c2
--- /dev/null
+++ b/posthog/temporal/health_checks/rollups/validation.py
@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import statistics
+from collections.abc import Sequence
+
+from posthog.temporal.health_checks.rollups.fill import DayFill
+from posthog.temporal.health_checks.rollups.framework import DailyRollup
+from posthog.temporal.health_checks.rollups.store import DayStats
+
+MIN_FRACTION_OF_MEDIAN = 0.5
+MIN_HISTORY_DAYS_FOR_MEDIAN = 3
+D2_CHANGE_THRESHOLD = 0.01
+
+
+def refusal_reason(rollup: type[DailyRollup], fill: DayFill, history: Sequence[DayStats], *, force: bool) -> str | None:
+ """Why the filled day must not be published, or None when it may be.
+
+ `force` lifts only the median floor. An empty day is always refused, because an empty day is what
+ a ClickHouse outage looks like. `fill_day` refuses a day over the row cap before this runs.
+ """
+ if fill.row_count == 0:
+ return "the day has no rows"
+ if force or len(history) < MIN_HISTORY_DAYS_FOR_MEDIAN:
+ return None
+ median_rows = statistics.median(stats.row_count for stats in history)
+ if fill.row_count < median_rows * MIN_FRACTION_OF_MEDIAN:
+ return (
+ f"the day has {fill.row_count:,} rows, under {MIN_FRACTION_OF_MEDIAN:.0%} of the "
+ f"{len(history)}-day median of {median_rows:,.0f}"
+ )
+ return None
+
+
+def _relative_change(published: int, filled: int) -> float:
+ if published == 0:
+ return 0.0 if filled == 0 else 1.0
+ return abs(filled - published) / published
+
+
+def day_moved(published: DayStats | None, fill: DayFill) -> bool:
+ """Whether a refill differs enough from the published day to replace it."""
+ if published is None:
+ return True
+ return (
+ _relative_change(published.row_count, fill.row_count) > D2_CHANGE_THRESHOLD
+ or _relative_change(published.event_count, fill.event_count) > D2_CHANGE_THRESHOLD
+ )
diff --git a/products/growth/backend/facade/api.py b/products/growth/backend/facade/api.py
index 9f169d022ab9..e929bd562e13 100644
--- a/products/growth/backend/facade/api.py
+++ b/products/growth/backend/facade/api.py
@@ -3,9 +3,10 @@ from typing import Literal
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
from posthog.models.instance_setting import get_instance_setting
+from posthog.temporal.health_checks.rollups.store import delete_rollup_rows_for_teams
from posthog.utils import get_instance_region
-from products.growth.backend.models import OrganizationEnrichment
+from products.growth.backend.models import OrganizationEnrichment, SdkVersionsDaily
from products.growth.backend.temporal.signup_enrichment.trigger import dispatch_wizard_stamp_rescore
WizardStampRescoreSkipReason = Literal["disabled", "no_enrichment_record", "dispatch_backlog_full", "dispatch_failed"]
@@ -36,3 +37,8 @@ def request_wizard_stamp_rescore(organization_id: str) -> WizardStampRescoreOutc
if failure is not None:
return WizardStampRescoreOutcome(queued=False, reason=failure)
return WizardStampRescoreOutcome(queued=True)
+
+
+def delete_sdk_version_rollups_for_teams(team_ids: list[int]) -> None:
+ """Remove the SDK version rollup rows of the given environments. Called from team deletion."""
+ delete_rollup_rows_for_teams(SdkVersionsDaily, team_ids)
diff --git a/products/growth/backend/migrations/0011_daily_health_rollups.py b/products/growth/backend/migrations/0011_daily_health_rollups.py
new file mode 100644
index 000000000000..98d122b068a8
--- /dev/null
+++ b/products/growth/backend/migrations/0011_daily_health_rollups.py
@@ -0,0 +1,38 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("growth", "0010_icp_scoring_rules"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="SdkVersionsDaily",
+ fields=[
+ (
+ "pk",
+ models.CompositePrimaryKey(
+ "team_id",
+ "day",
+ "lib",
+ "lib_version",
+ blank=True,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ ("team_id", models.BigIntegerField()),
+ ("day", models.DateField()),
+ ("lib", models.CharField(max_length=64)),
+ ("lib_version", models.CharField(max_length=64)),
+ ("event_count", models.BigIntegerField()),
+ ("max_timestamp", models.DateTimeField()),
+ ],
+ options={
+ "db_table": "growth_sdk_versions_daily",
+ "indexes": [models.Index(fields=["day"], name="growth_sdk_ver_daily_day_idx")],
+ },
+ ),
+ ]
diff --git a/products/growth/backend/migrations/0012_daily_health_rollups_autovacuum.py b/products/growth/backend/migrations/0012_daily_health_rollups_autovacuum.py
new file mode 100644
index 000000000000..ebdb9491f539
--- /dev/null
+++ b/products/growth/backend/migrations/0012_daily_health_rollups_autovacuum.py
@@ -0,0 +1,22 @@
+from django.db import migrations
+
+SCALE_FACTOR = 0.02
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("growth", "0011_daily_health_rollups"),
+ ]
+
+ operations = [
+ migrations.RunSQL(
+ sql=[
+ "SET LOCAL lock_timeout = '5s'",
+ f"ALTER TABLE growth_sdk_versions_daily SET (autovacuum_vacuum_scale_factor = {SCALE_FACTOR}, autovacuum_analyze_scale_factor = {SCALE_FACTOR})",
+ ],
+ reverse_sql=[
+ "SET LOCAL lock_timeout = '5s'",
+ "ALTER TABLE growth_sdk_versions_daily RESET (autovacuum_vacuum_scale_factor, autovacuum_analyze_scale_factor)",
+ ],
+ ),
+ ]
diff --git a/products/growth/backend/migrations/max_migration.txt b/products/growth/backend/migrations/max_migration.txt
index b556a473aa27..6be12d4b705b 100644
--- a/products/growth/backend/migrations/max_migration.txt
+++ b/products/growth/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0010_icp_scoring_rules
+0012_daily_health_rollups_autovacuum
diff --git a/products/growth/backend/models.py b/products/growth/backend/models.py
index 3c4fedac7f3c..a5d62684956c 100644
--- a/products/growth/backend/models.py
+++ b/products/growth/backend/models.py
@@ -6,6 +6,7 @@ from django.core.exceptions import ValidationError
from django.db import models, transaction
from django.db.models import Q
+from posthog.models.scoping.manager import EnvironmentScopedManager
from posthog.models.utils import UpdatedMetaFields, UUIDModel
from products.growth.backend.enrichment.icp_lists import clear_lists_cache
@@ -342,3 +343,26 @@ class EnrichmentSignupSnapshot(UUIDModel):
related_name="enrichment_signup_snapshot",
)
created_at = models.DateTimeField(auto_now_add=True)
+
+
+class SdkVersionsDaily(models.Model):
+ """Events per SDK version for one environment and one UTC day.
+
+ The daily rollup job replaces a whole day at a time, so readers sum days instead of
+ scanning raw events. `team_id` is the environment id from events, and has no foreign key
+ because team deletion removes the rows explicitly.
+ """
+
+ pk = models.CompositePrimaryKey("team_id", "day", "lib", "lib_version")
+ team_id = models.BigIntegerField()
+ day = models.DateField()
+ lib = models.CharField(max_length=64)
+ lib_version = models.CharField(max_length=64)
+ event_count = models.BigIntegerField()
+ max_timestamp = models.DateTimeField()
+
+ objects = EnvironmentScopedManager()
+
+ class Meta:
+ db_table = "growth_sdk_versions_daily"
+ indexes = [models.Index(fields=["day"], name="growth_sdk_ver_daily_day_idx")]
diff --git a/products/growth/backend/sdk_version_rollup.py b/products/growth/backend/sdk_version_rollup.py
new file mode 100644
index 000000000000..c1b80936ede9
--- /dev/null
+++ b/products/growth/backend/sdk_version_rollup.py
@@ -0,0 +1,50 @@
+from collections.abc import Mapping
+from typing import Any
+
+from posthog.clickhouse.query_tagging import Product
+from posthog.temporal.health_checks.rollups.framework import DailyRollup
+
+from products.growth.backend.constants import SDK_TYPES
+from products.growth.backend.models import SdkVersionsDaily
+
+MAX_VERSIONS_PER_TEAM_LIB_DAY = 100
+
+SDK_VERSIONS_FILL_SQL = """
+SELECT
+ team_id,
+ `mat_$lib` AS lib,
+ `mat_$lib_version` AS lib_version,
+ count() AS event_count,
+ max(timestamp) AS max_timestamp
+FROM events
+WHERE team_id >= %(team_id_start)s
+ AND team_id < %(team_id_end)s
+ AND timestamp >= %(day_start)s
+ AND timestamp < %(day_end)s
+ AND `mat_$lib` IN %(sdk_types)s
+ AND (`mat_$lib` = 'posthog-java' OR `mat_$lib_version` != '')
+ AND length(`mat_$lib_version`) <= %(max_version_length)s
+ AND position(`mat_$lib_version`, char(0)) = 0
+GROUP BY team_id, lib, lib_version
+ORDER BY team_id, lib, event_count DESC
+LIMIT %(max_versions)s BY team_id, lib
+"""
+
+
+class SdkVersionsDailyRollup(DailyRollup):
+ model = SdkVersionsDaily
+ product = Product.SDK_HEALTH
+ columns = ("team_id", "day", "lib", "lib_version", "event_count", "max_timestamp")
+ retention_days = 10
+
+ @classmethod
+ def fill_sql(cls) -> str:
+ return SDK_VERSIONS_FILL_SQL
+
+ @classmethod
+ def fill_params(cls) -> Mapping[str, Any]:
+ return {
+ "sdk_types": list(SDK_TYPES),
+ "max_version_length": SdkVersionsDaily._meta.get_field("lib_version").max_length,
+ "max_versions": MAX_VERSIONS_PER_TEAM_LIB_DAY,
+ }
diff --git a/products/growth/dags/sdk_versions_rollup.py b/products/growth/dags/sdk_versions_rollup.py
new file mode 100644
index 000000000000..6514f73f904f
--- /dev/null
+++ b/products/growth/dags/sdk_versions_rollup.py
@@ -0,0 +1,6 @@
+from posthog.dags.common import JobOwners
+from posthog.dags.daily_rollups import build_daily_rollup_definitions
+
+from products.growth.backend.sdk_version_rollup import SdkVersionsDailyRollup
+
+sdk_versions_daily_rollup = build_daily_rollup_definitions(SdkVersionsDailyRollup, owner=JobOwners.TEAM_GROWTH)