:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(growth): fill a daily sdk versions rollup in dagster (#126050)
Rafael Audibert6ca51906d16d
diff --git a/.github/workflows/ci-dagster.yml b/.github/workflows/ci-dagster.ymlindex e7040bc8ff48..787b4cd2a826 100644--- a/.github/workflows/ci-dagster.yml+++ b/.github/workflows/ci-dagster.yml@@ -158,6 +158,7 @@ jobs:# --- DAG code itself ---- 'posthog/dags/**'- 'products/*/dags/**'+ - 'posthog/temporal/health_checks/rollups/**' # daily rollup fills run in Dagster# --- posthog submodules that DAGs actually import ---# (verified by tracing imports of all 148 dagster Python files)- 'posthog/clickhouse/**'@@ -282,6 +283,7 @@ jobs:dagster_direct:- 'posthog/dags/**'- 'products/*/dags/**'+ - 'posthog/temporal/health_checks/rollups/**' # daily rollup fills run in Dagster- .github/workflows/ci-dagster.yml- .github/actions/setup-uv/**- tools/dagster_test_selection.pydiff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yamlindex 7efcccb62f9f..3670124d8d35 100644--- a/.semgrep/rules/security/idor-team-scoped-models.yaml+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml@@ -389,6 +389,7 @@ rules:|HeatmapCaptureRequest|HeatmapScreenshotHistory|ScheduledChange+ |SdkVersionsDaily|SessionGroupSummary|SessionRecording|SessionRecordingPlaylist@@ -881,6 +882,7 @@ rules:|HeatmapCaptureRequest|HeatmapScreenshotHistory|ScheduledChange+ |SdkVersionsDaily|SessionGroupSummary|SessionRecording|SessionRecordingPlaylist@@ -1236,6 +1238,7 @@ rules:|SandboxEnvironment|SavedHeatmap|ScoreDefinition+ |SdkVersionsDaily|SessionRecording|SessionRecordingEvent|SessionRecordingPlaylistdiff --git a/posthog/api/test/test_team_deletion.py b/posthog/api/test/test_team_deletion.pyindex 1d79a8d36910..144e1f71e9d4 100644--- a/posthog/api/test/test_team_deletion.py+++ b/posthog/api/test/test_team_deletion.py@@ -1,4 +1,5 @@import json+import datetime as dtimport time_machinefrom posthog.test.base import NonAtomicBaseTest@@ -18,6 +19,7 @@ from posthog.temporal.tests.delete_teams.inline import execute_deletion_workflowfrom products.batch_exports.backend.facade.temporal import ACTIVITIES, WORKFLOWSfrom products.early_access_features.backend.models import EarlyAccessFeature+from products.growth.backend.models import SdkVersionsDailyclass TestTeamDeletionSideEffects(NonAtomicBaseTest):@@ -170,11 +172,22 @@ class TestTeamDeletionSideEffects(NonAtomicBaseTest):EarlyAccessFeature.objects.create(team=team, name="Test flag", description="A fancy new flag.", stage="beta", feature_flag=flag)+ for team_id in (team.id, self.team.id):+ SdkVersionsDaily.objects.unscoped().create(+ team_id=team_id,+ day=dt.date(2026, 10, 22),+ lib="web",+ lib_version="1.0.0",+ event_count=1,+ max_timestamp=dt.datetime(2026, 10, 22, tzinfo=dt.UTC),+ )with execute_deletion_workflows_inline():response = self.client.delete(f"/api/environments/{team.id}")self.assertEqual(response.status_code, 204)self.assertFalse(Team.objects.filter(id=team.id).exists())+ self.assertFalse(SdkVersionsDaily.objects.for_team(team.id).exists())+ self.assertTrue(SdkVersionsDaily.objects.for_team(self.team.id).exists())# Verify personhog RPCs were called for persons-DB cleanupfake = get_active_fake()diff --git a/posthog/dags/daily_rollups.py b/posthog/dags/daily_rollups.pynew file mode 100644index 000000000000..6d2ec21068aa--- /dev/null+++ b/posthog/dags/daily_rollups.py@@ -0,0 +1,105 @@+import datetime as dt++import dagster+import pydantic++from posthog.dags.common import JobOwners+from posthog.dataclasses import frozen+from posthog.temporal.health_checks.rollups.framework import DailyRollup+from posthog.temporal.health_checks.rollups.runner import (+ RollupDayRefused,+ clean_up,+ plan_requested_days,+ plan_scheduled_days,+ published_days_in_retention,+ run_rollup_day,+)++DAILY_ROLLUP_CRON = "0 1 * * *"+# A first run fills every day that retention keeps, one ClickHouse day scan after another.+MAX_RUNTIME_SECONDS = 6 * 60 * 60+# A retry plans again from the published days, so it refills the refresh day that a transient error failed.+# The next scheduled run does not, because it refreshes a later day.+FILL_RETRY_POLICY = dagster.RetryPolicy(max_retries=2, delay=15 * 60)+++class DailyRollupConfig(dagster.Config):+ days: list[str] = pydantic.Field(+ default_factory=list,+ description=(+ "UTC days to fill, as YYYY-MM-DD. Leave empty to publish yesterday, refresh the day before, "+ "and fill every missing day that retention keeps."+ ),+ )+ force: bool = pydantic.Field(+ default=False,+ description="Publish a day even when it has fewer than half the rows of the recent median.",+ )+++@frozen+class DailyRollupDefinitions:+ job: dagster.JobDefinition+ schedule: dagster.ScheduleDefinition+++def _today() -> dt.date:+ return dt.datetime.now(dt.UTC).date()+++def build_daily_rollup_definitions(rollup: type[DailyRollup], *, owner: JobOwners) -> DailyRollupDefinitions:+ """A daily job that fills one rollup table from ClickHouse, and its schedule.++ Every planned day runs even when an earlier one fails, and clean-up runs last. The op then fails+ if any day failed, which leaves that day as it was. Dagster retries it unless every failed day was+ refused, because a refused day fails the same way again.+ """+ table = rollup.table()++ @dagster.op(name=f"fill_{table}", retry_policy=FILL_RETRY_POLICY)+ def fill_rollup(context: dagster.OpExecutionContext, config: DailyRollupConfig) -> None:+ today = _today()+ if config.days:+ plan = plan_requested_days([dt.date.fromisoformat(day) for day in config.days], today=today)+ else:+ plan = plan_scheduled_days(rollup, today=today, published=published_days_in_retention(rollup, today=today))+ context.log.info(f"Filling {table} for {', '.join(f'{step.day} ({step.mode})' for step in plan)}")++ failures: list[str] = []+ only_refusals = True+ for step in plan:+ try:+ result = run_rollup_day(rollup, step, force=config.force)+ except Exception as error:+ context.log.exception(f"Filling {table} for {step.day} failed")+ failures.append(f"{step.day}: {error}")+ only_refusals = only_refusals and isinstance(error, RollupDayRefused)+ continue+ context.log.info(f"{table} {result.day}: {result.outcome}, {result.rows:,} rows, {result.events:,} events")++ deleted_rows = clean_up(rollup, today=today)+ context.log.info(f"Clean-up removed {deleted_rows:,} rows of {table}")+ if failures:+ raise dagster.Failure(+ description=f"{len(failures)} of {len(plan)} days of {table} failed. "+ "Launch the job with `days` to fill them again, and `force` to publish a refused day. "+ + "; ".join(failures),+ allow_retries=not only_refusals,+ )++ @dagster.job(+ name=f"{table}_job",+ description=f"Fills the {table} daily rollup from ClickHouse events.",+ tags={"owner": owner.value, "dagster/max_runtime": MAX_RUNTIME_SECONDS},+ )+ def fill_rollup_job() -> None:+ fill_rollup()++ schedule = dagster.ScheduleDefinition(+ name=f"{table}_schedule",+ job=fill_rollup_job,+ cron_schedule=DAILY_ROLLUP_CRON,+ execution_timezone="UTC",+ default_status=dagster.DefaultScheduleStatus.RUNNING,+ )+ return DailyRollupDefinitions(job=fill_rollup_job, schedule=schedule)diff --git a/posthog/dags/locations/growth.py b/posthog/dags/locations/growth.pyindex 30ab3ec506d6..db9d7c17f52f 100644--- a/posthog/dags/locations/growth.py+++ b/posthog/dags/locations/growth.py@@ -7,6 +7,7 @@ from products.growth.dags import (identity_matching,oauth,product_push_campaigns,+ sdk_versions_rollup,team_production_event_activation,)@@ -18,6 +19,7 @@ jobs = [team_production_event_activation.detect_first_team_production_event_job,product_push_campaigns.product_push_campaigns_job,ai_enrichment.ai_enrichment_job,+ sdk_versions_rollup.sdk_versions_daily_rollup.job,# Manual-run only: growth names the organizations, product, and window per run.custom_product_push_campaigns.custom_product_push_campaigns_job,]@@ -27,6 +29,7 @@ schedules = [team_production_event_activation.detect_first_team_production_event_schedule,product_push_campaigns.product_push_campaigns_schedule,ai_enrichment.ai_enrichment_schedule,+ sdk_versions_rollup.sdk_versions_daily_rollup.schedule,]# Identity matching processes internal PostHog data that only exists on Cloud US (team 2),# so the job is not registered on Cloud EU.diff --git a/posthog/models/team/util.py b/posthog/models/team/util.pyindex 6d3f70ee2d7f..7d16f69f6854 100644--- a/posthog/models/team/util.py+++ b/posthog/models/team/util.py@@ -89,6 +89,7 @@ def _delete_misc_small_tables_for_teams(team_ids: list[int]) -> None:from products.data_modeling.backend.facade.models import Edge, Nodefrom products.early_access_features.backend.models import EarlyAccessFeature+ from products.growth.backend.facade import api as growth_facadefrom products.today.backend.facade import api as today_facadeerror_tracking_fingerprint = apps.get_model("error_tracking", "ErrorTrackingIssueFingerprintV2")@@ -105,6 +106,7 @@ def _delete_misc_small_tables_for_teams(team_ids: list[int]) -> None:_raw_delete_batch(error_tracking_fingerprint.objects.filter(team_id__in=team_ids))# Highlights carry team_id as a plain integer, so nothing cascades to them.today_facade.delete_highlights_for_teams(team_ids)+ growth_facade.delete_sdk_version_rollups_for_teams(team_ids)# FeatureFlagHashKeyOverride references Person, so it must go before persons are deleted._delete_hash_key_overrides_for_teams(team_ids)_delete_llm_evaluations_for_teams(team_ids)diff --git a/posthog/temporal/health_checks/README.md b/posthog/temporal/health_checks/README.mdindex 150b64eb3c0e..24a0836e2216 100644--- a/posthog/temporal/health_checks/README.md+++ b/posthog/temporal/health_checks/README.md@@ -227,6 +227,42 @@ rows = execute_clickhouse_health_team_query(Default ClickHouse settings: `max_execution_time=30`, `max_threads=2`.+## Daily rollups++Checks that would scan days of raw `events` read a daily rollup on Postgres instead.+A Dagster job per rollup reads one UTC day from ClickHouse and replaces that day in a small Postgres table in one transaction.++| Table | Product | Kept | Dagster job |+| --------------------------- | ------- | ------- | ------------------------------- |+| `growth_sdk_versions_daily` | growth | 10 days | `growth_sdk_versions_daily_job` |++### How a day is published++The job runs at 01:00 UTC. It publishes yesterday, and refills the day before only if its rows or events moved by more than 1%, which picks up late events.+It also publishes every day that retention keeps and that has no rows yet.+So the first run after a deploy fills the whole retention window, and a day lost to an outage comes back on the next run.++A filled day is refused, and the published day stays as it was, when:++- it has no rows, which is what a ClickHouse outage looks like+- it has more than 2M rows+- it has fewer than half the rows of the median of the last 7 published days, unless the run is forced++Rows for teams that no longer exist are dropped before writing, and team deletion removes a team's rows.+Every run also deletes the rows of teams that no longer exist, which a fill can write while a team deletion runs.+A refused or failed day does not stop the other days or the clean-up, and the run fails at the end, which alerts the owning team.+Dagster retries a failed run twice, 15 minutes apart, unless every failed day was refused.++To fill a day by hand, launch the job in Dagster with `days: ["2026-10-01", ...]`, and add `force: true` to publish a day the median rule refused.++### Adding a rollup++Subclass `DailyRollup` (`rollups/framework.py`) next to the product's model, with `columns`, `fill_sql()` and `retention_days`.+The fill drops rows whose strings exceed a column's `max_length` or hold a NUL byte, but filter client-controlled values in `fill_sql()` too, so the dropped count stays near zero.+A fill reads every team's events, so read event properties through their materialized columns where they exist.+Build its job and schedule with `build_daily_rollup_definitions` (`posthog/dags/daily_rollups.py`) in the product's `dags/`, and register both in the product's Dagster location.+The model needs a composite primary key over the natural key, an index on `day`, no foreign keys, and `EnvironmentScopedManager`, and the product facade must delete its rows on team deletion.+## ClickHouse kill switchHealth checks are automatically skipped when the ClickHouse kill switch is active (LIGHT or FULL).diff --git a/posthog/temporal/health_checks/rollups/__init__.py b/posthog/temporal/health_checks/rollups/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/posthog/temporal/health_checks/rollups/fill.py b/posthog/temporal/health_checks/rollups/fill.pynew file mode 100644index 000000000000..fb1e56b8976d--- /dev/null+++ b/posthog/temporal/health_checks/rollups/fill.py@@ -0,0 +1,188 @@+from __future__ import annotations++import datetime as dt+import tempfile+from collections.abc import (+ Iterable,+ Iterator,+ Sequence,+ Set as AbstractSet,+)+from typing import IO, Any++import structlog++from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.query_tagging import Feature, tags_context+from posthog.clickhouse.workload import Workload+from posthog.dataclasses import frozen+from posthog.temporal.health_checks.rollups.framework import DailyRollup+from posthog.utils import ensure_utc++logger = structlog.get_logger(__name__)++TEAM_ID_RANGE_COUNT = 16++SPOOL_MAX_BYTES = 64 * 1024 * 1024++FILL_QUERY_SETTINGS = {"max_execution_time": 1800, "max_threads": 8}+++class DayOverRowCap(Exception):+ """The day has more rows than the rollup allows, so nothing of it may be published."""+++@frozen+class TeamIdRange:+ start: int+ end: int+++@frozen(frozen=False)+class DayFill:+ """One filled day, written in COPY text format into `buffer` and ready to publish.++ Close it once published, or once it is refused, to free the spooled file.+ """++ day: dt.date+ row_count: int+ event_count: int+ buffer: IO[bytes]++ def chunks(self, size: int = 1024 * 1024) -> Iterator[bytes]:+ self.buffer.seek(0)+ while chunk := self.buffer.read(size):+ yield chunk++ def close(self) -> None:+ self.buffer.close()+++def team_id_ranges(team_ids: Iterable[int], count: int = TEAM_ID_RANGE_COUNT) -> list[TeamIdRange]:+ """Split the team ids into at most `count` ranges that hold about the same number of teams.++ Team ids are dense at the top, where active teams are, so equal-width ranges would put most+ teams in the last few ranges.+ """+ ids = sorted(set(team_ids))+ if not ids:+ return []+ count = max(1, min(count, len(ids)))+ starts = [ids[(len(ids) * index) // count] for index in range(count)]+ ends = [*starts[1:], ids[-1] + 1]+ return [TeamIdRange(start=start, end=end) for start, end in zip(starts, ends)]+++def _copy_text(value: Any) -> str:+ """Format one value for the Postgres COPY text format."""+ if value is None:+ return "\\N"+ if isinstance(value, bool):+ return "t" if value else "f"+ if isinstance(value, dt.datetime):+ return ensure_utc(value).isoformat()+ if isinstance(value, dt.date):+ return value.isoformat()+ text = str(value)+ if "\x00" in text:+ raise ValueError("COPY text cannot hold a NUL byte")+ return text.replace("\\", "\\\\").replace("\t", "\\t").replace("\n", "\\n").replace("\r", "\\r")+++def _row_columns(rollup: type[DailyRollup]) -> tuple[str, ...]:+ return (rollup.columns[0], *rollup.columns[2:])+++@frozen+class _StringLimit:+ position: int+ max_length: int+++def _string_limits(rollup: type[DailyRollup]) -> list[_StringLimit]:+ """The row position and max_length of every length-limited column."""+ limits = []+ for position, column in enumerate(_row_columns(rollup)):+ max_length = getattr(rollup.model._meta.get_field(column), "max_length", None)+ if max_length is not None:+ limits.append(_StringLimit(position=position, max_length=max_length))+ return limits+++def _storable(row: Sequence[Any], string_limits: list[_StringLimit]) -> bool:+ for limit in string_limits:+ value = row[limit.position]+ if isinstance(value, str) and (len(value) > limit.max_length or "\x00" in value):+ return False+ return True+++def _fetch_rows(rollup: type[DailyRollup], day: dt.date, team_ids: AbstractSet[int]) -> Iterator[Sequence[Any]]:+ params = {+ **rollup.fill_params(),+ "day_start": f"{day.isoformat()} 00:00:00",+ "day_end": f"{(day + dt.timedelta(days=1)).isoformat()} 00:00:00",+ }+ sql = rollup.fill_sql()+ # ClickHouse stops a range at the cap, so one oversized range cannot fill the worker's memory.+ settings = {+ **FILL_QUERY_SETTINGS,+ "max_result_rows": rollup.max_rows_per_day + 1,+ "result_overflow_mode": "break",+ }+ for team_range in team_id_ranges(team_ids):+ with tags_context(product=rollup.product, feature=Feature.HEALTH_CHECK, name=f"rollup_{rollup.table()}"):+ rows = sync_execute(+ sql,+ {**params, "team_id_start": team_range.start, "team_id_end": team_range.end},+ settings=settings,+ workload=Workload.OFFLINE,+ )+ if len(rows) > rollup.max_rows_per_day:+ raise DayOverRowCap(f"one team id range has more than {rollup.max_rows_per_day:,} rows")+ yield from rows+++def fill_day(rollup: type[DailyRollup], day: dt.date, live_team_ids: AbstractSet[int]) -> DayFill:+ """Read one UTC day from ClickHouse, one team id range at a time, into a COPY buffer.++ Rows for teams that no longer exist and rows that do not fit the table are dropped here,+ before anything reaches Postgres. A day over the row cap raises DayOverRowCap.+ """+ event_count_position = _row_columns(rollup).index("event_count")+ string_limits = _string_limits(rollup)++ buffer: IO[bytes] = tempfile.SpooledTemporaryFile(max_size=SPOOL_MAX_BYTES, mode="w+b")+ row_count = 0+ event_count = 0+ dropped_unknown_team = 0+ dropped_unstorable = 0+ try:+ for row in _fetch_rows(rollup, day, live_team_ids):+ if row[0] not in live_team_ids:+ dropped_unknown_team += 1+ continue+ if not _storable(row, string_limits):+ dropped_unstorable += 1+ continue+ values = (row[0], day, *row[1:])+ buffer.write(("\t".join(_copy_text(value) for value in values) + "\n").encode())+ row_count += 1+ event_count += int(row[event_count_position])+ if row_count > rollup.max_rows_per_day:+ raise DayOverRowCap(f"the day has more than {rollup.max_rows_per_day:,} rows")+ except BaseException:+ buffer.close()+ raise++ logger.info(+ "daily rollup filled",+ table=rollup.table(),+ day=day.isoformat(),+ rows=row_count,+ events=event_count,+ dropped_unknown_team=dropped_unknown_team,+ dropped_unstorable=dropped_unstorable,+ )+ return DayFill(day=day, row_count=row_count, event_count=event_count, buffer=buffer)diff --git a/posthog/temporal/health_checks/rollups/framework.py b/posthog/temporal/health_checks/rollups/framework.pynew file mode 100644index 000000000000..5443f3849e5a--- /dev/null+++ b/posthog/temporal/health_checks/rollups/framework.py@@ -0,0 +1,51 @@+from __future__ import annotations++from collections.abc import Mapping+from typing import TYPE_CHECKING, Any, ClassVar++from posthog.clickhouse.query_tagging import Product++if TYPE_CHECKING:+ from django.db import models++DEFAULT_MAX_ROWS_PER_DAY = 2_000_000+++class DailyRollup:+ """One Postgres table that holds one row per natural key per UTC day, filled from ClickHouse.++ A Dagster job built with `build_daily_rollup_job` fills a day by running `fill_sql()` once per+ team id range, and replaces the day in Postgres in one transaction.++ `fill_sql()` returns the table's columns in `columns` order, without `day`, which the framework+ adds. It must filter on `%(team_id_start)s <= team_id < %(team_id_end)s` and on+ `%(day_start)s <= timestamp < %(day_end)s`. `columns` must include `event_count`.+ """++ model: ClassVar[type[models.Model]]+ product: ClassVar[Product | None] = None+ columns: ClassVar[tuple[str, ...]]+ retention_days: ClassVar[int]+ max_rows_per_day: ClassVar[int] = DEFAULT_MAX_ROWS_PER_DAY++ def __init_subclass__(cls, **kwargs: object) -> None:+ super().__init_subclass__(**kwargs)+ if "model" not in cls.__dict__:+ return+ if cls.columns[:2] != ("team_id", "day"):+ raise ValueError(f"{cls.__name__}.columns must start with team_id and day")+ if "event_count" not in cls.columns:+ raise ValueError(f"{cls.__name__}.columns must include event_count")++ @classmethod+ def table(cls) -> str:+ return cls.model._meta.db_table++ @classmethod+ def fill_sql(cls) -> str:+ raise NotImplementedError++ @classmethod+ def fill_params(cls) -> Mapping[str, Any]:+ """Extra ClickHouse parameters for `fill_sql()`, such as an allowlist."""+ return {}diff --git a/posthog/temporal/health_checks/rollups/freshness.py b/posthog/temporal/health_checks/rollups/freshness.pynew file mode 100644index 000000000000..9f31d0b7fd22--- /dev/null+++ b/posthog/temporal/health_checks/rollups/freshness.py@@ -0,0 +1,117 @@+from __future__ import annotations++import datetime as dt+from collections.abc import Collection+from typing import Any++from django.core.cache import cache+from django.db import connections, models, router++import structlog++from posthog.dataclasses import frozen++logger = structlog.get_logger(__name__)++PUBLISHED_DAYS_CACHE_SECONDS = 5 * 60+MAX_LATEST_DAY_AGE_DAYS = 2+MAX_PUBLISHED_DAYS_LISTED = 64+++@frozen+class RollupWindow:+ start: dt.date+ end: dt.date++ def __post_init__(self) -> None:+ if self.start > self.end:+ raise ValueError(f"RollupWindow start {self.start} is after end {self.end}")++ @property+ def live_since(self) -> dt.datetime:+ """The UTC instant after the window, from which live queries cover the gap to now."""+ return dt.datetime.combine(self.end + dt.timedelta(days=1), dt.time.min, tzinfo=dt.UTC)+++def window_rows(model: type[models.Model], team_ids: Collection[int], window: RollupWindow) -> models.QuerySet[Any]:+ """The rollup rows of these environments inside the window. Name the model in READ_REPLICA_OPT_IN to read the replica."""+ return model.objects.unscoped().filter( # type: ignore[attr-defined]+ team_id__in=team_ids, day__range=(window.start, window.end)+ )+++def _published_days_cache_key(table: str) -> str:+ return f"daily_rollup:published_days:{table}"+++def _query_published_days(table: str, using: str) -> list[dt.date]:+ query = f"""+ WITH RECURSIVE published(day) AS (+ (SELECT day FROM {table} ORDER BY day DESC LIMIT 1)+ UNION ALL+ SELECT (SELECT t.day FROM {table} t WHERE t.day < published.day ORDER BY t.day DESC LIMIT 1)+ FROM published+ WHERE published.day IS NOT NULL+ )+ SELECT day FROM published WHERE day IS NOT NULL LIMIT %s+ """+ with connections[using].cursor() as cursor:+ cursor.execute(query, [MAX_PUBLISHED_DAYS_LISTED])+ return [row[0] for row in cursor.fetchall()]+++def published_days(model: type[models.Model]) -> list[dt.date]:+ """Published days of a rollup table, newest first, cached for a few minutes."""+ table = model._meta.db_table+ cache_key = _published_days_cache_key(table)+ cached = cache.get(cache_key)+ if cached is not None:+ return [dt.date.fromisoformat(day) for day in cached]+ days = _query_published_days(table, router.db_for_read(model))+ cache.set(cache_key, [day.isoformat() for day in days], PUBLISHED_DAYS_CACHE_SECONDS)+ return days+++def invalidate_published_days(model: type[models.Model]) -> None:+ cache.delete(_published_days_cache_key(model._meta.db_table))+++def window_from_published_days(+ published: list[dt.date], days: int, *, today: dt.date, table: str = ""+) -> RollupWindow | None:+ """The `days`-day window ending at the newest published day, or None when readers must not use it.++ None when nothing is published, when the newest day is too old, or when any day of the window+ is missing. A reader that sums a window with a gap would undercount, and a check would read+ the undercount as missing events.+ """+ if days <= 0:+ raise ValueError(f"days must be > 0, got {days}")+ if not published:+ logger.warning("daily rollup has no published days", table=table)+ return None+ latest = max(published)+ if latest < today - dt.timedelta(days=MAX_LATEST_DAY_AGE_DAYS):+ logger.warning("daily rollup is stale", table=table, latest_day=latest.isoformat(), today=today.isoformat())+ return None+ window = RollupWindow(start=latest - dt.timedelta(days=days - 1), end=latest)+ published_set = set(published)+ missing = [+ day for day in (window.start + dt.timedelta(days=offset) for offset in range(days)) if day not in published_set+ ]+ if missing:+ logger.warning(+ "daily rollup window has missing days",+ table=table,+ window_days=days,+ missing_days=[day.isoformat() for day in missing[:10]],+ missing_count=len(missing),+ )+ return None+ return window+++def rollup_window(model: type[models.Model], days: int) -> RollupWindow | None:+ return window_from_published_days(+ published_days(model), days, today=dt.datetime.now(dt.UTC).date(), table=model._meta.db_table+ )diff --git a/posthog/temporal/health_checks/rollups/runner.py b/posthog/temporal/health_checks/rollups/runner.pynew file mode 100644index 000000000000..4ae4d667f45e--- /dev/null+++ b/posthog/temporal/health_checks/rollups/runner.py@@ -0,0 +1,122 @@+from __future__ import annotations++import datetime as dt+from collections.abc import Collection+from typing import Literal++import structlog++from posthog.dataclasses import frozen+from posthog.models.team import Team+from posthog.temporal.health_checks.rollups.fill import DayOverRowCap, fill_day+from posthog.temporal.health_checks.rollups.framework import DailyRollup+from posthog.temporal.health_checks.rollups.freshness import invalidate_published_days+from posthog.temporal.health_checks.rollups.store import (+ day_stats,+ delete_days_before,+ delete_rows_of_deleted_teams,+ publish_day,+)+from posthog.temporal.health_checks.rollups.validation import day_moved, refusal_reason++logger = structlog.get_logger(__name__)++MEDIAN_HISTORY_DAYS = 7++RollupDayMode = Literal["publish", "refresh"]+RollupDayOutcome = Literal["published", "unchanged"]+++@frozen+class RollupDay:+ day: dt.date+ mode: RollupDayMode+++@frozen+class RollupDayResult:+ day: dt.date+ outcome: RollupDayOutcome+ rows: int+ events: int+++class RollupDayRefused(Exception):+ """The filled day looks wrong, so the published day stays as it was."""+++def plan_scheduled_days(+ rollup: type[DailyRollup], *, today: dt.date, published: Collection[dt.date]+) -> list[RollupDay]:+ """The days one scheduled run fills, newest first.++ It publishes yesterday and refreshes the day before, to pick up late events. It also publishes+ every missing day that retention keeps, so the first run after a deploy and the run after an+ outage fill the gap with no manual backfill.+ """+ yesterday = today - dt.timedelta(days=1)+ plan = [RollupDay(day=yesterday, mode="publish"), RollupDay(day=yesterday - dt.timedelta(days=1), mode="refresh")]+ oldest_kept = today - dt.timedelta(days=rollup.retention_days)+ day = yesterday - dt.timedelta(days=2)+ while day >= oldest_kept:+ if day not in published:+ plan.append(RollupDay(day=day, mode="publish"))+ day -= dt.timedelta(days=1)+ return plan+++def plan_requested_days(days: Collection[dt.date], *, today: dt.date) -> list[RollupDay]:+ future = sorted(day for day in days if day >= today)+ if future:+ raise ValueError(f"Cannot fill a day that has not ended: {', '.join(day.isoformat() for day in future)}")+ return [RollupDay(day=day, mode="publish") for day in sorted(set(days))]+++def published_days_in_retention(rollup: type[DailyRollup], *, today: dt.date) -> set[dt.date]:+ kept = [today - dt.timedelta(days=offset) for offset in range(1, rollup.retention_days + 1)]+ return set(day_stats(rollup, kept))+++def _live_team_ids() -> set[int]:+ return set(Team.objects.values_list("id", flat=True))+++def run_rollup_day(rollup: type[DailyRollup], step: RollupDay, *, force: bool = False) -> RollupDayResult:+ """Fill one day of one rollup and publish it unless it is refused or unchanged.++ Every slow part, the ClickHouse reads, runs before the Postgres transaction opens.+ """+ try:+ fill = fill_day(rollup, step.day, _live_team_ids())+ except DayOverRowCap as error:+ logger.warning("daily rollup day refused", table=rollup.table(), day=step.day.isoformat(), reason=str(error))+ raise RollupDayRefused(f"Refused to publish {rollup.table()} for {step.day.isoformat()}: {error}") from error+ try:+ if step.mode == "refresh" and not day_moved(day_stats(rollup, [step.day]).get(step.day), fill):+ logger.info(+ "daily rollup day unchanged", table=rollup.table(), day=step.day.isoformat(), rows=fill.row_count+ )+ return RollupDayResult(day=step.day, outcome="unchanged", rows=fill.row_count, events=fill.event_count)++ history_days = [step.day - dt.timedelta(days=offset) for offset in range(1, MEDIAN_HISTORY_DAYS + 1)]+ stats = day_stats(rollup, history_days)+ history = [stats[history_day] for history_day in history_days if history_day in stats]+ reason = refusal_reason(rollup, fill, history, force=force)+ if reason is not None:+ logger.warning("daily rollup day refused", table=rollup.table(), day=step.day.isoformat(), reason=reason)+ raise RollupDayRefused(f"Refused to publish {rollup.table()} for {step.day.isoformat()}: {reason}")++ publish_day(rollup, fill)+ invalidate_published_days(rollup.model)+ return RollupDayResult(day=step.day, outcome="published", rows=fill.row_count, events=fill.event_count)+ finally:+ fill.close()+++def clean_up(rollup: type[DailyRollup], *, today: dt.date) -> int:+ """Delete the days retention no longer keeps and the rows of deleted environments."""+ deleted = delete_days_before(rollup, today - dt.timedelta(days=rollup.retention_days))+ deleted += delete_rows_of_deleted_teams(rollup)+ if deleted:+ invalidate_published_days(rollup.model)+ return deleteddiff --git a/posthog/temporal/health_checks/rollups/store.py b/posthog/temporal/health_checks/rollups/store.pynew file mode 100644index 000000000000..ae7595581d47--- /dev/null+++ b/posthog/temporal/health_checks/rollups/store.py@@ -0,0 +1,118 @@+from __future__ import annotations++import datetime as dt+from collections.abc import Sequence++from django.db import connections, models, transaction+from django.db.models import Exists, OuterRef++import structlog++from posthog.dataclasses import frozen+from posthog.models.team import Team+from posthog.temporal.health_checks.rollups.fill import DayFill+from posthog.temporal.health_checks.rollups.framework import DailyRollup++logger = structlog.get_logger(__name__)++WRITE_DATABASE = "default"+DELETE_BATCH_SIZE = 10_000++PUBLISH_STATEMENT_TIMEOUT = "10min"+PUBLISH_LOCK_TIMEOUT = "30s"+++@frozen+class DayStats:+ row_count: int+ event_count: int+++def _advisory_lock_key(rollup: type[DailyRollup]) -> str:+ return f"daily_rollup:{rollup.table()}"+++def publish_day(rollup: type[DailyRollup], fill: DayFill) -> None:+ """Replace one day of the rollup with the filled rows, in one transaction.++ Readers on the replica see the old day or the new one, never a mix. The advisory lock is+ transaction-scoped, so it works behind PgBouncer and releases on commit or rollback.+ """+ table = rollup.table()+ column_list = ", ".join(rollup.columns)+ delete_sql = f"DELETE FROM {table} WHERE day = %s"+ copy_sql = f"COPY {table} ({column_list}) FROM STDIN"++ with transaction.atomic(using=WRITE_DATABASE), connections[WRITE_DATABASE].cursor() as cursor:+ cursor.execute(f"SET LOCAL statement_timeout = '{PUBLISH_STATEMENT_TIMEOUT}'")+ cursor.execute(f"SET LOCAL lock_timeout = '{PUBLISH_LOCK_TIMEOUT}'")+ cursor.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", [_advisory_lock_key(rollup)])+ cursor.execute(delete_sql, [fill.day])+ replaced_rows = cursor.rowcount+ with cursor.cursor.copy(copy_sql) as copy:+ for chunk in fill.chunks():+ copy.write(chunk)++ logger.info(+ "daily rollup published",+ table=table,+ day=fill.day.isoformat(),+ rows=fill.row_count,+ events=fill.event_count,+ replaced_rows=replaced_rows,+ )+++def day_stats(rollup: type[DailyRollup], days: Sequence[dt.date]) -> dict[dt.date, DayStats]:+ """Row and event counts per published day. A day with no rows is absent from the result."""+ if not days:+ return {}+ rows = (+ rollup.model.objects.unscoped() # type: ignore[attr-defined]+ .using(WRITE_DATABASE)+ .filter(day__in=list(days))+ .values("day")+ .annotate(row_count=models.Count("*"), event_count=models.Sum("event_count"))+ .order_by("day")+ )+ return {row["day"]: DayStats(row_count=row["row_count"], event_count=row["event_count"] or 0) for row in rows}+++def _delete_in_batches(table: str, condition: str, params: list[dt.date | list[int]]) -> int:+ delete_sql = f"DELETE FROM {table} WHERE ctid = ANY(ARRAY(SELECT ctid FROM {table} WHERE {condition} LIMIT %s))"+ deleted = 0+ while True:+ with connections[WRITE_DATABASE].cursor() as cursor:+ cursor.execute(delete_sql, [*params, DELETE_BATCH_SIZE])+ batch = cursor.rowcount+ deleted += batch+ if batch < DELETE_BATCH_SIZE:+ return deleted+++def delete_days_before(rollup: type[DailyRollup], cutoff: dt.date) -> int:+ deleted = _delete_in_batches(rollup.table(), "day < %s", [cutoff])+ logger.info("daily rollup retention", table=rollup.table(), cutoff=cutoff.isoformat(), deleted_rows=deleted)+ return deleted+++def delete_rollup_rows_for_teams(model: type[models.Model], team_ids: Sequence[int]) -> int:+ """Delete every row of the given environments. Team deletion calls this, because no foreign key cascades."""+ if not team_ids:+ return 0+ return _delete_in_batches(model._meta.db_table, "team_id = ANY(%s)", [list(team_ids)])+++def delete_rows_of_deleted_teams(rollup: type[DailyRollup]) -> int:+ """Delete the rows of environments that no longer exist.++ A fill that publishes while a team deletion runs can write rows after the deletion removed them.+ """+ deleted_team_ids = list(+ rollup.model.objects.unscoped() # type: ignore[attr-defined]+ .using(WRITE_DATABASE)+ .filter(~Exists(Team.objects.filter(id=OuterRef("team_id"))))+ .values_list("team_id", flat=True)+ .distinct()+ )+ return delete_rollup_rows_for_teams(rollup.model, deleted_team_ids)diff --git a/posthog/temporal/health_checks/rollups/tests/__init__.py b/posthog/temporal/health_checks/rollups/tests/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/posthog/temporal/health_checks/rollups/tests/test_rollup_rules.py b/posthog/temporal/health_checks/rollups/tests/test_rollup_rules.pynew file mode 100644index 000000000000..b163d8cf3d9a--- /dev/null+++ b/posthog/temporal/health_checks/rollups/tests/test_rollup_rules.py@@ -0,0 +1,155 @@+import io+import datetime as dt++import pytest++from django.test import SimpleTestCase++from parameterized import parameterized++from posthog.temporal.health_checks.rollups.fill import DayFill, _copy_text, team_id_ranges+from posthog.temporal.health_checks.rollups.freshness import window_from_published_days+from posthog.temporal.health_checks.rollups.runner import plan_requested_days, plan_scheduled_days+from posthog.temporal.health_checks.rollups.store import DayStats+from posthog.temporal.health_checks.rollups.validation import day_moved, refusal_reason++from products.growth.backend.sdk_version_rollup import SdkVersionsDailyRollup++TODAY = dt.date(2026, 10, 29)+++def _fill(row_count: int, event_count: int = 1000) -> DayFill:+ return DayFill(day=TODAY, row_count=row_count, event_count=event_count, buffer=io.BytesIO())+++def _history(*row_counts: int) -> list[DayStats]:+ return [DayStats(row_count=rows, event_count=rows * 10) for rows in row_counts]+++def _days_back(*offsets: int) -> list[dt.date]:+ return [TODAY - dt.timedelta(days=offset) for offset in offsets]+++class TestRollupRules(SimpleTestCase):+ @parameterized.expand(+ [+ ("empty", [], 16, 0),+ ("fewer_teams_than_ranges", [5, 9, 2], 16, 3),+ ("many_teams", list(range(1, 1001)), 16, 16),+ ("sparse_ids_with_duplicates", [1, 1, 50, 9_000, 9_001, 500_000], 4, 4),+ ]+ )+ def test_team_id_ranges_cover_every_team_exactly_once(+ self, _name: str, team_ids: list[int], count: int, expected_ranges: int+ ) -> None:+ ranges = team_id_ranges(team_ids, count)++ assert len(ranges) == expected_ranges+ for team_id in set(team_ids):+ assert sum(r.start <= team_id < r.end for r in ranges) == 1++ @parameterized.expand(+ [+ ("null", None, "\\N"),+ ("true", True, "t"),+ ("false", False, "f"),+ ("int", 42, "42"),+ ("date", dt.date(2026, 10, 22), "2026-10-22"),+ ("naive_datetime_is_utc", dt.datetime(2026, 10, 22, 23, 59, 41), "2026-10-22T23:59:41+00:00"),+ ("tab_newline_backslash", "1.0\t\n\r\\x", "1.0\\t\\n\\r\\\\x"),+ ]+ )+ def test_copy_text_formats_values_for_postgres(self, _name: str, value: object, expected: str) -> None:+ assert _copy_text(value) == expected++ def test_copy_text_rejects_nul(self) -> None:+ with pytest.raises(ValueError):+ _copy_text("1.0\x00")++ @parameterized.expand(+ [+ ("empty_day", 0, _history(100, 100, 100), False, "no rows"),+ ("empty_day_even_when_forced", 0, _history(100, 100, 100), True, "no rows"),+ ("under_half_the_median", 40, _history(100, 90, 110), False, "median"),+ ("under_half_the_median_but_forced", 40, _history(100, 90, 110), True, None),+ ("normal_day", 95, _history(100, 90, 110), False, None),+ ("too_little_history_for_a_median", 1, _history(100, 100), False, None),+ ]+ )+ def test_refusal_reason(+ self, _name: str, row_count: int, history: list[DayStats], force: bool, expected: str | None+ ) -> None:+ reason = refusal_reason(SdkVersionsDailyRollup, _fill(row_count), history, force=force)++ if expected is None:+ assert reason is None+ else:+ assert reason is not None and expected in reason++ @parameterized.expand(+ [+ ("never_published", None, 100, 1000, True),+ ("identical", DayStats(row_count=100, event_count=1000), 100, 1000, False),+ ("within_one_percent", DayStats(row_count=100, event_count=1000), 101, 1010, False),+ ("late_events", DayStats(row_count=100, event_count=1000), 100, 1011, True),+ ("new_rows", DayStats(row_count=100, event_count=1000), 102, 1000, True),+ ]+ )+ def test_day_moved(self, _name: str, published: DayStats | None, rows: int, events: int, expected: bool) -> None:+ assert day_moved(published, _fill(rows, events)) is expected++ @parameterized.expand(+ [+ ("full_window", _days_back(*range(1, 8)), 7, (TODAY - dt.timedelta(days=7), TODAY - dt.timedelta(days=1))),+ (+ "latest_is_two_days_old",+ _days_back(*range(2, 9)),+ 7,+ (TODAY - dt.timedelta(days=8), TODAY - dt.timedelta(days=2)),+ ),+ ("latest_is_three_days_old", _days_back(*range(3, 10)), 7, None),+ ("gap_inside_window", _days_back(1, 2, 3, 5, 6, 7, 8), 7, None),+ ("first_days_after_deploy", _days_back(1, 2, 3), 30, None),+ ("nothing_published", [], 7, None),+ ]+ )+ def test_window_from_published_days(+ self, _name: str, published: list[dt.date], days: int, expected: tuple[dt.date, dt.date] | None+ ) -> None:+ window = window_from_published_days(published, days, today=TODAY)++ assert (None if window is None else (window.start, window.end)) == expected++ @parameterized.expand(+ [+ ("every_day_published", _days_back(*range(1, 11)), [(1, "publish"), (2, "refresh")]),+ (+ "gaps_inside_retention",+ _days_back(1, 2, 3, 5, 6, 7, 8, 9),+ [(1, "publish"), (2, "refresh"), (4, "publish"), (10, "publish")],+ ),+ (+ "first_run_after_deploy",+ [],+ [(1, "publish"), (2, "refresh"), *[(offset, "publish") for offset in range(3, 11)]],+ ),+ ]+ )+ def test_scheduled_plan_refreshes_recent_days_and_fills_every_missing_day_retention_keeps(+ self, _name: str, published: list[dt.date], expected: list[tuple[int, str]]+ ) -> None:+ plan = plan_scheduled_days(SdkVersionsDailyRollup, today=TODAY, published=published)++ assert [(step.day, step.mode) for step in plan] == [+ (TODAY - dt.timedelta(days=offset), mode) for offset, mode in expected+ ]++ def test_requested_plan_publishes_each_day_once_and_refuses_today(self) -> None:+ plan = plan_requested_days(_days_back(9, 10, 9), today=TODAY)+ assert [(step.day, step.mode) for step in plan] == [+ (dt.date(2026, 10, 19), "publish"),+ (dt.date(2026, 10, 20), "publish"),+ ]++ with pytest.raises(ValueError):+ plan_requested_days([TODAY], today=TODAY)diff --git a/posthog/temporal/health_checks/rollups/tests/test_rollup_store.py b/posthog/temporal/health_checks/rollups/tests/test_rollup_store.pynew file mode 100644index 000000000000..4bb3ba9eae37--- /dev/null+++ b/posthog/temporal/health_checks/rollups/tests/test_rollup_store.py@@ -0,0 +1,169 @@+import datetime as dt++import pytest+import time_machine+from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events, materialized+from unittest.mock import patch++from django.core.cache import cache++import dagster++from posthog.dags.daily_rollups import DailyRollupConfig+from posthog.models.team import Team+from posthog.temporal.health_checks.rollups import store+from posthog.temporal.health_checks.rollups.runner import RollupDay, RollupDayRefused, run_rollup_day+from posthog.temporal.health_checks.rollups.store import delete_days_before, delete_rollup_rows_for_teams++from products.growth.backend.models import SdkVersionsDaily+from products.growth.backend.sdk_version_rollup import SdkVersionsDailyRollup+from products.growth.dags.sdk_versions_rollup import sdk_versions_daily_rollup++YESTERDAY = dt.date(2026, 10, 28)+NOON_YESTERDAY = dt.datetime.combine(YESTERDAY, dt.time(12), tzinfo=dt.UTC)+++def _sdk_row(team_id: int, day: dt.date, event_count: int = 1) -> SdkVersionsDaily:+ return SdkVersionsDaily(+ team_id=team_id,+ day=day,+ lib="web",+ lib_version="1.0.0",+ event_count=event_count,+ max_timestamp=dt.datetime.combine(day, dt.time(12), tzinfo=dt.UTC),+ )+++@time_machine.travel("2026-10-29T12:00:00Z", tick=False)+class TestRollupFillAndPublish(ClickhouseTestMixin, BaseTest):+ def setUp(self) -> None:+ super().setUp()+ cache.clear()++ def _capture_sdk_events(self, team: Team, events: list[tuple[str, str]], timestamp: dt.datetime) -> None:+ for lib, version in events:+ _create_event(+ team=team,+ event="$pageview",+ distinct_id="user",+ timestamp=timestamp,+ properties={"$lib": lib, "$lib_version": version},+ )++ def test_publish_replaces_one_day_and_drops_deleted_teams(self) -> None:+ deleted_team = Team.objects.create(organization=self.organization)+ with materialized("events", "$lib"), materialized("events", "$lib_version"):+ self._capture_sdk_events(self.team, [("web", "1.0.0")] * 3, NOON_YESTERDAY)+ self._capture_sdk_events(deleted_team, [("web", "1.0.0")], NOON_YESTERDAY)+ flush_persons_and_events()+ Team.objects.filter(pk=deleted_team.pk).delete()+ other_day = _sdk_row(self.team.id, YESTERDAY - dt.timedelta(days=5), event_count=7)+ SdkVersionsDaily.objects.unscoped().bulk_create([other_day])++ run_rollup_day(SdkVersionsDailyRollup, RollupDay(day=YESTERDAY, mode="publish"))+ run_rollup_day(SdkVersionsDailyRollup, RollupDay(day=YESTERDAY, mode="publish"))++ rows = list(SdkVersionsDaily.objects.unscoped().values_list("team_id", "day", "event_count").order_by("day"))+ assert rows == [(self.team.id, other_day.day, 7), (self.team.id, YESTERDAY, 3)]++ def test_refused_day_keeps_the_published_rows(self) -> None:+ SdkVersionsDaily.objects.unscoped().bulk_create([_sdk_row(self.team.id, YESTERDAY, event_count=9)])++ with materialized("events", "$lib"), materialized("events", "$lib_version"):+ with pytest.raises(RollupDayRefused, match="no rows"):+ run_rollup_day(SdkVersionsDailyRollup, RollupDay(day=YESTERDAY, mode="publish"))++ assert list(SdkVersionsDaily.objects.for_team(self.team.id).values_list("event_count", flat=True)) == [9]++ def test_day_over_the_row_cap_is_refused_even_when_forced(self) -> None:+ with materialized("events", "$lib"), materialized("events", "$lib_version"):+ self._capture_sdk_events(self.team, [("web", "1.0.0"), ("web", "1.1.0"), ("web", "1.2.0")], NOON_YESTERDAY)+ flush_persons_and_events()++ with (+ patch.object(SdkVersionsDailyRollup, "max_rows_per_day", 1),+ pytest.raises(RollupDayRefused, match="more than 1 rows"),+ ):+ run_rollup_day(SdkVersionsDailyRollup, RollupDay(day=YESTERDAY, mode="publish"), force=True)++ assert not SdkVersionsDaily.objects.for_team(self.team.id).exists()++ def test_sdk_fill_keeps_known_sdks_with_storable_versions(self) -> None:+ events = [+ ("web", "1.200.0"),+ ("web", "1.200.0"),+ ("posthog-java", ""),+ ("posthog-python", ""),+ ("unknown-sdk", "1.0.0"),+ ("posthog-python", "9" * 65),+ ("posthog-node", "4.0.0-beta.1"),+ ]+ with materialized("events", "$lib"), materialized("events", "$lib_version"):+ self._capture_sdk_events(self.team, events, NOON_YESTERDAY)+ flush_persons_and_events()++ run_rollup_day(SdkVersionsDailyRollup, RollupDay(day=YESTERDAY, mode="publish"))++ rows = set(SdkVersionsDaily.objects.for_team(self.team.id).values_list("lib", "lib_version", "event_count"))+ assert rows == {("web", "1.200.0", 2), ("posthog-java", "", 1), ("posthog-node", "4.0.0-beta.1", 1)}++ @patch.object(store, "DELETE_BATCH_SIZE", 2)+ def test_batched_deletes_remove_every_matching_row(self) -> None:+ other_team_id = self.team.id + 1+ SdkVersionsDaily.objects.unscoped().bulk_create(+ [_sdk_row(self.team.id, YESTERDAY - dt.timedelta(days=offset)) for offset in range(15)]+ + [_sdk_row(other_team_id, YESTERDAY)]+ )++ assert delete_days_before(SdkVersionsDailyRollup, YESTERDAY - dt.timedelta(days=9)) == 5+ assert SdkVersionsDaily.objects.for_team(self.team.id).count() == 10++ assert delete_rollup_rows_for_teams(SdkVersionsDaily, [self.team.id]) == 10+ assert SdkVersionsDaily.objects.unscoped().count() == 1++ def test_scheduled_job_publishes_the_good_days_and_cleans_up_before_failing_on_a_refused_day(self) -> None:+ expired_day = YESTERDAY - dt.timedelta(days=20)+ deleted_team_id = self.team.id + 1000+ SdkVersionsDaily.objects.unscoped().bulk_create(+ [_sdk_row(self.team.id, expired_day), _sdk_row(deleted_team_id, YESTERDAY - dt.timedelta(days=4))]+ )+ with materialized("events", "$lib"), materialized("events", "$lib_version"):+ self._capture_sdk_events(self.team, [("web", "1.0.0")], NOON_YESTERDAY)+ self._capture_sdk_events(self.team, [("web", "1.0.0")] * 2, NOON_YESTERDAY - dt.timedelta(days=3))+ flush_persons_and_events()++ result = sdk_versions_daily_rollup.job.execute_in_process(raise_on_error=False)++ assert not result.success+ failure = result.failure_data_for_node(f"fill_{SdkVersionsDailyRollup.table()}")+ assert failure is not None and "Launch the job with `days`" in str(failure.error)++ published = set(SdkVersionsDaily.objects.unscoped().values_list("team_id", "day", "event_count"))+ assert published == {(self.team.id, YESTERDAY, 1), (self.team.id, YESTERDAY - dt.timedelta(days=3), 2)}++ def test_requested_day_with_force_publishes_under_the_median(self) -> None:+ SdkVersionsDaily.objects.unscoped().bulk_create(+ [+ _sdk_row(team_id, YESTERDAY - dt.timedelta(days=offset))+ for offset in range(2, 6)+ for team_id in range(self.team.id, self.team.id + 10)+ ]+ )+ with materialized("events", "$lib"), materialized("events", "$lib_version"):+ self._capture_sdk_events(self.team, [("web", "1.0.0")], NOON_YESTERDAY)+ flush_persons_and_events()+ op = f"fill_{SdkVersionsDailyRollup.table()}"+ requested = DailyRollupConfig(days=[YESTERDAY.isoformat()])++ refused = sdk_versions_daily_rollup.job.execute_in_process(+ run_config=dagster.RunConfig(ops={op: requested}), raise_on_error=False+ )+ forced = sdk_versions_daily_rollup.job.execute_in_process(+ run_config=dagster.RunConfig(ops={op: DailyRollupConfig(days=requested.days, force=True)})+ )++ assert not refused.success+ assert forced.success+ assert list(+ SdkVersionsDaily.objects.for_team(self.team.id).filter(day=YESTERDAY).values_list("event_count", flat=True)+ ) == [1]diff --git a/posthog/temporal/health_checks/rollups/validation.py b/posthog/temporal/health_checks/rollups/validation.pynew file mode 100644index 000000000000..0470e13015c2--- /dev/null+++ b/posthog/temporal/health_checks/rollups/validation.py@@ -0,0 +1,47 @@+from __future__ import annotations++import statistics+from collections.abc import Sequence++from posthog.temporal.health_checks.rollups.fill import DayFill+from posthog.temporal.health_checks.rollups.framework import DailyRollup+from posthog.temporal.health_checks.rollups.store import DayStats++MIN_FRACTION_OF_MEDIAN = 0.5+MIN_HISTORY_DAYS_FOR_MEDIAN = 3+D2_CHANGE_THRESHOLD = 0.01+++def refusal_reason(rollup: type[DailyRollup], fill: DayFill, history: Sequence[DayStats], *, force: bool) -> str | None:+ """Why the filled day must not be published, or None when it may be.++ `force` lifts only the median floor. An empty day is always refused, because an empty day is what+ a ClickHouse outage looks like. `fill_day` refuses a day over the row cap before this runs.+ """+ if fill.row_count == 0:+ return "the day has no rows"+ if force or len(history) < MIN_HISTORY_DAYS_FOR_MEDIAN:+ return None+ median_rows = statistics.median(stats.row_count for stats in history)+ if fill.row_count < median_rows * MIN_FRACTION_OF_MEDIAN:+ return (+ f"the day has {fill.row_count:,} rows, under {MIN_FRACTION_OF_MEDIAN:.0%} of the "+ f"{len(history)}-day median of {median_rows:,.0f}"+ )+ return None+++def _relative_change(published: int, filled: int) -> float:+ if published == 0:+ return 0.0 if filled == 0 else 1.0+ return abs(filled - published) / published+++def day_moved(published: DayStats | None, fill: DayFill) -> bool:+ """Whether a refill differs enough from the published day to replace it."""+ if published is None:+ return True+ return (+ _relative_change(published.row_count, fill.row_count) > D2_CHANGE_THRESHOLD+ or _relative_change(published.event_count, fill.event_count) > D2_CHANGE_THRESHOLD+ )diff --git a/products/growth/backend/facade/api.py b/products/growth/backend/facade/api.pyindex 9f169d022ab9..e929bd562e13 100644--- a/products/growth/backend/facade/api.py+++ b/products/growth/backend/facade/api.py@@ -3,9 +3,10 @@ from typing import Literalfrom posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exceptionfrom posthog.models.instance_setting import get_instance_setting+from posthog.temporal.health_checks.rollups.store import delete_rollup_rows_for_teamsfrom posthog.utils import get_instance_region-from products.growth.backend.models import OrganizationEnrichment+from products.growth.backend.models import OrganizationEnrichment, SdkVersionsDailyfrom products.growth.backend.temporal.signup_enrichment.trigger import dispatch_wizard_stamp_rescoreWizardStampRescoreSkipReason = Literal["disabled", "no_enrichment_record", "dispatch_backlog_full", "dispatch_failed"]@@ -36,3 +37,8 @@ def request_wizard_stamp_rescore(organization_id: str) -> WizardStampRescoreOutcif failure is not None:return WizardStampRescoreOutcome(queued=False, reason=failure)return WizardStampRescoreOutcome(queued=True)+++def delete_sdk_version_rollups_for_teams(team_ids: list[int]) -> None:+ """Remove the SDK version rollup rows of the given environments. Called from team deletion."""+ delete_rollup_rows_for_teams(SdkVersionsDaily, team_ids)diff --git a/products/growth/backend/migrations/0011_daily_health_rollups.py b/products/growth/backend/migrations/0011_daily_health_rollups.pynew file mode 100644index 000000000000..98d122b068a8--- /dev/null+++ b/products/growth/backend/migrations/0011_daily_health_rollups.py@@ -0,0 +1,38 @@+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("growth", "0010_icp_scoring_rules"),+ ]++ operations = [+ migrations.CreateModel(+ name="SdkVersionsDaily",+ fields=[+ (+ "pk",+ models.CompositePrimaryKey(+ "team_id",+ "day",+ "lib",+ "lib_version",+ blank=True,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("team_id", models.BigIntegerField()),+ ("day", models.DateField()),+ ("lib", models.CharField(max_length=64)),+ ("lib_version", models.CharField(max_length=64)),+ ("event_count", models.BigIntegerField()),+ ("max_timestamp", models.DateTimeField()),+ ],+ options={+ "db_table": "growth_sdk_versions_daily",+ "indexes": [models.Index(fields=["day"], name="growth_sdk_ver_daily_day_idx")],+ },+ ),+ ]diff --git a/products/growth/backend/migrations/0012_daily_health_rollups_autovacuum.py b/products/growth/backend/migrations/0012_daily_health_rollups_autovacuum.pynew file mode 100644index 000000000000..ebdb9491f539--- /dev/null+++ b/products/growth/backend/migrations/0012_daily_health_rollups_autovacuum.py@@ -0,0 +1,22 @@+from django.db import migrations++SCALE_FACTOR = 0.02+++class Migration(migrations.Migration):+ dependencies = [+ ("growth", "0011_daily_health_rollups"),+ ]++ operations = [+ migrations.RunSQL(+ sql=[+ "SET LOCAL lock_timeout = '5s'",+ f"ALTER TABLE growth_sdk_versions_daily SET (autovacuum_vacuum_scale_factor = {SCALE_FACTOR}, autovacuum_analyze_scale_factor = {SCALE_FACTOR})",+ ],+ reverse_sql=[+ "SET LOCAL lock_timeout = '5s'",+ "ALTER TABLE growth_sdk_versions_daily RESET (autovacuum_vacuum_scale_factor, autovacuum_analyze_scale_factor)",+ ],+ ),+ ]diff --git a/products/growth/backend/migrations/max_migration.txt b/products/growth/backend/migrations/max_migration.txtindex b556a473aa27..6be12d4b705b 100644--- a/products/growth/backend/migrations/max_migration.txt+++ b/products/growth/backend/migrations/max_migration.txt@@ -1 +1 @@-0010_icp_scoring_rules+0012_daily_health_rollups_autovacuumdiff --git a/products/growth/backend/models.py b/products/growth/backend/models.pyindex 3c4fedac7f3c..a5d62684956c 100644--- a/products/growth/backend/models.py+++ b/products/growth/backend/models.py@@ -6,6 +6,7 @@ from django.core.exceptions import ValidationErrorfrom django.db import models, transactionfrom django.db.models import Q+from posthog.models.scoping.manager import EnvironmentScopedManagerfrom posthog.models.utils import UpdatedMetaFields, UUIDModelfrom products.growth.backend.enrichment.icp_lists import clear_lists_cache@@ -342,3 +343,26 @@ class EnrichmentSignupSnapshot(UUIDModel):related_name="enrichment_signup_snapshot",)created_at = models.DateTimeField(auto_now_add=True)+++class SdkVersionsDaily(models.Model):+ """Events per SDK version for one environment and one UTC day.++ The daily rollup job replaces a whole day at a time, so readers sum days instead of+ scanning raw events. `team_id` is the environment id from events, and has no foreign key+ because team deletion removes the rows explicitly.+ """++ pk = models.CompositePrimaryKey("team_id", "day", "lib", "lib_version")+ team_id = models.BigIntegerField()+ day = models.DateField()+ lib = models.CharField(max_length=64)+ lib_version = models.CharField(max_length=64)+ event_count = models.BigIntegerField()+ max_timestamp = models.DateTimeField()++ objects = EnvironmentScopedManager()++ class Meta:+ db_table = "growth_sdk_versions_daily"+ indexes = [models.Index(fields=["day"], name="growth_sdk_ver_daily_day_idx")]diff --git a/products/growth/backend/sdk_version_rollup.py b/products/growth/backend/sdk_version_rollup.pynew file mode 100644index 000000000000..c1b80936ede9--- /dev/null+++ b/products/growth/backend/sdk_version_rollup.py@@ -0,0 +1,50 @@+from collections.abc import Mapping+from typing import Any++from posthog.clickhouse.query_tagging import Product+from posthog.temporal.health_checks.rollups.framework import DailyRollup++from products.growth.backend.constants import SDK_TYPES+from products.growth.backend.models import SdkVersionsDaily++MAX_VERSIONS_PER_TEAM_LIB_DAY = 100++SDK_VERSIONS_FILL_SQL = """+SELECT+ team_id,+ `mat_$lib` AS lib,+ `mat_$lib_version` AS lib_version,+ count() AS event_count,+ max(timestamp) AS max_timestamp+FROM events+WHERE team_id >= %(team_id_start)s+ AND team_id < %(team_id_end)s+ AND timestamp >= %(day_start)s+ AND timestamp < %(day_end)s+ AND `mat_$lib` IN %(sdk_types)s+ AND (`mat_$lib` = 'posthog-java' OR `mat_$lib_version` != '')+ AND length(`mat_$lib_version`) <= %(max_version_length)s+ AND position(`mat_$lib_version`, char(0)) = 0+GROUP BY team_id, lib, lib_version+ORDER BY team_id, lib, event_count DESC+LIMIT %(max_versions)s BY team_id, lib+"""+++class SdkVersionsDailyRollup(DailyRollup):+ model = SdkVersionsDaily+ product = Product.SDK_HEALTH+ columns = ("team_id", "day", "lib", "lib_version", "event_count", "max_timestamp")+ retention_days = 10++ @classmethod+ def fill_sql(cls) -> str:+ return SDK_VERSIONS_FILL_SQL++ @classmethod+ def fill_params(cls) -> Mapping[str, Any]:+ return {+ "sdk_types": list(SDK_TYPES),+ "max_version_length": SdkVersionsDaily._meta.get_field("lib_version").max_length,+ "max_versions": MAX_VERSIONS_PER_TEAM_LIB_DAY,+ }diff --git a/products/growth/dags/sdk_versions_rollup.py b/products/growth/dags/sdk_versions_rollup.pynew file mode 100644index 000000000000..6514f73f904f--- /dev/null+++ b/products/growth/dags/sdk_versions_rollup.py@@ -0,0 +1,6 @@+from posthog.dags.common import JobOwners+from posthog.dags.daily_rollups import build_daily_rollup_definitions++from products.growth.backend.sdk_version_rollup import SdkVersionsDailyRollup++sdk_versions_daily_rollup = build_daily_rollup_definitions(SdkVersionsDailyRollup, owner=JobOwners.TEAM_GROWTH)