:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(metrics): drop replayed duplicate points behind a flag (#125988)
Daniel Visca1aeedc305756
diff --git a/products/metrics/backend/metric_query_runner.py b/products/metrics/backend/metric_query_runner.pyindex a6dfd171f28f..c4d38008cc9b 100644--- a/products/metrics/backend/metric_query_runner.py+++ b/products/metrics/backend/metric_query_runner.py@@ -10,6 +10,7 @@ import mathimport bisectimport datetime as dtfrom collections.abc import Iterator, Sequence+from functools import cached_propertyfrom typing import Any, Literal, NamedTuplefrom zoneinfo import ZoneInfo@@ -22,10 +23,11 @@ from posthog.hogql.query import execute_hogql_queryfrom posthog.clickhouse.client.connection import Workloadfrom posthog.dataclasses import frozenfrom posthog.models import Team+from posthog.permissions import posthog_feature_flag_enabledfrom products.metrics.backend.facade.contracts import MetricFilter, MetricGroupBy, MetricQueryTooManyRowsErrorfrom products.metrics.backend.facade.enums import FilterOp, MetricType-from products.metrics.backend.metrics4_samples import metric_name_expr, samples_points_query+from products.metrics.backend.metrics4_samples import dedupe_points_limit_by, metric_name_expr, samples_points_queryAttributeScope = Literal["resource", "attribute", "auto"]@@ -468,6 +470,10 @@ def time_range_expr(date_from: dt.datetime, date_to: dt.datetime) -> ast.Expr:)+# Drops points that a replayed Kafka batch wrote twice. Off until its query cost is measured.+METRICS_QUERY_DEDUPE_POINTS_FLAG = "metrics-query-dedupe-points"++def _utc_hour(value: dt.datetime) -> dt.datetime:return value.astimezone(dt.UTC).replace(minute=0, second=0, microsecond=0)@@ -483,6 +489,7 @@ def points_query(row_filters: Sequence[ast.Expr] = (),point_filters: Sequence[ast.Expr] = (),samples_row_filters: Sequence[ast.Expr] = (),+ dedupe_points: bool = False,) -> ast.SelectQuery:"""Return one row per metric point in `[date_from, date_to)`.@@ -490,6 +497,7 @@ def points_query(or after the metrics4 cut-over. Otherwise read the `metrics` view.`row_filters` can use only columns that are constant in a series-hour.`samples_row_filters` apply only to `metric_samples`, for example array lookups.+ `dedupe_points` drops exact copies of a point, as written by a replayed Kafka batch."""if from_samples:return samples_points_query(@@ -500,6 +508,7 @@ def points_query(timezone=timezone,row_filters=[*row_filters, *samples_row_filters],point_filters=point_filters,+ dedupe_points=dedupe_points,)query = parse_select("SELECT 1 FROM posthog.metrics WHERE {name_filter} AND {time_range}",@@ -509,6 +518,8 @@ def points_query(query.select = [ast.Field(chain=[column]) for column in columns]if row_filters or point_filters:query.where = ast.And(exprs=[query.where, *row_filters, *point_filters])+ if dedupe_points:+ query.limit_by = dedupe_points_limit_by([column for column in columns if column != "series_fingerprint"])return query@@ -851,7 +862,9 @@ class MetricQueryRunner:)return series_scope_expr(self.metric_name, self.filters, self.date_from)- def _points_query(self, columns: Sequence[str], point_filters: Sequence[ast.Expr] = ()) -> ast.SelectQuery:+ def _points_query(+ self, columns: Sequence[str], point_filters: Sequence[ast.Expr] = (), dedupe_points: bool = True+ ) -> ast.SelectQuery:return points_query(from_samples=self.reads_samples,columns=columns,@@ -861,6 +874,17 @@ class MetricQueryRunner:timezone=self.team.timezone,row_filters=(self._series_scope_expr(), self._type_filter_expr()),point_filters=point_filters,+ dedupe_points=dedupe_points and self._dedupe_points,+ )++ @cached_property+ def _dedupe_points(self) -> bool:+ return posthog_feature_flag_enabled(+ METRICS_QUERY_DEDUPE_POINTS_FLAG,+ str(self.team.uuid),+ organization_id=self.team.organization_id,+ team_id=self.team.pk,+ only_evaluate_locally=True,)def _combine_series(self, per_series: ast.SelectQuery) -> ast.SelectQuery:@@ -921,7 +945,8 @@ class MetricQueryRunner:""",placeholders={"interval": _interval_expr(self.interval),- "points": self._points_query(("timestamp", "series_fingerprint", "value")),+ # A copy cannot change argMax, so skip the dedupe cost.+ "points": self._points_query(("timestamp", "series_fingerprint", "value"), dedupe_points=False),"series_key": _series_key_expr(),},)diff --git a/products/metrics/backend/metrics4_samples.py b/products/metrics/backend/metrics4_samples.pyindex 0aba0fccf5d4..bf74e8ac9810 100644--- a/products/metrics/backend/metrics4_samples.py+++ b/products/metrics/backend/metrics4_samples.py@@ -105,6 +105,7 @@ def samples_points_query(timezone: str,row_filters: Sequence[ast.Expr] = (),point_filters: Sequence[ast.Expr] = (),+ dedupe_points: bool = False,) -> ast.SelectQuery:"""Return one row per point in `[date_from, date_to)`, with the column names of `metrics`.@@ -114,6 +115,9 @@ def samples_points_query(`row_filters` can use only row columns. They apply before the `ARRAY JOIN`.`point_filters` can use the selected columns. They apply after it.`metric_names=None` reads every metric name.+ `dedupe_points` drops exact copies of a point. A replayed Kafka batch writes+ its points again, and they stay in the arrays after merges. A copy has the same+ series, timestamp and value in every selected column. Other points stay."""unknown = [column for column in columns if column not in _POINT_ARRAYS and column not in _ROW_COLUMNS]if unknown:@@ -122,6 +126,8 @@ def samples_points_query(# The time filter always needs the timestamp array.point_columns = ["timestamp", *(column for column in columns if column in _POINT_ARRAYS and column != "timestamp")]row_columns = [column for column in columns if column in _ROW_COLUMNS]+ if dedupe_points and "series_fingerprint" not in row_columns:+ row_columns.append("series_fingerprint")rows = parse_select("SELECT 1 FROM posthog.metric_samples WHERE {metric_name} AND {row_range}",@@ -162,6 +168,23 @@ def samples_points_query(for column in point_columns],where=ast.And(exprs=[point_range, *point_filters]),+ limit_by=dedupe_points_limit_by(+ [+ "point_timestamp",+ *point_columns[1:],+ *(column for column in row_columns if column != "series_fingerprint"),+ ]+ )+ if dedupe_points+ else None,+ )+++def dedupe_points_limit_by(point_fields: Sequence[str]) -> ast.LimitByExpr:+ """Keep one row for each series and each distinct combination of `point_fields`."""+ return ast.LimitByExpr(+ n=ast.Constant(value=1),+ exprs=[ast.Field(chain=[field]) for field in ("series_fingerprint", *point_fields)],)diff --git a/products/metrics/backend/tests/test_metric_query_runner.py b/products/metrics/backend/tests/test_metric_query_runner.pyindex 2228f946f957..4f4a1fa0efb7 100644--- a/products/metrics/backend/tests/test_metric_query_runner.py+++ b/products/metrics/backend/tests/test_metric_query_runner.py@@ -1,3 +1,4 @@+import reimport datetime as dtimport threadingfrom typing import Any@@ -1351,17 +1352,50 @@ class TestRateIncrease(ClickhouseTestMixin, APIBaseTest):# The reset contributes its post-reset value.self.assertEqual([row["value"] for row in rows], [25.0])- def test_delta_temporality_sums_samples_directly(self):- self._seed_counter(- [- (self.anchor + dt.timedelta(seconds=0), 3.0),- (self.anchor + dt.timedelta(seconds=15), 4.0),- (self.anchor + dt.timedelta(seconds=30), 5.0),- ],- aggregation_temporality="delta",- )- rows = self._run("increase")- self.assertEqual([row["value"] for row in rows], [12.0])+ @parameterized.expand(+ [+ ("delivered_once", [[(0, 3.0), (15, 4.0), (30, 5.0)]], False, 12.0),+ ("delivered_once_with_dedupe", [[(0, 3.0), (15, 4.0), (30, 5.0)]], True, 12.0),+ # An at-least-once replay sends the first points again in a new block, so+ # ClickHouse insert deduplication does not drop them.+ ("replayed_with_dedupe", [[(0, 3.0), (15, 4.0)], [(0, 3.0), (15, 4.0), (30, 5.0)]], True, 12.0),+ # Dedupe drops only exact copies, not a different point at the same time.+ ("same_time_different_value_with_dedupe", [[(0, 3.0)], [(0, 4.0)]], True, 7.0),+ ]+ )+ def test_delta_temporality_sums_samples_directly(+ self, _name: str, batches: list[list[tuple[int, float]]], dedupe: bool, expected: float+ ):+ for batch in batches:+ self._seed_counter(+ [(self.anchor + dt.timedelta(seconds=offset), value) for offset, value in batch],+ aggregation_temporality="delta",+ )+ with patch(+ "products.metrics.backend.metric_query_runner.posthog_feature_flag_enabled", return_value=dedupe+ ) as flag:+ rows = self._run("increase")+ self.assertEqual([row["value"] for row in rows], [expected])+ self.assertEqual(flag.call_args.args[0], "metrics-query-dedupe-points")++ @parameterized.expand(+ [+ # A copy cannot change argMax, so a last-value query skips the dedupe cost.+ ("last_value", "sum", {}, False),+ ("increase", "sum", {"range_function": "increase"}, True),+ ]+ )+ def test_dedupe_flag_applies_only_where_copies_change_the_result(+ self, _name: str, aggregation: str, overrides: dict[str, Any], expect_dedupe: bool+ ):+ with (+ patch("products.metrics.backend.metric_query_runner.posthog_feature_flag_enabled", return_value=True),+ patch(+ "products.metrics.backend.metric_query_runner.execute_hogql_query", wraps=execute_hogql_query+ ) as execute,+ ):+ self._run(aggregation, **overrides)+ self.assertEqual(bool(re.search(r"LIMIT 1\s+BY", execute.call_args.kwargs["query"].to_hogql())), expect_dedupe)def test_deltas_are_computed_per_underlying_series(self):# Diff each pod separately.