Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(metrics): drop replayed duplicate points behind a flag (#125988)

Daniel Visca1aeedc305756
diff --git a/products/metrics/backend/metric_query_runner.py b/products/metrics/backend/metric_query_runner.py
index a6dfd171f28f..c4d38008cc9b 100644
--- a/products/metrics/backend/metric_query_runner.py
+++ b/products/metrics/backend/metric_query_runner.py
@@ -10,6 +10,7 @@ import math
import bisect
import datetime as dt
from collections.abc import Iterator, Sequence
+from functools import cached_property
from typing import Any, Literal, NamedTuple
from zoneinfo import ZoneInfo
@@ -22,10 +23,11 @@ from posthog.hogql.query import execute_hogql_query
from posthog.clickhouse.client.connection import Workload
from posthog.dataclasses import frozen
from posthog.models import Team
+from posthog.permissions import posthog_feature_flag_enabled
from products.metrics.backend.facade.contracts import MetricFilter, MetricGroupBy, MetricQueryTooManyRowsError
from products.metrics.backend.facade.enums import FilterOp, MetricType
-from products.metrics.backend.metrics4_samples import metric_name_expr, samples_points_query
+from products.metrics.backend.metrics4_samples import dedupe_points_limit_by, metric_name_expr, samples_points_query
AttributeScope = Literal["resource", "attribute", "auto"]
@@ -468,6 +470,10 @@ def time_range_expr(date_from: dt.datetime, date_to: dt.datetime) -> ast.Expr:
)
+# Drops points that a replayed Kafka batch wrote twice. Off until its query cost is measured.
+METRICS_QUERY_DEDUPE_POINTS_FLAG = "metrics-query-dedupe-points"
+
+
def _utc_hour(value: dt.datetime) -> dt.datetime:
return value.astimezone(dt.UTC).replace(minute=0, second=0, microsecond=0)
@@ -483,6 +489,7 @@ def points_query(
row_filters: Sequence[ast.Expr] = (),
point_filters: Sequence[ast.Expr] = (),
samples_row_filters: Sequence[ast.Expr] = (),
+ dedupe_points: bool = False,
) -> ast.SelectQuery:
"""Return one row per metric point in `[date_from, date_to)`.
@@ -490,6 +497,7 @@ def points_query(
or after the metrics4 cut-over. Otherwise read the `metrics` view.
`row_filters` can use only columns that are constant in a series-hour.
`samples_row_filters` apply only to `metric_samples`, for example array lookups.
+ `dedupe_points` drops exact copies of a point, as written by a replayed Kafka batch.
"""
if from_samples:
return samples_points_query(
@@ -500,6 +508,7 @@ def points_query(
timezone=timezone,
row_filters=[*row_filters, *samples_row_filters],
point_filters=point_filters,
+ dedupe_points=dedupe_points,
)
query = parse_select(
"SELECT 1 FROM posthog.metrics WHERE {name_filter} AND {time_range}",
@@ -509,6 +518,8 @@ def points_query(
query.select = [ast.Field(chain=[column]) for column in columns]
if row_filters or point_filters:
query.where = ast.And(exprs=[query.where, *row_filters, *point_filters])
+ if dedupe_points:
+ query.limit_by = dedupe_points_limit_by([column for column in columns if column != "series_fingerprint"])
return query
@@ -851,7 +862,9 @@ class MetricQueryRunner:
)
return series_scope_expr(self.metric_name, self.filters, self.date_from)
- def _points_query(self, columns: Sequence[str], point_filters: Sequence[ast.Expr] = ()) -> ast.SelectQuery:
+ def _points_query(
+ self, columns: Sequence[str], point_filters: Sequence[ast.Expr] = (), dedupe_points: bool = True
+ ) -> ast.SelectQuery:
return points_query(
from_samples=self.reads_samples,
columns=columns,
@@ -861,6 +874,17 @@ class MetricQueryRunner:
timezone=self.team.timezone,
row_filters=(self._series_scope_expr(), self._type_filter_expr()),
point_filters=point_filters,
+ dedupe_points=dedupe_points and self._dedupe_points,
+ )
+
+ @cached_property
+ def _dedupe_points(self) -> bool:
+ return posthog_feature_flag_enabled(
+ METRICS_QUERY_DEDUPE_POINTS_FLAG,
+ str(self.team.uuid),
+ organization_id=self.team.organization_id,
+ team_id=self.team.pk,
+ only_evaluate_locally=True,
)
def _combine_series(self, per_series: ast.SelectQuery) -> ast.SelectQuery:
@@ -921,7 +945,8 @@ class MetricQueryRunner:
""",
placeholders={
"interval": _interval_expr(self.interval),
- "points": self._points_query(("timestamp", "series_fingerprint", "value")),
+ # A copy cannot change argMax, so skip the dedupe cost.
+ "points": self._points_query(("timestamp", "series_fingerprint", "value"), dedupe_points=False),
"series_key": _series_key_expr(),
},
)
diff --git a/products/metrics/backend/metrics4_samples.py b/products/metrics/backend/metrics4_samples.py
index 0aba0fccf5d4..bf74e8ac9810 100644
--- a/products/metrics/backend/metrics4_samples.py
+++ b/products/metrics/backend/metrics4_samples.py
@@ -105,6 +105,7 @@ def samples_points_query(
timezone: str,
row_filters: Sequence[ast.Expr] = (),
point_filters: Sequence[ast.Expr] = (),
+ dedupe_points: bool = False,
) -> ast.SelectQuery:
"""Return one row per point in `[date_from, date_to)`, with the column names of `metrics`.
@@ -114,6 +115,9 @@ def samples_points_query(
`row_filters` can use only row columns. They apply before the `ARRAY JOIN`.
`point_filters` can use the selected columns. They apply after it.
`metric_names=None` reads every metric name.
+ `dedupe_points` drops exact copies of a point. A replayed Kafka batch writes
+ its points again, and they stay in the arrays after merges. A copy has the same
+ series, timestamp and value in every selected column. Other points stay.
"""
unknown = [column for column in columns if column not in _POINT_ARRAYS and column not in _ROW_COLUMNS]
if unknown:
@@ -122,6 +126,8 @@ def samples_points_query(
# The time filter always needs the timestamp array.
point_columns = ["timestamp", *(column for column in columns if column in _POINT_ARRAYS and column != "timestamp")]
row_columns = [column for column in columns if column in _ROW_COLUMNS]
+ if dedupe_points and "series_fingerprint" not in row_columns:
+ row_columns.append("series_fingerprint")
rows = parse_select(
"SELECT 1 FROM posthog.metric_samples WHERE {metric_name} AND {row_range}",
@@ -162,6 +168,23 @@ def samples_points_query(
for column in point_columns
],
where=ast.And(exprs=[point_range, *point_filters]),
+ limit_by=dedupe_points_limit_by(
+ [
+ "point_timestamp",
+ *point_columns[1:],
+ *(column for column in row_columns if column != "series_fingerprint"),
+ ]
+ )
+ if dedupe_points
+ else None,
+ )
+
+
+def dedupe_points_limit_by(point_fields: Sequence[str]) -> ast.LimitByExpr:
+ """Keep one row for each series and each distinct combination of `point_fields`."""
+ return ast.LimitByExpr(
+ n=ast.Constant(value=1),
+ exprs=[ast.Field(chain=[field]) for field in ("series_fingerprint", *point_fields)],
)
diff --git a/products/metrics/backend/tests/test_metric_query_runner.py b/products/metrics/backend/tests/test_metric_query_runner.py
index 2228f946f957..4f4a1fa0efb7 100644
--- a/products/metrics/backend/tests/test_metric_query_runner.py
+++ b/products/metrics/backend/tests/test_metric_query_runner.py
@@ -1,3 +1,4 @@
+import re
import datetime as dt
import threading
from typing import Any
@@ -1351,17 +1352,50 @@ class TestRateIncrease(ClickhouseTestMixin, APIBaseTest):
# The reset contributes its post-reset value.
self.assertEqual([row["value"] for row in rows], [25.0])
- def test_delta_temporality_sums_samples_directly(self):
- self._seed_counter(
- [
- (self.anchor + dt.timedelta(seconds=0), 3.0),
- (self.anchor + dt.timedelta(seconds=15), 4.0),
- (self.anchor + dt.timedelta(seconds=30), 5.0),
- ],
- aggregation_temporality="delta",
- )
- rows = self._run("increase")
- self.assertEqual([row["value"] for row in rows], [12.0])
+ @parameterized.expand(
+ [
+ ("delivered_once", [[(0, 3.0), (15, 4.0), (30, 5.0)]], False, 12.0),
+ ("delivered_once_with_dedupe", [[(0, 3.0), (15, 4.0), (30, 5.0)]], True, 12.0),
+ # An at-least-once replay sends the first points again in a new block, so
+ # ClickHouse insert deduplication does not drop them.
+ ("replayed_with_dedupe", [[(0, 3.0), (15, 4.0)], [(0, 3.0), (15, 4.0), (30, 5.0)]], True, 12.0),
+ # Dedupe drops only exact copies, not a different point at the same time.
+ ("same_time_different_value_with_dedupe", [[(0, 3.0)], [(0, 4.0)]], True, 7.0),
+ ]
+ )
+ def test_delta_temporality_sums_samples_directly(
+ self, _name: str, batches: list[list[tuple[int, float]]], dedupe: bool, expected: float
+ ):
+ for batch in batches:
+ self._seed_counter(
+ [(self.anchor + dt.timedelta(seconds=offset), value) for offset, value in batch],
+ aggregation_temporality="delta",
+ )
+ with patch(
+ "products.metrics.backend.metric_query_runner.posthog_feature_flag_enabled", return_value=dedupe
+ ) as flag:
+ rows = self._run("increase")
+ self.assertEqual([row["value"] for row in rows], [expected])
+ self.assertEqual(flag.call_args.args[0], "metrics-query-dedupe-points")
+
+ @parameterized.expand(
+ [
+ # A copy cannot change argMax, so a last-value query skips the dedupe cost.
+ ("last_value", "sum", {}, False),
+ ("increase", "sum", {"range_function": "increase"}, True),
+ ]
+ )
+ def test_dedupe_flag_applies_only_where_copies_change_the_result(
+ self, _name: str, aggregation: str, overrides: dict[str, Any], expect_dedupe: bool
+ ):
+ with (
+ patch("products.metrics.backend.metric_query_runner.posthog_feature_flag_enabled", return_value=True),
+ patch(
+ "products.metrics.backend.metric_query_runner.execute_hogql_query", wraps=execute_hogql_query
+ ) as execute,
+ ):
+ self._run(aggregation, **overrides)
+ self.assertEqual(bool(re.search(r"LIMIT 1\s+BY", execute.call_args.kwargs["query"].to_hogql())), expect_dedupe)
def test_deltas_are_computed_per_underlying_series(self):
# Diff each pod separately.