Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(metrics): dedupe replayed points in event samples behind the flag (#129797)

Daniel Visca91d36c4fdd56
diff --git a/products/metrics/backend/metric_event_samples_query_runner.py b/products/metrics/backend/metric_event_samples_query_runner.py
index bd7cbd249b4b..5fc38652da28 100644
--- a/products/metrics/backend/metric_event_samples_query_runner.py
+++ b/products/metrics/backend/metric_event_samples_query_runner.py
@@ -21,7 +21,12 @@ from posthog.models import Team
from products.metrics.backend.facade.contracts import MetricFilter
from products.metrics.backend.facade.enums import MetricType
-from products.metrics.backend.metric_query_runner import points_query, series_scope_expr, type_filter_expr
+from products.metrics.backend.metric_query_runner import (
+ dedupe_points_enabled,
+ points_query,
+ series_scope_expr,
+ type_filter_expr,
+)
from products.metrics.backend.metrics4_samples import reads_metrics4_only
# This query uses the shared ClickHouse cluster. Limit reads and fail on overflow.
@@ -121,6 +126,7 @@ class MetricEventSamplesQueryRunner:
),
point_filters=point_filters,
samples_row_filters=samples_row_filters,
+ dedupe_points=dedupe_points_enabled(self.team),
)
def run(self) -> list[dict[str, Any]]:
diff --git a/products/metrics/backend/metric_query_runner.py b/products/metrics/backend/metric_query_runner.py
index c4d38008cc9b..22efdbcf7584 100644
--- a/products/metrics/backend/metric_query_runner.py
+++ b/products/metrics/backend/metric_query_runner.py
@@ -474,6 +474,16 @@ def time_range_expr(date_from: dt.datetime, date_to: dt.datetime) -> ast.Expr:
METRICS_QUERY_DEDUPE_POINTS_FLAG = "metrics-query-dedupe-points"
+def dedupe_points_enabled(team: Team) -> bool:
+ return posthog_feature_flag_enabled(
+ METRICS_QUERY_DEDUPE_POINTS_FLAG,
+ str(team.uuid),
+ organization_id=team.organization_id,
+ team_id=team.pk,
+ only_evaluate_locally=True,
+ )
+
+
def _utc_hour(value: dt.datetime) -> dt.datetime:
return value.astimezone(dt.UTC).replace(minute=0, second=0, microsecond=0)
@@ -879,13 +889,7 @@ class MetricQueryRunner:
@cached_property
def _dedupe_points(self) -> bool:
- return posthog_feature_flag_enabled(
- METRICS_QUERY_DEDUPE_POINTS_FLAG,
- str(self.team.uuid),
- organization_id=self.team.organization_id,
- team_id=self.team.pk,
- only_evaluate_locally=True,
- )
+ return dedupe_points_enabled(self.team)
def _combine_series(self, per_series: ast.SelectQuery) -> ast.SelectQuery:
"""Combine the per-series query, whose rows are `(time, series_fingerprint, series_value)`.
diff --git a/products/metrics/backend/tests/test_metric_event_samples_query_runner.py b/products/metrics/backend/tests/test_metric_event_samples_query_runner.py
index 409ff3672fc3..4ca0477b72fe 100644
--- a/products/metrics/backend/tests/test_metric_event_samples_query_runner.py
+++ b/products/metrics/backend/tests/test_metric_event_samples_query_runner.py
@@ -2,6 +2,7 @@ import base64
import datetime as dt
from posthog.test.base import APIBaseTest, ClickhouseTestMixin
+from unittest.mock import patch
from django.utils import timezone
@@ -262,6 +263,39 @@ class TestMetricEventSamplesQueryRunner(ClickhouseTestMixin, APIBaseTest):
self.assertEqual(samples[0].metric_type, "") # column default
self.assertEqual(samples[0].attributes, {})
+ @parameterized.expand(
+ [
+ # An at-least-once replay sends the first point again in a new block, so
+ # ClickHouse insert deduplication does not drop it.
+ ("replayed_without_dedupe", [[(0, 1.0)], [(0, 1.0), (15, 2.0)]], False, [2.0, 1.0, 1.0]),
+ ("replayed_with_dedupe", [[(0, 1.0)], [(0, 1.0), (15, 2.0)]], True, [2.0, 1.0]),
+ # Dedupe drops only exact copies, not a different point at the same time.
+ ("same_time_different_value_with_dedupe", [[(0, 1.0)], [(0, 3.0)]], True, [3.0, 1.0]),
+ ]
+ )
+ def test_replayed_points_are_deduped_behind_flag(
+ self, _name: str, batches: list[list[tuple[int, float]]], dedupe: bool, expected: list[float]
+ ):
+ anchor = timezone.now().replace(microsecond=0) - dt.timedelta(minutes=5)
+ for batch in batches:
+ seed_metric_event(
+ team_id=self.team.id,
+ metric_name="m",
+ points=[(anchor + dt.timedelta(seconds=offset), value) for offset, value in batch],
+ trace_id=TRACE_A_B64,
+ )
+ with patch(
+ "products.metrics.backend.metric_query_runner.posthog_feature_flag_enabled", return_value=dedupe
+ ) as flag:
+ samples = list_metric_event_samples(
+ team=self.team,
+ metric_name="m",
+ date_from=anchor - dt.timedelta(hours=1),
+ date_to=anchor + dt.timedelta(hours=1),
+ )
+ self.assertEqual(sorted((s.value for s in samples), reverse=True), expected)
+ self.assertEqual(flag.call_args.args[0], "metrics-query-dedupe-points")
+
def test_samples_api_requires_authentication(self):
self.client.logout()
response = self.client.post(