:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(metrics): stop metrics4_view reading expired metrics2 data (#129134)
Frank Hamandbcfcf865d687
diff --git a/posthog/clickhouse/hcl/golden/dev/logs.hcl b/posthog/clickhouse/hcl/golden/dev/logs.hclindex 03d8076731c5..06345bc6cccf 100644--- a/posthog/clickhouse/hcl/golden/dev/logs.hcl+++ b/posthog/clickhouse/hcl/golden/dev/logs.hcl@@ -5048,38 +5048,6 @@ SQLview "metrics4_view" {query = <<SQL-SELECT- team_id,- metric_name,- time_bucket,- series_fingerprint,- resource_fingerprint,- timestamp,- original_expiry_timestamp,- service_name,- metric_type,- value,- count,- histogram_bounds,- histogram_counts,- trace_id,- span_id,- trace_flags,- has_labels,- unit,- aggregation_temporality,- is_monotonic,- instrumentation_scope-FROM posthog.metrics2-WHERE- (time_bucket > toDateTime('2026-08-25 00:00:00'))-AND- (time_bucket < toDateTime('2026-09-14 00:00:00'))-AND- (timestamp > toDateTime('2026-08-25 00:00:00'))-AND- (timestamp < toDateTime('2026-09-14 00:00:00'))-UNION ALLSELECTteam_id,metric_name,diff --git a/posthog/clickhouse/hcl/golden/local-multi/logs.hcl b/posthog/clickhouse/hcl/golden/local-multi/logs.hclindex f3bc0e118830..fcdd418a6401 100644--- a/posthog/clickhouse/hcl/golden/local-multi/logs.hcl+++ b/posthog/clickhouse/hcl/golden/local-multi/logs.hcl@@ -4742,38 +4742,6 @@ SQLview "metrics4_view" {query = <<SQL-SELECT- team_id,- metric_name,- time_bucket,- series_fingerprint,- resource_fingerprint,- timestamp,- original_expiry_timestamp,- service_name,- metric_type,- value,- count,- histogram_bounds,- histogram_counts,- trace_id,- span_id,- trace_flags,- has_labels,- unit,- aggregation_temporality,- is_monotonic,- instrumentation_scope-FROM posthog.metrics2-WHERE- (time_bucket > toDateTime('2026-08-25 00:00:00'))-AND- (time_bucket < toDateTime('2026-09-14 00:00:00'))-AND- (timestamp > toDateTime('2026-08-25 00:00:00'))-AND- (timestamp < toDateTime('2026-09-14 00:00:00'))-UNION ALLSELECTteam_id,metric_name,diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hclindex 1cea290222d1..415381254058 100644--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl@@ -26531,38 +26531,6 @@ SQLview "metrics4_view" {query = <<SQL-SELECT- team_id,- metric_name,- time_bucket,- series_fingerprint,- resource_fingerprint,- timestamp,- original_expiry_timestamp,- service_name,- metric_type,- value,- count,- histogram_bounds,- histogram_counts,- trace_id,- span_id,- trace_flags,- has_labels,- unit,- aggregation_temporality,- is_monotonic,- instrumentation_scope-FROM posthog.metrics2-WHERE- (time_bucket > toDateTime('2026-08-25 00:00:00'))-AND- (time_bucket < toDateTime('2026-09-14 00:00:00'))-AND- (timestamp > toDateTime('2026-08-25 00:00:00'))-AND- (timestamp < toDateTime('2026-09-14 00:00:00'))-UNION ALLSELECTteam_id,metric_name,diff --git a/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl b/posthog/clickhouse/hcl/golden/prod-eu/logs.hclindex eba94cd3001a..25507c27906c 100644--- a/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl+++ b/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl@@ -5281,38 +5281,6 @@ SQLview "metrics4_view" {query = <<SQL-SELECT- team_id,- metric_name,- time_bucket,- series_fingerprint,- resource_fingerprint,- timestamp,- original_expiry_timestamp,- service_name,- metric_type,- value,- count,- histogram_bounds,- histogram_counts,- trace_id,- span_id,- trace_flags,- has_labels,- unit,- aggregation_temporality,- is_monotonic,- instrumentation_scope-FROM posthog.metrics2-WHERE- (time_bucket > toDateTime('2026-08-25 00:00:00'))-AND- (time_bucket < toDateTime('2026-09-14 00:00:00'))-AND- (timestamp > toDateTime('2026-08-25 00:00:00'))-AND- (timestamp < toDateTime('2026-09-14 00:00:00'))-UNION ALLSELECTteam_id,metric_name,diff --git a/posthog/clickhouse/hcl/golden/prod-us/logs.hcl b/posthog/clickhouse/hcl/golden/prod-us/logs.hclindex 4a5a2312bc1f..a22c718b2636 100644--- a/posthog/clickhouse/hcl/golden/prod-us/logs.hcl+++ b/posthog/clickhouse/hcl/golden/prod-us/logs.hcl@@ -5263,38 +5263,6 @@ SQLview "metrics4_view" {query = <<SQL-SELECT- team_id,- metric_name,- time_bucket,- series_fingerprint,- resource_fingerprint,- timestamp,- original_expiry_timestamp,- service_name,- metric_type,- value,- count,- histogram_bounds,- histogram_counts,- trace_id,- span_id,- trace_flags,- has_labels,- unit,- aggregation_temporality,- is_monotonic,- instrumentation_scope-FROM posthog.metrics2-WHERE- (time_bucket > toDateTime('2026-08-25 00:00:00'))-AND- (time_bucket < toDateTime('2026-09-14 00:00:00'))-AND- (timestamp > toDateTime('2026-08-25 00:00:00'))-AND- (timestamp < toDateTime('2026-09-14 00:00:00'))-UNION ALLSELECTteam_id,metric_name,diff --git a/posthog/clickhouse/hcl/roles/logs/metrics/sql/metrics4_view.sql b/posthog/clickhouse/hcl/roles/logs/metrics/sql/metrics4_view.sqlindex 466bc9f29726..ddbbc5c8df5e 100644--- a/posthog/clickhouse/hcl/roles/logs/metrics/sql/metrics4_view.sql+++ b/posthog/clickhouse/hcl/roles/logs/metrics/sql/metrics4_view.sql@@ -1,35 +1,3 @@-SELECT- team_id,- metric_name,- time_bucket,- series_fingerprint,- resource_fingerprint,- timestamp,- original_expiry_timestamp,- service_name,- metric_type,- value,- count,- histogram_bounds,- histogram_counts,- trace_id,- span_id,- trace_flags,- has_labels,- unit,- aggregation_temporality,- is_monotonic,- instrumentation_scope-FROM posthog.metrics2-WHERE- (time_bucket > toDateTime('2026-08-25 00:00:00'))-AND- (time_bucket < toDateTime('2026-09-14 00:00:00'))-AND- (timestamp > toDateTime('2026-08-25 00:00:00'))-AND- (timestamp < toDateTime('2026-09-14 00:00:00'))-UNION ALLSELECTteam_id,metric_name,diff --git a/posthog/clickhouse/hcl/sql/dev/logs.sql b/posthog/clickhouse/hcl/sql/dev/logs.sqlindex 561cc73f2b95..3baa04ba4202 100644--- a/posthog/clickhouse/hcl/sql/dev/logs.sql+++ b/posthog/clickhouse/hcl/sql/dev/logs.sql@@ -1564,38 +1564,6 @@ CREATE VIEW posthog.custom_metrics_test AS SELECT'Test to check that the metric endpoint is working' AS help,'gauge' AS type;CREATE VIEW posthog.metrics4_view AS SELECT- team_id,- metric_name,- time_bucket,- series_fingerprint,- resource_fingerprint,- timestamp,- original_expiry_timestamp,- service_name,- metric_type,- value,- count,- histogram_bounds,- histogram_counts,- trace_id,- span_id,- trace_flags,- has_labels,- unit,- aggregation_temporality,- is_monotonic,- instrumentation_scope-FROM posthog.metrics2-WHERE- (time_bucket > toDateTime('2026-08-25 00:00:00'))-AND- (time_bucket < toDateTime('2026-09-14 00:00:00'))-AND- (timestamp > toDateTime('2026-08-25 00:00:00'))-AND- (timestamp < toDateTime('2026-09-14 00:00:00'))-UNION ALL-SELECTteam_id,metric_name,time_bucket,diff --git a/posthog/clickhouse/hcl/sql/local-multi/logs.sql b/posthog/clickhouse/hcl/sql/local-multi/logs.sqlindex c00f774c64a1..ee793346cfb4 100644--- a/posthog/clickhouse/hcl/sql/local-multi/logs.sql+++ b/posthog/clickhouse/hcl/sql/local-multi/logs.sql@@ -1405,38 +1405,6 @@ FROM posthog.trace_spansGROUP BY_partition, _topic;CREATE VIEW posthog.metrics4_view AS SELECT- team_id,- metric_name,- time_bucket,- series_fingerprint,- resource_fingerprint,- timestamp,- original_expiry_timestamp,- service_name,- metric_type,- value,- count,- histogram_bounds,- histogram_counts,- trace_id,- span_id,- trace_flags,- has_labels,- unit,- aggregation_temporality,- is_monotonic,- instrumentation_scope-FROM posthog.metrics2-WHERE- (time_bucket > toDateTime('2026-08-25 00:00:00'))-AND- (time_bucket < toDateTime('2026-09-14 00:00:00'))-AND- (timestamp > toDateTime('2026-08-25 00:00:00'))-AND- (timestamp < toDateTime('2026-09-14 00:00:00'))-UNION ALL-SELECTteam_id,metric_name,time_bucket,diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sqlindex 2a5c87ae4e2b..74203b0b5181 100644--- a/posthog/clickhouse/hcl/sql/local-single/all.sql+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql@@ -6303,38 +6303,6 @@ CREATE VIEW posthog.custom_metrics_test AS SELECT'Test to check that the metric endpoint is working' AS help,'gauge' AS type;CREATE VIEW posthog.metrics4_view AS SELECT- team_id,- metric_name,- time_bucket,- series_fingerprint,- resource_fingerprint,- timestamp,- original_expiry_timestamp,- service_name,- metric_type,- value,- count,- histogram_bounds,- histogram_counts,- trace_id,- span_id,- trace_flags,- has_labels,- unit,- aggregation_temporality,- is_monotonic,- instrumentation_scope-FROM posthog.metrics2-WHERE- (time_bucket > toDateTime('2026-08-25 00:00:00'))-AND- (time_bucket < toDateTime('2026-09-14 00:00:00'))-AND- (timestamp > toDateTime('2026-08-25 00:00:00'))-AND- (timestamp < toDateTime('2026-09-14 00:00:00'))-UNION ALL-SELECTteam_id,metric_name,time_bucket,diff --git a/posthog/clickhouse/hcl/sql/prod-eu/logs.sql b/posthog/clickhouse/hcl/sql/prod-eu/logs.sqlindex 0418fe664aa8..54d69e182b89 100644--- a/posthog/clickhouse/hcl/sql/prod-eu/logs.sql+++ b/posthog/clickhouse/hcl/sql/prod-eu/logs.sql@@ -1641,38 +1641,6 @@ CREATE VIEW posthog.custom_metrics_test AS SELECT'Test to check that the metric endpoint is working' AS help,'gauge' AS type;CREATE VIEW posthog.metrics4_view AS SELECT- team_id,- metric_name,- time_bucket,- series_fingerprint,- resource_fingerprint,- timestamp,- original_expiry_timestamp,- service_name,- metric_type,- value,- count,- histogram_bounds,- histogram_counts,- trace_id,- span_id,- trace_flags,- has_labels,- unit,- aggregation_temporality,- is_monotonic,- instrumentation_scope-FROM posthog.metrics2-WHERE- (time_bucket > toDateTime('2026-08-25 00:00:00'))-AND- (time_bucket < toDateTime('2026-09-14 00:00:00'))-AND- (timestamp > toDateTime('2026-08-25 00:00:00'))-AND- (timestamp < toDateTime('2026-09-14 00:00:00'))-UNION ALL-SELECTteam_id,metric_name,time_bucket,diff --git a/posthog/clickhouse/hcl/sql/prod-us/logs.sql b/posthog/clickhouse/hcl/sql/prod-us/logs.sqlindex b973b438921a..9da449575a4a 100644--- a/posthog/clickhouse/hcl/sql/prod-us/logs.sql+++ b/posthog/clickhouse/hcl/sql/prod-us/logs.sql@@ -1635,38 +1635,6 @@ CREATE VIEW posthog.custom_metrics_test AS SELECT'Test to check that the metric endpoint is working' AS help,'gauge' AS type;CREATE VIEW posthog.metrics4_view AS SELECT- team_id,- metric_name,- time_bucket,- series_fingerprint,- resource_fingerprint,- timestamp,- original_expiry_timestamp,- service_name,- metric_type,- value,- count,- histogram_bounds,- histogram_counts,- trace_id,- span_id,- trace_flags,- has_labels,- unit,- aggregation_temporality,- is_monotonic,- instrumentation_scope-FROM posthog.metrics2-WHERE- (time_bucket > toDateTime('2026-08-25 00:00:00'))-AND- (time_bucket < toDateTime('2026-09-14 00:00:00'))-AND- (timestamp > toDateTime('2026-08-25 00:00:00'))-AND- (timestamp < toDateTime('2026-09-14 00:00:00'))-UNION ALL-SELECTteam_id,metric_name,time_bucket,diff --git a/posthog/clickhouse/metrics/metrics4.py b/posthog/clickhouse/metrics/metrics4.pyindex 7e818e11d9f0..a5d08a8b1f34 100644--- a/posthog/clickhouse/metrics/metrics4.py+++ b/posthog/clickhouse/metrics/metrics4.py@@ -49,9 +49,7 @@ WRITABLE_METRICS4_NAMES_TABLE_NAME = "writable_metrics4_names"WRITABLE_METRICS4_ATTRIBUTES_TABLE_NAME = "writable_metrics4_attributes"METRICS4_MAX_SAMPLES_PER_SERIES_HOUR = 10_000METRICS4_VIEW_NAME = "metrics4_view"-# metrics2 serves the hours before the cut-over. metrics4_samples serves the cut-over hour and later.-# The view is temporary. After 2026-10-14, the 30-day retention removes all metrics2 data.-METRICS4_VIEW_METRICS2_START = "2026-08-25 00:00:00"+# The first hour that metrics4_samples serves. The view skips earlier rows, which overlap the old metrics2 data.METRICS4_VIEW_CUTOVER = "2026-09-14 00:00:00"# Each tuple maps an input column to its metrics4 array element type.@@ -508,39 +506,11 @@ def METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV() -> str:def METRICS4_VIEW_SQL() -> str:- # The view gives the metrics2 row shape. ARRAY JOIN makes one row for each point.+ # The view gives one row for each point, in the old metrics2 row shape. ARRAY JOIN expands the arrays.# Partial rows of one series-hour hold different points, so no FINAL is necessary.db = _db()return f"""CREATE OR REPLACE VIEW {db}.{METRICS4_VIEW_NAME} AS-SELECT- team_id,- metric_name,- time_bucket,- series_fingerprint,- resource_fingerprint,- timestamp,- original_expiry_timestamp,- service_name,- metric_type,- value,- count,- histogram_bounds,- histogram_counts,- trace_id,- span_id,- trace_flags,- has_labels,- unit,- aggregation_temporality,- is_monotonic,- instrumentation_scope-FROM {db}.metrics2-WHERE time_bucket > toDateTime('{METRICS4_VIEW_METRICS2_START}')- AND time_bucket < toDateTime('{METRICS4_VIEW_CUTOVER}')- AND timestamp > toDateTime('{METRICS4_VIEW_METRICS2_START}')- AND timestamp < toDateTime('{METRICS4_VIEW_CUTOVER}')-UNION ALLSELECTteam_id,metric_name,diff --git a/posthog/clickhouse/migrations/0367_metrics4_view_drop_metrics2.py b/posthog/clickhouse/migrations/0367_metrics4_view_drop_metrics2.pynew file mode 100644index 000000000000..26815ba06c39--- /dev/null+++ b/posthog/clickhouse/migrations/0367_metrics4_view_drop_metrics2.py@@ -0,0 +1,7 @@+from posthog.clickhouse.client.connection import NodeRole+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions+from posthog.clickhouse.metrics import METRICS4_VIEW_SQL++operations = [+ run_sql_with_exceptions(METRICS4_VIEW_SQL(), node_roles=[NodeRole.LOGS]),+]diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txtindex 27a09f6e6254..1f048aa29546 100644--- a/posthog/clickhouse/migrations/max_migration.txt+++ b/posthog/clickhouse/migrations/max_migration.txt@@ -1 +1 @@-0366_metrics4_stop_writing_observed_timestamp+0367_metrics4_view_drop_metrics2