Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(metrics): stop metrics4_view reading expired metrics2 data (#129134)

Frank Hamandbcfcf865d687
diff --git a/posthog/clickhouse/hcl/golden/dev/logs.hcl b/posthog/clickhouse/hcl/golden/dev/logs.hcl
index 03d8076731c5..06345bc6cccf 100644
--- a/posthog/clickhouse/hcl/golden/dev/logs.hcl
+++ b/posthog/clickhouse/hcl/golden/dev/logs.hcl
@@ -5048,38 +5048,6 @@ SQL
view "metrics4_view" {
query = <<SQL
-SELECT
- team_id,
- metric_name,
- time_bucket,
- series_fingerprint,
- resource_fingerprint,
- timestamp,
- original_expiry_timestamp,
- service_name,
- metric_type,
- value,
- count,
- histogram_bounds,
- histogram_counts,
- trace_id,
- span_id,
- trace_flags,
- has_labels,
- unit,
- aggregation_temporality,
- is_monotonic,
- instrumentation_scope
-FROM posthog.metrics2
-WHERE
- (time_bucket > toDateTime('2026-08-25 00:00:00'))
-AND
- (time_bucket < toDateTime('2026-09-14 00:00:00'))
-AND
- (timestamp > toDateTime('2026-08-25 00:00:00'))
-AND
- (timestamp < toDateTime('2026-09-14 00:00:00'))
-UNION ALL
SELECT
team_id,
metric_name,
diff --git a/posthog/clickhouse/hcl/golden/local-multi/logs.hcl b/posthog/clickhouse/hcl/golden/local-multi/logs.hcl
index f3bc0e118830..fcdd418a6401 100644
--- a/posthog/clickhouse/hcl/golden/local-multi/logs.hcl
+++ b/posthog/clickhouse/hcl/golden/local-multi/logs.hcl
@@ -4742,38 +4742,6 @@ SQL
view "metrics4_view" {
query = <<SQL
-SELECT
- team_id,
- metric_name,
- time_bucket,
- series_fingerprint,
- resource_fingerprint,
- timestamp,
- original_expiry_timestamp,
- service_name,
- metric_type,
- value,
- count,
- histogram_bounds,
- histogram_counts,
- trace_id,
- span_id,
- trace_flags,
- has_labels,
- unit,
- aggregation_temporality,
- is_monotonic,
- instrumentation_scope
-FROM posthog.metrics2
-WHERE
- (time_bucket > toDateTime('2026-08-25 00:00:00'))
-AND
- (time_bucket < toDateTime('2026-09-14 00:00:00'))
-AND
- (timestamp > toDateTime('2026-08-25 00:00:00'))
-AND
- (timestamp < toDateTime('2026-09-14 00:00:00'))
-UNION ALL
SELECT
team_id,
metric_name,
diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hcl
index 1cea290222d1..415381254058 100644
--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl
+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl
@@ -26531,38 +26531,6 @@ SQL
view "metrics4_view" {
query = <<SQL
-SELECT
- team_id,
- metric_name,
- time_bucket,
- series_fingerprint,
- resource_fingerprint,
- timestamp,
- original_expiry_timestamp,
- service_name,
- metric_type,
- value,
- count,
- histogram_bounds,
- histogram_counts,
- trace_id,
- span_id,
- trace_flags,
- has_labels,
- unit,
- aggregation_temporality,
- is_monotonic,
- instrumentation_scope
-FROM posthog.metrics2
-WHERE
- (time_bucket > toDateTime('2026-08-25 00:00:00'))
-AND
- (time_bucket < toDateTime('2026-09-14 00:00:00'))
-AND
- (timestamp > toDateTime('2026-08-25 00:00:00'))
-AND
- (timestamp < toDateTime('2026-09-14 00:00:00'))
-UNION ALL
SELECT
team_id,
metric_name,
diff --git a/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl b/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl
index eba94cd3001a..25507c27906c 100644
--- a/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl
@@ -5281,38 +5281,6 @@ SQL
view "metrics4_view" {
query = <<SQL
-SELECT
- team_id,
- metric_name,
- time_bucket,
- series_fingerprint,
- resource_fingerprint,
- timestamp,
- original_expiry_timestamp,
- service_name,
- metric_type,
- value,
- count,
- histogram_bounds,
- histogram_counts,
- trace_id,
- span_id,
- trace_flags,
- has_labels,
- unit,
- aggregation_temporality,
- is_monotonic,
- instrumentation_scope
-FROM posthog.metrics2
-WHERE
- (time_bucket > toDateTime('2026-08-25 00:00:00'))
-AND
- (time_bucket < toDateTime('2026-09-14 00:00:00'))
-AND
- (timestamp > toDateTime('2026-08-25 00:00:00'))
-AND
- (timestamp < toDateTime('2026-09-14 00:00:00'))
-UNION ALL
SELECT
team_id,
metric_name,
diff --git a/posthog/clickhouse/hcl/golden/prod-us/logs.hcl b/posthog/clickhouse/hcl/golden/prod-us/logs.hcl
index 4a5a2312bc1f..a22c718b2636 100644
--- a/posthog/clickhouse/hcl/golden/prod-us/logs.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-us/logs.hcl
@@ -5263,38 +5263,6 @@ SQL
view "metrics4_view" {
query = <<SQL
-SELECT
- team_id,
- metric_name,
- time_bucket,
- series_fingerprint,
- resource_fingerprint,
- timestamp,
- original_expiry_timestamp,
- service_name,
- metric_type,
- value,
- count,
- histogram_bounds,
- histogram_counts,
- trace_id,
- span_id,
- trace_flags,
- has_labels,
- unit,
- aggregation_temporality,
- is_monotonic,
- instrumentation_scope
-FROM posthog.metrics2
-WHERE
- (time_bucket > toDateTime('2026-08-25 00:00:00'))
-AND
- (time_bucket < toDateTime('2026-09-14 00:00:00'))
-AND
- (timestamp > toDateTime('2026-08-25 00:00:00'))
-AND
- (timestamp < toDateTime('2026-09-14 00:00:00'))
-UNION ALL
SELECT
team_id,
metric_name,
diff --git a/posthog/clickhouse/hcl/roles/logs/metrics/sql/metrics4_view.sql b/posthog/clickhouse/hcl/roles/logs/metrics/sql/metrics4_view.sql
index 466bc9f29726..ddbbc5c8df5e 100644
--- a/posthog/clickhouse/hcl/roles/logs/metrics/sql/metrics4_view.sql
+++ b/posthog/clickhouse/hcl/roles/logs/metrics/sql/metrics4_view.sql
@@ -1,35 +1,3 @@
-SELECT
- team_id,
- metric_name,
- time_bucket,
- series_fingerprint,
- resource_fingerprint,
- timestamp,
- original_expiry_timestamp,
- service_name,
- metric_type,
- value,
- count,
- histogram_bounds,
- histogram_counts,
- trace_id,
- span_id,
- trace_flags,
- has_labels,
- unit,
- aggregation_temporality,
- is_monotonic,
- instrumentation_scope
-FROM posthog.metrics2
-WHERE
- (time_bucket > toDateTime('2026-08-25 00:00:00'))
-AND
- (time_bucket < toDateTime('2026-09-14 00:00:00'))
-AND
- (timestamp > toDateTime('2026-08-25 00:00:00'))
-AND
- (timestamp < toDateTime('2026-09-14 00:00:00'))
-UNION ALL
SELECT
team_id,
metric_name,
diff --git a/posthog/clickhouse/hcl/sql/dev/logs.sql b/posthog/clickhouse/hcl/sql/dev/logs.sql
index 561cc73f2b95..3baa04ba4202 100644
--- a/posthog/clickhouse/hcl/sql/dev/logs.sql
+++ b/posthog/clickhouse/hcl/sql/dev/logs.sql
@@ -1564,38 +1564,6 @@ CREATE VIEW posthog.custom_metrics_test AS SELECT
'Test to check that the metric endpoint is working' AS help,
'gauge' AS type;
CREATE VIEW posthog.metrics4_view AS SELECT
- team_id,
- metric_name,
- time_bucket,
- series_fingerprint,
- resource_fingerprint,
- timestamp,
- original_expiry_timestamp,
- service_name,
- metric_type,
- value,
- count,
- histogram_bounds,
- histogram_counts,
- trace_id,
- span_id,
- trace_flags,
- has_labels,
- unit,
- aggregation_temporality,
- is_monotonic,
- instrumentation_scope
-FROM posthog.metrics2
-WHERE
- (time_bucket > toDateTime('2026-08-25 00:00:00'))
-AND
- (time_bucket < toDateTime('2026-09-14 00:00:00'))
-AND
- (timestamp > toDateTime('2026-08-25 00:00:00'))
-AND
- (timestamp < toDateTime('2026-09-14 00:00:00'))
-UNION ALL
-SELECT
team_id,
metric_name,
time_bucket,
diff --git a/posthog/clickhouse/hcl/sql/local-multi/logs.sql b/posthog/clickhouse/hcl/sql/local-multi/logs.sql
index c00f774c64a1..ee793346cfb4 100644
--- a/posthog/clickhouse/hcl/sql/local-multi/logs.sql
+++ b/posthog/clickhouse/hcl/sql/local-multi/logs.sql
@@ -1405,38 +1405,6 @@ FROM posthog.trace_spans
GROUP BY
_partition, _topic;
CREATE VIEW posthog.metrics4_view AS SELECT
- team_id,
- metric_name,
- time_bucket,
- series_fingerprint,
- resource_fingerprint,
- timestamp,
- original_expiry_timestamp,
- service_name,
- metric_type,
- value,
- count,
- histogram_bounds,
- histogram_counts,
- trace_id,
- span_id,
- trace_flags,
- has_labels,
- unit,
- aggregation_temporality,
- is_monotonic,
- instrumentation_scope
-FROM posthog.metrics2
-WHERE
- (time_bucket > toDateTime('2026-08-25 00:00:00'))
-AND
- (time_bucket < toDateTime('2026-09-14 00:00:00'))
-AND
- (timestamp > toDateTime('2026-08-25 00:00:00'))
-AND
- (timestamp < toDateTime('2026-09-14 00:00:00'))
-UNION ALL
-SELECT
team_id,
metric_name,
time_bucket,
diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sql
index 2a5c87ae4e2b..74203b0b5181 100644
--- a/posthog/clickhouse/hcl/sql/local-single/all.sql
+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql
@@ -6303,38 +6303,6 @@ CREATE VIEW posthog.custom_metrics_test AS SELECT
'Test to check that the metric endpoint is working' AS help,
'gauge' AS type;
CREATE VIEW posthog.metrics4_view AS SELECT
- team_id,
- metric_name,
- time_bucket,
- series_fingerprint,
- resource_fingerprint,
- timestamp,
- original_expiry_timestamp,
- service_name,
- metric_type,
- value,
- count,
- histogram_bounds,
- histogram_counts,
- trace_id,
- span_id,
- trace_flags,
- has_labels,
- unit,
- aggregation_temporality,
- is_monotonic,
- instrumentation_scope
-FROM posthog.metrics2
-WHERE
- (time_bucket > toDateTime('2026-08-25 00:00:00'))
-AND
- (time_bucket < toDateTime('2026-09-14 00:00:00'))
-AND
- (timestamp > toDateTime('2026-08-25 00:00:00'))
-AND
- (timestamp < toDateTime('2026-09-14 00:00:00'))
-UNION ALL
-SELECT
team_id,
metric_name,
time_bucket,
diff --git a/posthog/clickhouse/hcl/sql/prod-eu/logs.sql b/posthog/clickhouse/hcl/sql/prod-eu/logs.sql
index 0418fe664aa8..54d69e182b89 100644
--- a/posthog/clickhouse/hcl/sql/prod-eu/logs.sql
+++ b/posthog/clickhouse/hcl/sql/prod-eu/logs.sql
@@ -1641,38 +1641,6 @@ CREATE VIEW posthog.custom_metrics_test AS SELECT
'Test to check that the metric endpoint is working' AS help,
'gauge' AS type;
CREATE VIEW posthog.metrics4_view AS SELECT
- team_id,
- metric_name,
- time_bucket,
- series_fingerprint,
- resource_fingerprint,
- timestamp,
- original_expiry_timestamp,
- service_name,
- metric_type,
- value,
- count,
- histogram_bounds,
- histogram_counts,
- trace_id,
- span_id,
- trace_flags,
- has_labels,
- unit,
- aggregation_temporality,
- is_monotonic,
- instrumentation_scope
-FROM posthog.metrics2
-WHERE
- (time_bucket > toDateTime('2026-08-25 00:00:00'))
-AND
- (time_bucket < toDateTime('2026-09-14 00:00:00'))
-AND
- (timestamp > toDateTime('2026-08-25 00:00:00'))
-AND
- (timestamp < toDateTime('2026-09-14 00:00:00'))
-UNION ALL
-SELECT
team_id,
metric_name,
time_bucket,
diff --git a/posthog/clickhouse/hcl/sql/prod-us/logs.sql b/posthog/clickhouse/hcl/sql/prod-us/logs.sql
index b973b438921a..9da449575a4a 100644
--- a/posthog/clickhouse/hcl/sql/prod-us/logs.sql
+++ b/posthog/clickhouse/hcl/sql/prod-us/logs.sql
@@ -1635,38 +1635,6 @@ CREATE VIEW posthog.custom_metrics_test AS SELECT
'Test to check that the metric endpoint is working' AS help,
'gauge' AS type;
CREATE VIEW posthog.metrics4_view AS SELECT
- team_id,
- metric_name,
- time_bucket,
- series_fingerprint,
- resource_fingerprint,
- timestamp,
- original_expiry_timestamp,
- service_name,
- metric_type,
- value,
- count,
- histogram_bounds,
- histogram_counts,
- trace_id,
- span_id,
- trace_flags,
- has_labels,
- unit,
- aggregation_temporality,
- is_monotonic,
- instrumentation_scope
-FROM posthog.metrics2
-WHERE
- (time_bucket > toDateTime('2026-08-25 00:00:00'))
-AND
- (time_bucket < toDateTime('2026-09-14 00:00:00'))
-AND
- (timestamp > toDateTime('2026-08-25 00:00:00'))
-AND
- (timestamp < toDateTime('2026-09-14 00:00:00'))
-UNION ALL
-SELECT
team_id,
metric_name,
time_bucket,
diff --git a/posthog/clickhouse/metrics/metrics4.py b/posthog/clickhouse/metrics/metrics4.py
index 7e818e11d9f0..a5d08a8b1f34 100644
--- a/posthog/clickhouse/metrics/metrics4.py
+++ b/posthog/clickhouse/metrics/metrics4.py
@@ -49,9 +49,7 @@ WRITABLE_METRICS4_NAMES_TABLE_NAME = "writable_metrics4_names"
WRITABLE_METRICS4_ATTRIBUTES_TABLE_NAME = "writable_metrics4_attributes"
METRICS4_MAX_SAMPLES_PER_SERIES_HOUR = 10_000
METRICS4_VIEW_NAME = "metrics4_view"
-# metrics2 serves the hours before the cut-over. metrics4_samples serves the cut-over hour and later.
-# The view is temporary. After 2026-10-14, the 30-day retention removes all metrics2 data.
-METRICS4_VIEW_METRICS2_START = "2026-08-25 00:00:00"
+# The first hour that metrics4_samples serves. The view skips earlier rows, which overlap the old metrics2 data.
METRICS4_VIEW_CUTOVER = "2026-09-14 00:00:00"
# Each tuple maps an input column to its metrics4 array element type.
@@ -508,39 +506,11 @@ def METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV() -> str:
def METRICS4_VIEW_SQL() -> str:
- # The view gives the metrics2 row shape. ARRAY JOIN makes one row for each point.
+ # The view gives one row for each point, in the old metrics2 row shape. ARRAY JOIN expands the arrays.
# Partial rows of one series-hour hold different points, so no FINAL is necessary.
db = _db()
return f"""
CREATE OR REPLACE VIEW {db}.{METRICS4_VIEW_NAME} AS
-SELECT
- team_id,
- metric_name,
- time_bucket,
- series_fingerprint,
- resource_fingerprint,
- timestamp,
- original_expiry_timestamp,
- service_name,
- metric_type,
- value,
- count,
- histogram_bounds,
- histogram_counts,
- trace_id,
- span_id,
- trace_flags,
- has_labels,
- unit,
- aggregation_temporality,
- is_monotonic,
- instrumentation_scope
-FROM {db}.metrics2
-WHERE time_bucket > toDateTime('{METRICS4_VIEW_METRICS2_START}')
- AND time_bucket < toDateTime('{METRICS4_VIEW_CUTOVER}')
- AND timestamp > toDateTime('{METRICS4_VIEW_METRICS2_START}')
- AND timestamp < toDateTime('{METRICS4_VIEW_CUTOVER}')
-UNION ALL
SELECT
team_id,
metric_name,
diff --git a/posthog/clickhouse/migrations/0367_metrics4_view_drop_metrics2.py b/posthog/clickhouse/migrations/0367_metrics4_view_drop_metrics2.py
new file mode 100644
index 000000000000..26815ba06c39
--- /dev/null
+++ b/posthog/clickhouse/migrations/0367_metrics4_view_drop_metrics2.py
@@ -0,0 +1,7 @@
+from posthog.clickhouse.client.connection import NodeRole
+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions
+from posthog.clickhouse.metrics import METRICS4_VIEW_SQL
+
+operations = [
+ run_sql_with_exceptions(METRICS4_VIEW_SQL(), node_roles=[NodeRole.LOGS]),
+]
diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txt
index 27a09f6e6254..1f048aa29546 100644
--- a/posthog/clickhouse/migrations/max_migration.txt
+++ b/posthog/clickhouse/migrations/max_migration.txt
@@ -1 +1 @@
-0366_metrics4_stop_writing_observed_timestamp
+0367_metrics4_view_drop_metrics2