:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(metrics): split metrics4 sample rows by histogram bucket layout (#130947)
Frank Hamand646c939f711f
diff --git a/posthog/clickhouse/hcl/golden/dev/apm.hcl b/posthog/clickhouse/hcl/golden/dev/apm.hclindex 3a1dd2584fcb..3830945cc19d 100644--- a/posthog/clickhouse/hcl/golden/dev/apm.hcl+++ b/posthog/clickhouse/hcl/golden/dev/apm.hcl@@ -740,6 +740,9 @@ database "posthog" {column "histogram_bounds" {type = "SimpleAggregateFunction(anyLast, Array(Float64))"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "_topic" {type = "SimpleAggregateFunction(any, LowCardinality(String))"}@@ -1910,6 +1913,7 @@ SELECTmetric_name,toDateTime(toStartOfHour(timestamp)) AS time_bucket,series_fingerprint,+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,toDate32(original_expiry_timestamp) AS original_expiry_date,any(resource_fingerprint) AS resource_fingerprint,any(service_name) AS service_name,@@ -1928,9 +1932,9 @@ SELECTgroupArray(10000)(trace_id) AS trace_id_arr,groupArray(10000)(span_id) AS span_id_arr,groupArray(10000)(trace_flags) AS trace_flags_arr-FROM posthog.metrics4_input+FROM posthog.metrics4_input AS inputGROUP BY- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_dateSQLcolumn "team_id" {@@ -1945,6 +1949,9 @@ SQLcolumn "series_fingerprint" {type = "UInt64"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "original_expiry_date" {type = "Date32"}diff --git a/posthog/clickhouse/hcl/golden/dev/logs.hcl b/posthog/clickhouse/hcl/golden/dev/logs.hclindex c3c2b130ead0..0ec8f27cb9a9 100644--- a/posthog/clickhouse/hcl/golden/dev/logs.hcl+++ b/posthog/clickhouse/hcl/golden/dev/logs.hcl@@ -1930,7 +1930,8 @@ SQL}table "metrics4_samples" {- order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]+ primary_key = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]+ order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint", "histogram_bounds_hash"]partition_by = "original_expiry_date"ttl = "original_expiry_date"settings = {@@ -1981,6 +1982,9 @@ SQLcolumn "histogram_bounds" {type = "SimpleAggregateFunction(anyLast, Array(Float64))"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "_topic" {type = "SimpleAggregateFunction(any, LowCardinality(String))"}diff --git a/posthog/clickhouse/hcl/golden/local-multi/logs.hcl b/posthog/clickhouse/hcl/golden/local-multi/logs.hclindex a1ae75beea23..309f7452a525 100644--- a/posthog/clickhouse/hcl/golden/local-multi/logs.hcl+++ b/posthog/clickhouse/hcl/golden/local-multi/logs.hcl@@ -2431,7 +2431,8 @@ SQL}table "metrics4_samples" {- order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]+ primary_key = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]+ order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint", "histogram_bounds_hash"]partition_by = "original_expiry_date"ttl = "original_expiry_date"settings = {@@ -2482,6 +2483,9 @@ SQLcolumn "histogram_bounds" {type = "SimpleAggregateFunction(anyLast, Array(Float64))"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "_topic" {type = "SimpleAggregateFunction(any, LowCardinality(String))"}diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hclindex 24d199abe7bb..aa6b77baf1f0 100644--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl@@ -7914,7 +7914,8 @@ SQL}table "metrics4_samples" {- order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]+ primary_key = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]+ order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint", "histogram_bounds_hash"]partition_by = "original_expiry_date"ttl = "original_expiry_date"settings = {@@ -7965,6 +7966,9 @@ SQLcolumn "histogram_bounds" {type = "SimpleAggregateFunction(anyLast, Array(Float64))"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "_topic" {type = "SimpleAggregateFunction(any, LowCardinality(String))"}@@ -18239,6 +18243,9 @@ SQLcolumn "histogram_bounds" {type = "SimpleAggregateFunction(anyLast, Array(Float64))"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "_topic" {type = "SimpleAggregateFunction(any, LowCardinality(String))"}@@ -23210,6 +23217,7 @@ SELECTmetric_name,toDateTime(toStartOfHour(timestamp)) AS time_bucket,series_fingerprint,+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,toDate32(original_expiry_timestamp) AS original_expiry_date,any(resource_fingerprint) AS resource_fingerprint,any(service_name) AS service_name,@@ -23228,9 +23236,9 @@ SELECTgroupArray(10000)(trace_id) AS trace_id_arr,groupArray(10000)(span_id) AS span_id_arr,groupArray(10000)(trace_flags) AS trace_flags_arr-FROM posthog.metrics4_input+FROM posthog.metrics4_input AS inputGROUP BY- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_dateSQLcolumn "team_id" {@@ -23245,6 +23253,9 @@ SQLcolumn "series_fingerprint" {type = "UInt64"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "original_expiry_date" {type = "Date32"}diff --git a/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl b/posthog/clickhouse/hcl/golden/prod-eu/apm.hclindex 4eb0db91d364..edc72ead921e 100644--- a/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl+++ b/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl@@ -475,6 +475,9 @@ database "posthog" {column "histogram_bounds" {type = "SimpleAggregateFunction(anyLast, Array(Float64))"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "_topic" {type = "SimpleAggregateFunction(any, LowCardinality(String))"}@@ -1386,6 +1389,7 @@ SELECTmetric_name,toDateTime(toStartOfHour(timestamp)) AS time_bucket,series_fingerprint,+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,toDate32(original_expiry_timestamp) AS original_expiry_date,any(resource_fingerprint) AS resource_fingerprint,any(service_name) AS service_name,@@ -1404,9 +1408,9 @@ SELECTgroupArray(10000)(trace_id) AS trace_id_arr,groupArray(10000)(span_id) AS span_id_arr,groupArray(10000)(trace_flags) AS trace_flags_arr-FROM posthog.metrics4_input+FROM posthog.metrics4_input AS inputGROUP BY- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_dateSQLcolumn "team_id" {@@ -1421,6 +1425,9 @@ SQLcolumn "series_fingerprint" {type = "UInt64"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "original_expiry_date" {type = "Date32"}diff --git a/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl b/posthog/clickhouse/hcl/golden/prod-eu/logs.hclindex 4dee554e8c7d..b86c0c5990b3 100644--- a/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl+++ b/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl@@ -2017,7 +2017,8 @@ SQL}table "metrics4_samples" {- order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]+ primary_key = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]+ order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint", "histogram_bounds_hash"]partition_by = "original_expiry_date"ttl = "original_expiry_date"settings = {@@ -2068,6 +2069,9 @@ SQLcolumn "histogram_bounds" {type = "SimpleAggregateFunction(anyLast, Array(Float64))"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "_topic" {type = "SimpleAggregateFunction(any, LowCardinality(String))"}diff --git a/posthog/clickhouse/hcl/golden/prod-us/apm.hcl b/posthog/clickhouse/hcl/golden/prod-us/apm.hclindex 4eb0db91d364..edc72ead921e 100644--- a/posthog/clickhouse/hcl/golden/prod-us/apm.hcl+++ b/posthog/clickhouse/hcl/golden/prod-us/apm.hcl@@ -475,6 +475,9 @@ database "posthog" {column "histogram_bounds" {type = "SimpleAggregateFunction(anyLast, Array(Float64))"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "_topic" {type = "SimpleAggregateFunction(any, LowCardinality(String))"}@@ -1386,6 +1389,7 @@ SELECTmetric_name,toDateTime(toStartOfHour(timestamp)) AS time_bucket,series_fingerprint,+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,toDate32(original_expiry_timestamp) AS original_expiry_date,any(resource_fingerprint) AS resource_fingerprint,any(service_name) AS service_name,@@ -1404,9 +1408,9 @@ SELECTgroupArray(10000)(trace_id) AS trace_id_arr,groupArray(10000)(span_id) AS span_id_arr,groupArray(10000)(trace_flags) AS trace_flags_arr-FROM posthog.metrics4_input+FROM posthog.metrics4_input AS inputGROUP BY- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_dateSQLcolumn "team_id" {@@ -1421,6 +1425,9 @@ SQLcolumn "series_fingerprint" {type = "UInt64"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "original_expiry_date" {type = "Date32"}diff --git a/posthog/clickhouse/hcl/golden/prod-us/logs.hcl b/posthog/clickhouse/hcl/golden/prod-us/logs.hclindex 8917634c3724..a025448d011a 100644--- a/posthog/clickhouse/hcl/golden/prod-us/logs.hcl+++ b/posthog/clickhouse/hcl/golden/prod-us/logs.hcl@@ -2017,7 +2017,8 @@ SQL}table "metrics4_samples" {- order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]+ primary_key = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]+ order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint", "histogram_bounds_hash"]partition_by = "original_expiry_date"ttl = "original_expiry_date"settings = {@@ -2068,6 +2069,9 @@ SQLcolumn "histogram_bounds" {type = "SimpleAggregateFunction(anyLast, Array(Float64))"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "_topic" {type = "SimpleAggregateFunction(any, LowCardinality(String))"}diff --git a/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl b/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hclindex b667a1291f79..2a16f3112b9e 100644--- a/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl+++ b/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl@@ -90,6 +90,7 @@ database "posthog" {column "has_labels" { type = "SimpleAggregateFunction(max, UInt8)" }column "instrumentation_scope" { type = "SimpleAggregateFunction(any, String)" }column "histogram_bounds" { type = "SimpleAggregateFunction(anyLast, Array(Float64))" }+ column "histogram_bounds_hash" { type = "UInt64" }column "_topic" { type = "SimpleAggregateFunction(any, LowCardinality(String))" }column "timestamp_arr" { type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))" }column "observed_timestamp_arr" { type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))" }@@ -238,6 +239,7 @@ database "posthog" {column "metric_name" { type = "LowCardinality(String)" }column "time_bucket" { type = "DateTime" }column "series_fingerprint" { type = "UInt64" }+ column "histogram_bounds_hash" { type = "UInt64" }column "original_expiry_date" { type = "Date32" }column "resource_fingerprint" { type = "UInt64" }column "service_name" { type = "String" }diff --git a/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics4_samples.sql b/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics4_samples.sqlindex 08199089b679..792a69220ac4 100644--- a/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics4_samples.sql+++ b/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics4_samples.sql@@ -3,6 +3,7 @@ SELECTmetric_name,toDateTime(toStartOfHour(timestamp)) AS time_bucket,series_fingerprint,+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,toDate32(original_expiry_timestamp) AS original_expiry_date,any(resource_fingerprint) AS resource_fingerprint,any(service_name) AS service_name,@@ -21,10 +22,11 @@ SELECTgroupArray(10000)(trace_id) AS trace_id_arr,groupArray(10000)(span_id) AS span_id_arr,groupArray(10000)(trace_flags) AS trace_flags_arr-FROM posthog.metrics4_input+FROM posthog.metrics4_input AS inputGROUP BYteam_id,metric_name,time_bucket,series_fingerprint,+ histogram_bounds_hash,original_expiry_datediff --git a/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl b/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hclindex 11601df19120..58d93f237644 100644--- a/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl+++ b/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl@@ -109,7 +109,8 @@ database "posthog" {}table "metrics4_samples" {- order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]+ order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint", "histogram_bounds_hash"]+ primary_key = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]partition_by = "original_expiry_date"ttl = "original_expiry_date"settings = {@@ -160,6 +161,9 @@ database "posthog" {column "histogram_bounds" {type = "SimpleAggregateFunction(anyLast, Array(Float64))"}+ column "histogram_bounds_hash" {+ type = "UInt64"+ }column "_topic" {type = "SimpleAggregateFunction(any, LowCardinality(String))"}diff --git a/posthog/clickhouse/hcl/sql/dev/apm.sql b/posthog/clickhouse/hcl/sql/dev/apm.sqlindex 28f0245e4767..39b087543def 100644--- a/posthog/clickhouse/hcl/sql/dev/apm.sql+++ b/posthog/clickhouse/hcl/sql/dev/apm.sql@@ -224,6 +224,7 @@ CREATE TABLE posthog.writable_metrics4_samples (has_labels SimpleAggregateFunction(max, UInt8),instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),+ histogram_bounds_hash UInt64,_topic SimpleAggregateFunction(any, LowCardinality(String)),timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),@@ -611,11 +612,12 @@ FROMGROUP BYteam_id, metric_name, time_bucket, original_expiry_time_bucket, service_name, filtered_attributes);-CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECT+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, histogram_bounds_hash UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECTteam_id,metric_name,toDateTime(toStartOfHour(timestamp)) AS time_bucket,series_fingerprint,+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,toDate32(original_expiry_timestamp) AS original_expiry_date,any(resource_fingerprint) AS resource_fingerprint,any(service_name) AS service_name,@@ -634,9 +636,9 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.wgroupArray(10000)(trace_id) AS trace_id_arr,groupArray(10000)(span_id) AS span_id_arr,groupArray(10000)(trace_flags) AS trace_flags_arr-FROM posthog.metrics4_input+FROM posthog.metrics4_input AS inputGROUP BY- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date;+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_date;CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.writable_metrics4_series (team_id Int32, metric_name LowCardinality(String), series_fingerprint UInt64, metric_type LowCardinality(String), unit LowCardinality(String), aggregation_temporality LowCardinality(String), is_monotonic Bool, service_name LowCardinality(String), instrumentation_scope String, resource_attributes Map(LowCardinality(String), String), attributes Map(LowCardinality(String), String), timestamp DateTime64(6), original_expiry_timestamp DateTime64(6)) AS SELECTteam_id,metric_name,diff --git a/posthog/clickhouse/hcl/sql/dev/logs.sql b/posthog/clickhouse/hcl/sql/dev/logs.sqlindex 09b13baf542b..8a2bf4930dca 100644--- a/posthog/clickhouse/hcl/sql/dev/logs.sql+++ b/posthog/clickhouse/hcl/sql/dev/logs.sql@@ -498,6 +498,7 @@ CREATE TABLE posthog.metrics4_samples (has_labels SimpleAggregateFunction(max, UInt8),instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),+ histogram_bounds_hash UInt64,_topic SimpleAggregateFunction(any, LowCardinality(String)),timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),@@ -517,7 +518,7 @@ CREATE TABLE posthog.metrics4_samples (PROJECTION service_freshness_by_hour (SELECT team_id, time_bucket, service_name, max(arrayMax(timestamp_arr))GROUP BYteam_id, time_bucket, service_name)-) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') ORDER BY (team_id, metric_name, time_bucket, series_fingerprint) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;+) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint) ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;CREATE TABLE posthog.metrics4_series (team_id Int32,metric_name LowCardinality(String),diff --git a/posthog/clickhouse/hcl/sql/local-multi/logs.sql b/posthog/clickhouse/hcl/sql/local-multi/logs.sqlindex 7184ec2b9873..a03e6dc51e8e 100644--- a/posthog/clickhouse/hcl/sql/local-multi/logs.sql+++ b/posthog/clickhouse/hcl/sql/local-multi/logs.sql@@ -590,6 +590,7 @@ CREATE TABLE posthog.metrics4_samples (has_labels SimpleAggregateFunction(max, UInt8),instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),+ histogram_bounds_hash UInt64,_topic SimpleAggregateFunction(any, LowCardinality(String)),timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),@@ -609,7 +610,7 @@ CREATE TABLE posthog.metrics4_samples (PROJECTION service_freshness_by_hour (SELECT team_id, time_bucket, service_name, max(arrayMax(timestamp_arr))GROUP BYteam_id, time_bucket, service_name)-) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') ORDER BY (team_id, metric_name, time_bucket, series_fingerprint) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;+) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint) ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;CREATE TABLE posthog.metrics4_series (team_id Int32,metric_name LowCardinality(String),diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sqlindex ada095d71903..c5f92826dfbf 100644--- a/posthog/clickhouse/hcl/sql/local-single/all.sql+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql@@ -1610,6 +1610,7 @@ CREATE TABLE posthog.metrics4_samples (has_labels SimpleAggregateFunction(max, UInt8),instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),+ histogram_bounds_hash UInt64,_topic SimpleAggregateFunction(any, LowCardinality(String)),timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),@@ -1629,7 +1630,7 @@ CREATE TABLE posthog.metrics4_samples (PROJECTION service_freshness_by_hour (SELECT team_id, time_bucket, service_name, max(arrayMax(timestamp_arr))GROUP BYteam_id, time_bucket, service_name)-) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') ORDER BY (team_id, metric_name, time_bucket, series_fingerprint) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;+) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint) ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;CREATE TABLE posthog.metrics4_series (team_id Int32,metric_name LowCardinality(String),@@ -4016,6 +4017,7 @@ CREATE TABLE posthog.writable_metrics4_samples (has_labels SimpleAggregateFunction(max, UInt8),instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),+ histogram_bounds_hash UInt64,_topic SimpleAggregateFunction(any, LowCardinality(String)),timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),@@ -5404,11 +5406,12 @@ FROMGROUP BYteam_id, metric_name, time_bucket, original_expiry_time_bucket, service_name, filtered_attributes);-CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECT+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, histogram_bounds_hash UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECTteam_id,metric_name,toDateTime(toStartOfHour(timestamp)) AS time_bucket,series_fingerprint,+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,toDate32(original_expiry_timestamp) AS original_expiry_date,any(resource_fingerprint) AS resource_fingerprint,any(service_name) AS service_name,@@ -5427,9 +5430,9 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.wgroupArray(10000)(trace_id) AS trace_id_arr,groupArray(10000)(span_id) AS span_id_arr,groupArray(10000)(trace_flags) AS trace_flags_arr-FROM posthog.metrics4_input+FROM posthog.metrics4_input AS inputGROUP BY- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date;+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_date;CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.writable_metrics4_series (team_id Int32, metric_name LowCardinality(String), series_fingerprint UInt64, metric_type LowCardinality(String), unit LowCardinality(String), aggregation_temporality LowCardinality(String), is_monotonic Bool, service_name LowCardinality(String), instrumentation_scope String, resource_attributes Map(LowCardinality(String), String), attributes Map(LowCardinality(String), String), timestamp DateTime64(6), original_expiry_timestamp DateTime64(6)) AS SELECTteam_id,metric_name,diff --git a/posthog/clickhouse/hcl/sql/prod-eu/apm.sql b/posthog/clickhouse/hcl/sql/prod-eu/apm.sqlindex bbeec37e76e4..11453122ba53 100644--- a/posthog/clickhouse/hcl/sql/prod-eu/apm.sql+++ b/posthog/clickhouse/hcl/sql/prod-eu/apm.sql@@ -146,6 +146,7 @@ CREATE TABLE posthog.writable_metrics4_samples (has_labels SimpleAggregateFunction(max, UInt8),instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),+ histogram_bounds_hash UInt64,_topic SimpleAggregateFunction(any, LowCardinality(String)),timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),@@ -456,11 +457,12 @@ FROMGROUP BYteam_id, metric_name, time_bucket, original_expiry_time_bucket, service_name, filtered_attributes);-CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECT+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, histogram_bounds_hash UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECTteam_id,metric_name,toDateTime(toStartOfHour(timestamp)) AS time_bucket,series_fingerprint,+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,toDate32(original_expiry_timestamp) AS original_expiry_date,any(resource_fingerprint) AS resource_fingerprint,any(service_name) AS service_name,@@ -479,9 +481,9 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.wgroupArray(10000)(trace_id) AS trace_id_arr,groupArray(10000)(span_id) AS span_id_arr,groupArray(10000)(trace_flags) AS trace_flags_arr-FROM posthog.metrics4_input+FROM posthog.metrics4_input AS inputGROUP BY- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date;+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_date;CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.writable_metrics4_series (team_id Int32, metric_name LowCardinality(String), series_fingerprint UInt64, metric_type LowCardinality(String), unit LowCardinality(String), aggregation_temporality LowCardinality(String), is_monotonic Bool, service_name LowCardinality(String), instrumentation_scope String, resource_attributes Map(LowCardinality(String), String), attributes Map(LowCardinality(String), String), timestamp DateTime64(6), original_expiry_timestamp DateTime64(6)) AS SELECTteam_id,metric_name,diff --git a/posthog/clickhouse/hcl/sql/prod-eu/logs.sql b/posthog/clickhouse/hcl/sql/prod-eu/logs.sqlindex f90cfcb9b9b5..7946ccef47f7 100644--- a/posthog/clickhouse/hcl/sql/prod-eu/logs.sql+++ b/posthog/clickhouse/hcl/sql/prod-eu/logs.sql@@ -522,6 +522,7 @@ CREATE TABLE posthog.metrics4_samples (has_labels SimpleAggregateFunction(max, UInt8),instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),+ histogram_bounds_hash UInt64,_topic SimpleAggregateFunction(any, LowCardinality(String)),timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),@@ -541,7 +542,7 @@ CREATE TABLE posthog.metrics4_samples (PROJECTION service_freshness_by_hour (SELECT team_id, time_bucket, service_name, max(arrayMax(timestamp_arr))GROUP BYteam_id, time_bucket, service_name)-) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') ORDER BY (team_id, metric_name, time_bucket, series_fingerprint) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;+) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint) ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;CREATE TABLE posthog.metrics4_series (team_id Int32,metric_name LowCardinality(String),diff --git a/posthog/clickhouse/hcl/sql/prod-us/apm.sql b/posthog/clickhouse/hcl/sql/prod-us/apm.sqlindex 92e6f9ae1e8b..dfa7046fea8e 100644--- a/posthog/clickhouse/hcl/sql/prod-us/apm.sql+++ b/posthog/clickhouse/hcl/sql/prod-us/apm.sql@@ -146,6 +146,7 @@ CREATE TABLE posthog.writable_metrics4_samples (has_labels SimpleAggregateFunction(max, UInt8),instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),+ histogram_bounds_hash UInt64,_topic SimpleAggregateFunction(any, LowCardinality(String)),timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),@@ -456,11 +457,12 @@ FROMGROUP BYteam_id, metric_name, time_bucket, original_expiry_time_bucket, service_name, filtered_attributes);-CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECT+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, histogram_bounds_hash UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECTteam_id,metric_name,toDateTime(toStartOfHour(timestamp)) AS time_bucket,series_fingerprint,+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,toDate32(original_expiry_timestamp) AS original_expiry_date,any(resource_fingerprint) AS resource_fingerprint,any(service_name) AS service_name,@@ -479,9 +481,9 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.wgroupArray(10000)(trace_id) AS trace_id_arr,groupArray(10000)(span_id) AS span_id_arr,groupArray(10000)(trace_flags) AS trace_flags_arr-FROM posthog.metrics4_input+FROM posthog.metrics4_input AS inputGROUP BY- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date;+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_date;CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.writable_metrics4_series (team_id Int32, metric_name LowCardinality(String), series_fingerprint UInt64, metric_type LowCardinality(String), unit LowCardinality(String), aggregation_temporality LowCardinality(String), is_monotonic Bool, service_name LowCardinality(String), instrumentation_scope String, resource_attributes Map(LowCardinality(String), String), attributes Map(LowCardinality(String), String), timestamp DateTime64(6), original_expiry_timestamp DateTime64(6)) AS SELECTteam_id,metric_name,diff --git a/posthog/clickhouse/hcl/sql/prod-us/logs.sql b/posthog/clickhouse/hcl/sql/prod-us/logs.sqlindex 107a7f66d046..2cdc90a0c91e 100644--- a/posthog/clickhouse/hcl/sql/prod-us/logs.sql+++ b/posthog/clickhouse/hcl/sql/prod-us/logs.sql@@ -522,6 +522,7 @@ CREATE TABLE posthog.metrics4_samples (has_labels SimpleAggregateFunction(max, UInt8),instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),+ histogram_bounds_hash UInt64,_topic SimpleAggregateFunction(any, LowCardinality(String)),timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),@@ -541,7 +542,7 @@ CREATE TABLE posthog.metrics4_samples (PROJECTION service_freshness_by_hour (SELECT team_id, time_bucket, service_name, max(arrayMax(timestamp_arr))GROUP BYteam_id, time_bucket, service_name)-) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') ORDER BY (team_id, metric_name, time_bucket, series_fingerprint) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;+) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint) ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;CREATE TABLE posthog.metrics4_series (team_id Int32,metric_name LowCardinality(String),diff --git a/posthog/clickhouse/metrics/__init__.py b/posthog/clickhouse/metrics/__init__.pyindex bbd29551d8e0..fe9b273c200d 100644--- a/posthog/clickhouse/metrics/__init__.py+++ b/posthog/clickhouse/metrics/__init__.py@@ -52,6 +52,7 @@ from .metrics4 import (METRICS4_INPUT_TO_METRICS4_NAMES_MV,METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV,METRICS4_INPUT_TO_METRICS4_SAMPLES_MV,+ METRICS4_INPUT_TO_METRICS4_SAMPLES_MV_SELECT,METRICS4_INPUT_TO_METRICS4_SERIES_MV,METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV,METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT,@@ -82,6 +83,7 @@ __all__ = ["METRICS4_INPUT_TO_METRICS4_NAMES_MV","METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV","METRICS4_INPUT_TO_METRICS4_SAMPLES_MV",+ "METRICS4_INPUT_TO_METRICS4_SAMPLES_MV_SELECT","METRICS4_INPUT_TO_METRICS4_SERIES_MV","METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV","METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT",diff --git a/posthog/clickhouse/metrics/metrics4.py b/posthog/clickhouse/metrics/metrics4.pyindex 90531e03a668..dcd4c043378d 100644--- a/posthog/clickhouse/metrics/metrics4.py+++ b/posthog/clickhouse/metrics/metrics4.py@@ -141,6 +141,7 @@ def WRITABLE_METRICS4_SAMPLES_TABLE_SQL() -> str:`has_labels` SimpleAggregateFunction(max, UInt8),`instrumentation_scope` SimpleAggregateFunction(any, String),`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),+ `histogram_bounds_hash` UInt64,`_topic` SimpleAggregateFunction(any, LowCardinality(String)),{arrays}""",)@@ -234,6 +235,7 @@ CREATE TABLE IF NOT EXISTS {_db()}.{METRICS4_SAMPLES_TABLE_NAME}`has_labels` SimpleAggregateFunction(max, UInt8),`instrumentation_scope` SimpleAggregateFunction(any, String),`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),+ `histogram_bounds_hash` UInt64,`_topic` SimpleAggregateFunction(any, LowCardinality(String)),{arrays},`timestamp_min` DateTime64(6) ALIAS arrayMin(timestamp_arr),@@ -255,7 +257,8 @@ CREATE TABLE IF NOT EXISTS {_db()}.{METRICS4_SAMPLES_TABLE_NAME})ENGINE = {AggregatingMergeTree(METRICS4_SAMPLES_TABLE_NAME, replication_scheme=ReplicationScheme.REPLICATED)}PARTITION BY original_expiry_date-ORDER BY (team_id, metric_name, time_bucket, series_fingerprint)+PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint)+ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash)TTL original_expiry_dateSETTINGSindex_granularity = 128,@@ -362,20 +365,22 @@ SETTINGS"""-def METRICS4_INPUT_TO_METRICS4_SAMPLES_MV() -> str:- db = _db()+def METRICS4_INPUT_TO_METRICS4_SAMPLES_MV_SELECT() -> str:group_arrays = ",\n".join(f" groupArray({METRICS4_MAX_SAMPLES_PER_SERIES_HOUR})({name}) AS {name}_arr"for name, _ in METRICS4_POINT_ARRAY_COLUMNSif name not in METRICS4_UNWRITTEN_POINT_ARRAY_COLUMNS)- return f"""-CREATE MATERIALIZED VIEW IF NOT EXISTS {db}.{METRICS4_INPUT_TABLE_NAME}_to_{METRICS4_SAMPLES_TABLE_NAME} TO {db}.{WRITABLE_METRICS4_SAMPLES_TABLE_NAME}-AS SELECT+ # A series can change its histogram bucket layout inside one hour. Each layout gets its own row,+ # so the row's histogram_bounds are correct for every point in it. Points without bounds use hash 0,+ # so their rows merge with the rows that were written before the hash column existed.+ # The input. prefix reads the point column, not the series-hour aggregate of the same name.+ return f"""SELECTteam_id,metric_name,toDateTime(toStartOfHour(timestamp)) AS time_bucket,series_fingerprint,+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,toDate32(original_expiry_timestamp) AS original_expiry_date,any(resource_fingerprint) AS resource_fingerprint,any(service_name) AS service_name,@@ -388,13 +393,21 @@ AS SELECTanyLast(histogram_bounds) AS histogram_bounds,any(_topic) AS _topic,{group_arrays}-FROM {db}.{METRICS4_INPUT_TABLE_NAME}+FROM {_db()}.{METRICS4_INPUT_TABLE_NAME} AS inputGROUP BYteam_id,metric_name,time_bucket,series_fingerprint,- original_expiry_date+ histogram_bounds_hash,+ original_expiry_date"""+++def METRICS4_INPUT_TO_METRICS4_SAMPLES_MV() -> str:+ db = _db()+ return f"""+CREATE MATERIALIZED VIEW IF NOT EXISTS {db}.{METRICS4_INPUT_TABLE_NAME}_to_{METRICS4_SAMPLES_TABLE_NAME} TO {db}.{WRITABLE_METRICS4_SAMPLES_TABLE_NAME}+AS {METRICS4_INPUT_TO_METRICS4_SAMPLES_MV_SELECT()}"""diff --git a/posthog/clickhouse/migrations/0372_metrics4_samples_histogram_bounds_hash.py b/posthog/clickhouse/migrations/0372_metrics4_samples_histogram_bounds_hash.pynew file mode 100644index 000000000000..3757597e970f--- /dev/null+++ b/posthog/clickhouse/migrations/0372_metrics4_samples_histogram_bounds_hash.py@@ -0,0 +1,36 @@+from django.conf import settings++from posthog.clickhouse.client.connection import NodeRole+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions+from posthog.clickhouse.metrics import METRICS4_INPUT_TO_METRICS4_SAMPLES_MV_SELECT++DB = settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE++# The storage table and the writable table get the column before the view writes it.+# ClickHouse can add a new column to the end of the sorting key only in the ALTER that adds the column.+# The primary key does not change, so the primary index and the existing parts stay valid.+# Existing rows read the column as 0, which is the hash the view writes for points without bounds.+operations = [+ run_sql_with_exceptions(+ f"ALTER TABLE {DB}.metrics4_samples "+ "ADD COLUMN IF NOT EXISTS histogram_bounds_hash UInt64 AFTER histogram_bounds, "+ "MODIFY ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash)",+ node_roles=[NodeRole.LOGS],+ sharded=False,+ is_alter_on_replicated_table=True,+ ),+ run_sql_with_exceptions(+ f"ALTER TABLE {DB}.writable_metrics4_samples "+ "ADD COLUMN IF NOT EXISTS histogram_bounds_hash UInt64 AFTER histogram_bounds",+ node_roles=[NodeRole.APM],+ sharded=False,+ is_alter_on_replicated_table=False,+ ),+ run_sql_with_exceptions(+ f"ALTER TABLE {DB}.metrics4_input_to_metrics4_samples MODIFY QUERY\n"+ f"{METRICS4_INPUT_TO_METRICS4_SAMPLES_MV_SELECT()}",+ node_roles=[NodeRole.APM],+ sharded=False,+ is_alter_on_replicated_table=False,+ ),+]diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txtindex 7fb6c6b56b75..b210d2d0f9ce 100644--- a/posthog/clickhouse/migrations/max_migration.txt+++ b/posthog/clickhouse/migrations/max_migration.txt@@ -1 +1 @@-0371_metrics4_timestamp_delta_codec+0372_metrics4_samples_histogram_bounds_hashdiff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambrindex 0874fe13a141..818dd9392ab1 100644--- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr+++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr@@ -5346,6 +5346,7 @@metric_name,toDateTime(toStartOfHour(timestamp)) AS time_bucket,series_fingerprint,+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,toDate32(original_expiry_timestamp) AS original_expiry_date,any(resource_fingerprint) AS resource_fingerprint,any(service_name) AS service_name,@@ -5364,12 +5365,13 @@groupArray(10000)(trace_id) AS trace_id_arr,groupArray(10000)(span_id) AS span_id_arr,groupArray(10000)(trace_flags) AS trace_flags_arr- FROM posthog_test.metrics4_input+ FROM posthog_test.metrics4_input AS inputGROUP BYteam_id,metric_name,time_bucket,series_fingerprint,+ histogram_bounds_hash,original_expiry_date'''@@ -5463,6 +5465,7 @@`has_labels` SimpleAggregateFunction(max, UInt8),`instrumentation_scope` SimpleAggregateFunction(any, String),`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),+ `histogram_bounds_hash` UInt64,`_topic` SimpleAggregateFunction(any, LowCardinality(String)),`timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta, Default),`observed_timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),@@ -5491,7 +5494,8 @@)ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.metrics4_samples', '{replica}-{shard}')PARTITION BY original_expiry_date- ORDER BY (team_id, metric_name, time_bucket, series_fingerprint)+ PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint)+ ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash)TTL original_expiry_dateSETTINGSindex_granularity = 128,@@ -11245,6 +11249,7 @@`has_labels` SimpleAggregateFunction(max, UInt8),`instrumentation_scope` SimpleAggregateFunction(any, String),`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),+ `histogram_bounds_hash` UInt64,`_topic` SimpleAggregateFunction(any, LowCardinality(String)),`timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),`observed_timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),@@ -13335,6 +13340,7 @@`has_labels` SimpleAggregateFunction(max, UInt8),`instrumentation_scope` SimpleAggregateFunction(any, String),`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),+ `histogram_bounds_hash` UInt64,`_topic` SimpleAggregateFunction(any, LowCardinality(String)),`timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta, Default),`observed_timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),@@ -13363,7 +13369,8 @@)ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.metrics4_samples', '{replica}-{shard}')PARTITION BY original_expiry_date- ORDER BY (team_id, metric_name, time_bucket, series_fingerprint)+ PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint)+ ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash)TTL original_expiry_dateSETTINGSindex_granularity = 128,diff --git a/products/metrics/backend/tests/test_metric_query_runner.py b/products/metrics/backend/tests/test_metric_query_runner.pyindex 806c778a51c3..399f63bc5b19 100644--- a/products/metrics/backend/tests/test_metric_query_runner.py+++ b/products/metrics/backend/tests/test_metric_query_runner.py@@ -1639,15 +1639,23 @@ class TestHistogramQuantileRunner(ClickhouseTestMixin, APIBaseTest):earliest = dt.datetime.fromisoformat(rows[0]["time"]).astimezone(dt.UTC)self.assertEqual(earliest, self.anchor)- def test_mismatched_bounds_merge_onto_one_layout(self):+ @parameterized.expand(+ [+ ("two_series", {"resource_labels": {"k8s.pod.name": "other"}}),+ # One series changes its buckets inside one hour. The merge must keep each point on its own layout.+ ("one_series_after_merge", {}),+ ]+ )+ def test_mismatched_bounds_merge_onto_one_layout(self, _name, second_series_labels):# Two endpoints with their own buckets chart together instead of failing.self._seed_histogram([(self.anchor + dt.timedelta(seconds=0), [10, 0, 0, 0])], temporality="delta")self._seed_histogram([(self.anchor + dt.timedelta(seconds=10), [0, 10, 0, 0])],temporality="delta",bounds=[0.2, 0.6, 2.0],- resource_labels={"k8s.pod.name": "other"},+ **second_series_labels,)+ sync_execute("OPTIMIZE TABLE metrics4_samples FINAL")rows = self._run(0.75)self.assertEqual(len(rows), 1)# Union edges [0.1, 0.2, 0.5, 0.6, 1.0, 2.0] get counts [10, 0, 7.5, 2.5, 0, 0, 0], so p75 is 2/3 into (0.2, 0.5].