Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(metrics): split metrics4 sample rows by histogram bucket layout (#130947)

Frank Hamand646c939f711f
diff --git a/posthog/clickhouse/hcl/golden/dev/apm.hcl b/posthog/clickhouse/hcl/golden/dev/apm.hcl
index 3a1dd2584fcb..3830945cc19d 100644
--- a/posthog/clickhouse/hcl/golden/dev/apm.hcl
+++ b/posthog/clickhouse/hcl/golden/dev/apm.hcl
@@ -740,6 +740,9 @@ database "posthog" {
column "histogram_bounds" {
type = "SimpleAggregateFunction(anyLast, Array(Float64))"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "_topic" {
type = "SimpleAggregateFunction(any, LowCardinality(String))"
}
@@ -1910,6 +1913,7 @@ SELECT
metric_name,
toDateTime(toStartOfHour(timestamp)) AS time_bucket,
series_fingerprint,
+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,
toDate32(original_expiry_timestamp) AS original_expiry_date,
any(resource_fingerprint) AS resource_fingerprint,
any(service_name) AS service_name,
@@ -1928,9 +1932,9 @@ SELECT
groupArray(10000)(trace_id) AS trace_id_arr,
groupArray(10000)(span_id) AS span_id_arr,
groupArray(10000)(trace_flags) AS trace_flags_arr
-FROM posthog.metrics4_input
+FROM posthog.metrics4_input AS input
GROUP BY
- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date
+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_date
SQL
column "team_id" {
@@ -1945,6 +1949,9 @@ SQL
column "series_fingerprint" {
type = "UInt64"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "original_expiry_date" {
type = "Date32"
}
diff --git a/posthog/clickhouse/hcl/golden/dev/logs.hcl b/posthog/clickhouse/hcl/golden/dev/logs.hcl
index c3c2b130ead0..0ec8f27cb9a9 100644
--- a/posthog/clickhouse/hcl/golden/dev/logs.hcl
+++ b/posthog/clickhouse/hcl/golden/dev/logs.hcl
@@ -1930,7 +1930,8 @@ SQL
}
table "metrics4_samples" {
- order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]
+ primary_key = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]
+ order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint", "histogram_bounds_hash"]
partition_by = "original_expiry_date"
ttl = "original_expiry_date"
settings = {
@@ -1981,6 +1982,9 @@ SQL
column "histogram_bounds" {
type = "SimpleAggregateFunction(anyLast, Array(Float64))"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "_topic" {
type = "SimpleAggregateFunction(any, LowCardinality(String))"
}
diff --git a/posthog/clickhouse/hcl/golden/local-multi/logs.hcl b/posthog/clickhouse/hcl/golden/local-multi/logs.hcl
index a1ae75beea23..309f7452a525 100644
--- a/posthog/clickhouse/hcl/golden/local-multi/logs.hcl
+++ b/posthog/clickhouse/hcl/golden/local-multi/logs.hcl
@@ -2431,7 +2431,8 @@ SQL
}
table "metrics4_samples" {
- order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]
+ primary_key = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]
+ order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint", "histogram_bounds_hash"]
partition_by = "original_expiry_date"
ttl = "original_expiry_date"
settings = {
@@ -2482,6 +2483,9 @@ SQL
column "histogram_bounds" {
type = "SimpleAggregateFunction(anyLast, Array(Float64))"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "_topic" {
type = "SimpleAggregateFunction(any, LowCardinality(String))"
}
diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hcl
index 24d199abe7bb..aa6b77baf1f0 100644
--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl
+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl
@@ -7914,7 +7914,8 @@ SQL
}
table "metrics4_samples" {
- order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]
+ primary_key = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]
+ order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint", "histogram_bounds_hash"]
partition_by = "original_expiry_date"
ttl = "original_expiry_date"
settings = {
@@ -7965,6 +7966,9 @@ SQL
column "histogram_bounds" {
type = "SimpleAggregateFunction(anyLast, Array(Float64))"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "_topic" {
type = "SimpleAggregateFunction(any, LowCardinality(String))"
}
@@ -18239,6 +18243,9 @@ SQL
column "histogram_bounds" {
type = "SimpleAggregateFunction(anyLast, Array(Float64))"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "_topic" {
type = "SimpleAggregateFunction(any, LowCardinality(String))"
}
@@ -23210,6 +23217,7 @@ SELECT
metric_name,
toDateTime(toStartOfHour(timestamp)) AS time_bucket,
series_fingerprint,
+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,
toDate32(original_expiry_timestamp) AS original_expiry_date,
any(resource_fingerprint) AS resource_fingerprint,
any(service_name) AS service_name,
@@ -23228,9 +23236,9 @@ SELECT
groupArray(10000)(trace_id) AS trace_id_arr,
groupArray(10000)(span_id) AS span_id_arr,
groupArray(10000)(trace_flags) AS trace_flags_arr
-FROM posthog.metrics4_input
+FROM posthog.metrics4_input AS input
GROUP BY
- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date
+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_date
SQL
column "team_id" {
@@ -23245,6 +23253,9 @@ SQL
column "series_fingerprint" {
type = "UInt64"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "original_expiry_date" {
type = "Date32"
}
diff --git a/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl b/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl
index 4eb0db91d364..edc72ead921e 100644
--- a/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl
@@ -475,6 +475,9 @@ database "posthog" {
column "histogram_bounds" {
type = "SimpleAggregateFunction(anyLast, Array(Float64))"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "_topic" {
type = "SimpleAggregateFunction(any, LowCardinality(String))"
}
@@ -1386,6 +1389,7 @@ SELECT
metric_name,
toDateTime(toStartOfHour(timestamp)) AS time_bucket,
series_fingerprint,
+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,
toDate32(original_expiry_timestamp) AS original_expiry_date,
any(resource_fingerprint) AS resource_fingerprint,
any(service_name) AS service_name,
@@ -1404,9 +1408,9 @@ SELECT
groupArray(10000)(trace_id) AS trace_id_arr,
groupArray(10000)(span_id) AS span_id_arr,
groupArray(10000)(trace_flags) AS trace_flags_arr
-FROM posthog.metrics4_input
+FROM posthog.metrics4_input AS input
GROUP BY
- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date
+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_date
SQL
column "team_id" {
@@ -1421,6 +1425,9 @@ SQL
column "series_fingerprint" {
type = "UInt64"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "original_expiry_date" {
type = "Date32"
}
diff --git a/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl b/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl
index 4dee554e8c7d..b86c0c5990b3 100644
--- a/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl
@@ -2017,7 +2017,8 @@ SQL
}
table "metrics4_samples" {
- order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]
+ primary_key = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]
+ order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint", "histogram_bounds_hash"]
partition_by = "original_expiry_date"
ttl = "original_expiry_date"
settings = {
@@ -2068,6 +2069,9 @@ SQL
column "histogram_bounds" {
type = "SimpleAggregateFunction(anyLast, Array(Float64))"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "_topic" {
type = "SimpleAggregateFunction(any, LowCardinality(String))"
}
diff --git a/posthog/clickhouse/hcl/golden/prod-us/apm.hcl b/posthog/clickhouse/hcl/golden/prod-us/apm.hcl
index 4eb0db91d364..edc72ead921e 100644
--- a/posthog/clickhouse/hcl/golden/prod-us/apm.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-us/apm.hcl
@@ -475,6 +475,9 @@ database "posthog" {
column "histogram_bounds" {
type = "SimpleAggregateFunction(anyLast, Array(Float64))"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "_topic" {
type = "SimpleAggregateFunction(any, LowCardinality(String))"
}
@@ -1386,6 +1389,7 @@ SELECT
metric_name,
toDateTime(toStartOfHour(timestamp)) AS time_bucket,
series_fingerprint,
+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,
toDate32(original_expiry_timestamp) AS original_expiry_date,
any(resource_fingerprint) AS resource_fingerprint,
any(service_name) AS service_name,
@@ -1404,9 +1408,9 @@ SELECT
groupArray(10000)(trace_id) AS trace_id_arr,
groupArray(10000)(span_id) AS span_id_arr,
groupArray(10000)(trace_flags) AS trace_flags_arr
-FROM posthog.metrics4_input
+FROM posthog.metrics4_input AS input
GROUP BY
- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date
+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_date
SQL
column "team_id" {
@@ -1421,6 +1425,9 @@ SQL
column "series_fingerprint" {
type = "UInt64"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "original_expiry_date" {
type = "Date32"
}
diff --git a/posthog/clickhouse/hcl/golden/prod-us/logs.hcl b/posthog/clickhouse/hcl/golden/prod-us/logs.hcl
index 8917634c3724..a025448d011a 100644
--- a/posthog/clickhouse/hcl/golden/prod-us/logs.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-us/logs.hcl
@@ -2017,7 +2017,8 @@ SQL
}
table "metrics4_samples" {
- order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]
+ primary_key = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]
+ order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint", "histogram_bounds_hash"]
partition_by = "original_expiry_date"
ttl = "original_expiry_date"
settings = {
@@ -2068,6 +2069,9 @@ SQL
column "histogram_bounds" {
type = "SimpleAggregateFunction(anyLast, Array(Float64))"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "_topic" {
type = "SimpleAggregateFunction(any, LowCardinality(String))"
}
diff --git a/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl b/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl
index b667a1291f79..2a16f3112b9e 100644
--- a/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl
+++ b/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl
@@ -90,6 +90,7 @@ database "posthog" {
column "has_labels" { type = "SimpleAggregateFunction(max, UInt8)" }
column "instrumentation_scope" { type = "SimpleAggregateFunction(any, String)" }
column "histogram_bounds" { type = "SimpleAggregateFunction(anyLast, Array(Float64))" }
+ column "histogram_bounds_hash" { type = "UInt64" }
column "_topic" { type = "SimpleAggregateFunction(any, LowCardinality(String))" }
column "timestamp_arr" { type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))" }
column "observed_timestamp_arr" { type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))" }
@@ -238,6 +239,7 @@ database "posthog" {
column "metric_name" { type = "LowCardinality(String)" }
column "time_bucket" { type = "DateTime" }
column "series_fingerprint" { type = "UInt64" }
+ column "histogram_bounds_hash" { type = "UInt64" }
column "original_expiry_date" { type = "Date32" }
column "resource_fingerprint" { type = "UInt64" }
column "service_name" { type = "String" }
diff --git a/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics4_samples.sql b/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics4_samples.sql
index 08199089b679..792a69220ac4 100644
--- a/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics4_samples.sql
+++ b/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics4_samples.sql
@@ -3,6 +3,7 @@ SELECT
metric_name,
toDateTime(toStartOfHour(timestamp)) AS time_bucket,
series_fingerprint,
+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,
toDate32(original_expiry_timestamp) AS original_expiry_date,
any(resource_fingerprint) AS resource_fingerprint,
any(service_name) AS service_name,
@@ -21,10 +22,11 @@ SELECT
groupArray(10000)(trace_id) AS trace_id_arr,
groupArray(10000)(span_id) AS span_id_arr,
groupArray(10000)(trace_flags) AS trace_flags_arr
-FROM posthog.metrics4_input
+FROM posthog.metrics4_input AS input
GROUP BY
team_id,
metric_name,
time_bucket,
series_fingerprint,
+ histogram_bounds_hash,
original_expiry_date
diff --git a/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl b/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl
index 11601df19120..58d93f237644 100644
--- a/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl
+++ b/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl
@@ -109,7 +109,8 @@ database "posthog" {
}
table "metrics4_samples" {
- order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]
+ order_by = ["team_id", "metric_name", "time_bucket", "series_fingerprint", "histogram_bounds_hash"]
+ primary_key = ["team_id", "metric_name", "time_bucket", "series_fingerprint"]
partition_by = "original_expiry_date"
ttl = "original_expiry_date"
settings = {
@@ -160,6 +161,9 @@ database "posthog" {
column "histogram_bounds" {
type = "SimpleAggregateFunction(anyLast, Array(Float64))"
}
+ column "histogram_bounds_hash" {
+ type = "UInt64"
+ }
column "_topic" {
type = "SimpleAggregateFunction(any, LowCardinality(String))"
}
diff --git a/posthog/clickhouse/hcl/sql/dev/apm.sql b/posthog/clickhouse/hcl/sql/dev/apm.sql
index 28f0245e4767..39b087543def 100644
--- a/posthog/clickhouse/hcl/sql/dev/apm.sql
+++ b/posthog/clickhouse/hcl/sql/dev/apm.sql
@@ -224,6 +224,7 @@ CREATE TABLE posthog.writable_metrics4_samples (
has_labels SimpleAggregateFunction(max, UInt8),
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
+ histogram_bounds_hash UInt64,
_topic SimpleAggregateFunction(any, LowCardinality(String)),
timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),
@@ -611,11 +612,12 @@ FROM
GROUP BY
team_id, metric_name, time_bucket, original_expiry_time_bucket, service_name, filtered_attributes
);
-CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECT
+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, histogram_bounds_hash UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECT
team_id,
metric_name,
toDateTime(toStartOfHour(timestamp)) AS time_bucket,
series_fingerprint,
+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,
toDate32(original_expiry_timestamp) AS original_expiry_date,
any(resource_fingerprint) AS resource_fingerprint,
any(service_name) AS service_name,
@@ -634,9 +636,9 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.w
groupArray(10000)(trace_id) AS trace_id_arr,
groupArray(10000)(span_id) AS span_id_arr,
groupArray(10000)(trace_flags) AS trace_flags_arr
-FROM posthog.metrics4_input
+FROM posthog.metrics4_input AS input
GROUP BY
- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date;
+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_date;
CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.writable_metrics4_series (team_id Int32, metric_name LowCardinality(String), series_fingerprint UInt64, metric_type LowCardinality(String), unit LowCardinality(String), aggregation_temporality LowCardinality(String), is_monotonic Bool, service_name LowCardinality(String), instrumentation_scope String, resource_attributes Map(LowCardinality(String), String), attributes Map(LowCardinality(String), String), timestamp DateTime64(6), original_expiry_timestamp DateTime64(6)) AS SELECT
team_id,
metric_name,
diff --git a/posthog/clickhouse/hcl/sql/dev/logs.sql b/posthog/clickhouse/hcl/sql/dev/logs.sql
index 09b13baf542b..8a2bf4930dca 100644
--- a/posthog/clickhouse/hcl/sql/dev/logs.sql
+++ b/posthog/clickhouse/hcl/sql/dev/logs.sql
@@ -498,6 +498,7 @@ CREATE TABLE posthog.metrics4_samples (
has_labels SimpleAggregateFunction(max, UInt8),
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
+ histogram_bounds_hash UInt64,
_topic SimpleAggregateFunction(any, LowCardinality(String)),
timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
@@ -517,7 +518,7 @@ CREATE TABLE posthog.metrics4_samples (
PROJECTION service_freshness_by_hour (SELECT team_id, time_bucket, service_name, max(arrayMax(timestamp_arr))
GROUP BY
team_id, time_bucket, service_name)
-) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') ORDER BY (team_id, metric_name, time_bucket, series_fingerprint) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;
+) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint) ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;
CREATE TABLE posthog.metrics4_series (
team_id Int32,
metric_name LowCardinality(String),
diff --git a/posthog/clickhouse/hcl/sql/local-multi/logs.sql b/posthog/clickhouse/hcl/sql/local-multi/logs.sql
index 7184ec2b9873..a03e6dc51e8e 100644
--- a/posthog/clickhouse/hcl/sql/local-multi/logs.sql
+++ b/posthog/clickhouse/hcl/sql/local-multi/logs.sql
@@ -590,6 +590,7 @@ CREATE TABLE posthog.metrics4_samples (
has_labels SimpleAggregateFunction(max, UInt8),
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
+ histogram_bounds_hash UInt64,
_topic SimpleAggregateFunction(any, LowCardinality(String)),
timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
@@ -609,7 +610,7 @@ CREATE TABLE posthog.metrics4_samples (
PROJECTION service_freshness_by_hour (SELECT team_id, time_bucket, service_name, max(arrayMax(timestamp_arr))
GROUP BY
team_id, time_bucket, service_name)
-) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') ORDER BY (team_id, metric_name, time_bucket, series_fingerprint) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;
+) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint) ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;
CREATE TABLE posthog.metrics4_series (
team_id Int32,
metric_name LowCardinality(String),
diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sql
index ada095d71903..c5f92826dfbf 100644
--- a/posthog/clickhouse/hcl/sql/local-single/all.sql
+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql
@@ -1610,6 +1610,7 @@ CREATE TABLE posthog.metrics4_samples (
has_labels SimpleAggregateFunction(max, UInt8),
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
+ histogram_bounds_hash UInt64,
_topic SimpleAggregateFunction(any, LowCardinality(String)),
timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
@@ -1629,7 +1630,7 @@ CREATE TABLE posthog.metrics4_samples (
PROJECTION service_freshness_by_hour (SELECT team_id, time_bucket, service_name, max(arrayMax(timestamp_arr))
GROUP BY
team_id, time_bucket, service_name)
-) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') ORDER BY (team_id, metric_name, time_bucket, series_fingerprint) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;
+) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint) ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;
CREATE TABLE posthog.metrics4_series (
team_id Int32,
metric_name LowCardinality(String),
@@ -4016,6 +4017,7 @@ CREATE TABLE posthog.writable_metrics4_samples (
has_labels SimpleAggregateFunction(max, UInt8),
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
+ histogram_bounds_hash UInt64,
_topic SimpleAggregateFunction(any, LowCardinality(String)),
timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),
@@ -5404,11 +5406,12 @@ FROM
GROUP BY
team_id, metric_name, time_bucket, original_expiry_time_bucket, service_name, filtered_attributes
);
-CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECT
+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, histogram_bounds_hash UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECT
team_id,
metric_name,
toDateTime(toStartOfHour(timestamp)) AS time_bucket,
series_fingerprint,
+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,
toDate32(original_expiry_timestamp) AS original_expiry_date,
any(resource_fingerprint) AS resource_fingerprint,
any(service_name) AS service_name,
@@ -5427,9 +5430,9 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.w
groupArray(10000)(trace_id) AS trace_id_arr,
groupArray(10000)(span_id) AS span_id_arr,
groupArray(10000)(trace_flags) AS trace_flags_arr
-FROM posthog.metrics4_input
+FROM posthog.metrics4_input AS input
GROUP BY
- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date;
+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_date;
CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.writable_metrics4_series (team_id Int32, metric_name LowCardinality(String), series_fingerprint UInt64, metric_type LowCardinality(String), unit LowCardinality(String), aggregation_temporality LowCardinality(String), is_monotonic Bool, service_name LowCardinality(String), instrumentation_scope String, resource_attributes Map(LowCardinality(String), String), attributes Map(LowCardinality(String), String), timestamp DateTime64(6), original_expiry_timestamp DateTime64(6)) AS SELECT
team_id,
metric_name,
diff --git a/posthog/clickhouse/hcl/sql/prod-eu/apm.sql b/posthog/clickhouse/hcl/sql/prod-eu/apm.sql
index bbeec37e76e4..11453122ba53 100644
--- a/posthog/clickhouse/hcl/sql/prod-eu/apm.sql
+++ b/posthog/clickhouse/hcl/sql/prod-eu/apm.sql
@@ -146,6 +146,7 @@ CREATE TABLE posthog.writable_metrics4_samples (
has_labels SimpleAggregateFunction(max, UInt8),
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
+ histogram_bounds_hash UInt64,
_topic SimpleAggregateFunction(any, LowCardinality(String)),
timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),
@@ -456,11 +457,12 @@ FROM
GROUP BY
team_id, metric_name, time_bucket, original_expiry_time_bucket, service_name, filtered_attributes
);
-CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECT
+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, histogram_bounds_hash UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECT
team_id,
metric_name,
toDateTime(toStartOfHour(timestamp)) AS time_bucket,
series_fingerprint,
+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,
toDate32(original_expiry_timestamp) AS original_expiry_date,
any(resource_fingerprint) AS resource_fingerprint,
any(service_name) AS service_name,
@@ -479,9 +481,9 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.w
groupArray(10000)(trace_id) AS trace_id_arr,
groupArray(10000)(span_id) AS span_id_arr,
groupArray(10000)(trace_flags) AS trace_flags_arr
-FROM posthog.metrics4_input
+FROM posthog.metrics4_input AS input
GROUP BY
- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date;
+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_date;
CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.writable_metrics4_series (team_id Int32, metric_name LowCardinality(String), series_fingerprint UInt64, metric_type LowCardinality(String), unit LowCardinality(String), aggregation_temporality LowCardinality(String), is_monotonic Bool, service_name LowCardinality(String), instrumentation_scope String, resource_attributes Map(LowCardinality(String), String), attributes Map(LowCardinality(String), String), timestamp DateTime64(6), original_expiry_timestamp DateTime64(6)) AS SELECT
team_id,
metric_name,
diff --git a/posthog/clickhouse/hcl/sql/prod-eu/logs.sql b/posthog/clickhouse/hcl/sql/prod-eu/logs.sql
index f90cfcb9b9b5..7946ccef47f7 100644
--- a/posthog/clickhouse/hcl/sql/prod-eu/logs.sql
+++ b/posthog/clickhouse/hcl/sql/prod-eu/logs.sql
@@ -522,6 +522,7 @@ CREATE TABLE posthog.metrics4_samples (
has_labels SimpleAggregateFunction(max, UInt8),
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
+ histogram_bounds_hash UInt64,
_topic SimpleAggregateFunction(any, LowCardinality(String)),
timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
@@ -541,7 +542,7 @@ CREATE TABLE posthog.metrics4_samples (
PROJECTION service_freshness_by_hour (SELECT team_id, time_bucket, service_name, max(arrayMax(timestamp_arr))
GROUP BY
team_id, time_bucket, service_name)
-) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') ORDER BY (team_id, metric_name, time_bucket, series_fingerprint) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;
+) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint) ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;
CREATE TABLE posthog.metrics4_series (
team_id Int32,
metric_name LowCardinality(String),
diff --git a/posthog/clickhouse/hcl/sql/prod-us/apm.sql b/posthog/clickhouse/hcl/sql/prod-us/apm.sql
index 92e6f9ae1e8b..dfa7046fea8e 100644
--- a/posthog/clickhouse/hcl/sql/prod-us/apm.sql
+++ b/posthog/clickhouse/hcl/sql/prod-us/apm.sql
@@ -146,6 +146,7 @@ CREATE TABLE posthog.writable_metrics4_samples (
has_labels SimpleAggregateFunction(max, UInt8),
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
+ histogram_bounds_hash UInt64,
_topic SimpleAggregateFunction(any, LowCardinality(String)),
timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),
@@ -456,11 +457,12 @@ FROM
GROUP BY
team_id, metric_name, time_bucket, original_expiry_time_bucket, service_name, filtered_attributes
);
-CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECT
+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.writable_metrics4_samples (team_id Int32, metric_name LowCardinality(String), time_bucket DateTime, series_fingerprint UInt64, histogram_bounds_hash UInt64, original_expiry_date Date32, resource_fingerprint UInt64, service_name String, metric_type String, unit String, aggregation_temporality String, is_monotonic UInt8, has_labels UInt8, instrumentation_scope String, histogram_bounds Array(Float64), _topic String, timestamp_arr Array(DateTime64(6)), value_arr Array(Float64), count_arr Array(UInt64), histogram_counts_arr Array(Array(UInt64)), trace_id_arr Array(String), span_id_arr Array(String), trace_flags_arr Array(Int32)) AS SELECT
team_id,
metric_name,
toDateTime(toStartOfHour(timestamp)) AS time_bucket,
series_fingerprint,
+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,
toDate32(original_expiry_timestamp) AS original_expiry_date,
any(resource_fingerprint) AS resource_fingerprint,
any(service_name) AS service_name,
@@ -479,9 +481,9 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_samples TO posthog.w
groupArray(10000)(trace_id) AS trace_id_arr,
groupArray(10000)(span_id) AS span_id_arr,
groupArray(10000)(trace_flags) AS trace_flags_arr
-FROM posthog.metrics4_input
+FROM posthog.metrics4_input AS input
GROUP BY
- team_id, metric_name, time_bucket, series_fingerprint, original_expiry_date;
+ team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash, original_expiry_date;
CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.writable_metrics4_series (team_id Int32, metric_name LowCardinality(String), series_fingerprint UInt64, metric_type LowCardinality(String), unit LowCardinality(String), aggregation_temporality LowCardinality(String), is_monotonic Bool, service_name LowCardinality(String), instrumentation_scope String, resource_attributes Map(LowCardinality(String), String), attributes Map(LowCardinality(String), String), timestamp DateTime64(6), original_expiry_timestamp DateTime64(6)) AS SELECT
team_id,
metric_name,
diff --git a/posthog/clickhouse/hcl/sql/prod-us/logs.sql b/posthog/clickhouse/hcl/sql/prod-us/logs.sql
index 107a7f66d046..2cdc90a0c91e 100644
--- a/posthog/clickhouse/hcl/sql/prod-us/logs.sql
+++ b/posthog/clickhouse/hcl/sql/prod-us/logs.sql
@@ -522,6 +522,7 @@ CREATE TABLE posthog.metrics4_samples (
has_labels SimpleAggregateFunction(max, UInt8),
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
+ histogram_bounds_hash UInt64,
_topic SimpleAggregateFunction(any, LowCardinality(String)),
timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
@@ -541,7 +542,7 @@ CREATE TABLE posthog.metrics4_samples (
PROJECTION service_freshness_by_hour (SELECT team_id, time_bucket, service_name, max(arrayMax(timestamp_arr))
GROUP BY
team_id, time_bucket, service_name)
-) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') ORDER BY (team_id, metric_name, time_bucket, series_fingerprint) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;
+) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/noshard/posthog.metrics4_samples', '{replica}-{shard}') PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint) ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash) PARTITION BY original_expiry_date TTL original_expiry_date SETTINGS deduplicate_merge_projection_mode = 'rebuild', index_granularity = 128, ttl_only_drop_parts = 1;
CREATE TABLE posthog.metrics4_series (
team_id Int32,
metric_name LowCardinality(String),
diff --git a/posthog/clickhouse/metrics/__init__.py b/posthog/clickhouse/metrics/__init__.py
index bbd29551d8e0..fe9b273c200d 100644
--- a/posthog/clickhouse/metrics/__init__.py
+++ b/posthog/clickhouse/metrics/__init__.py
@@ -52,6 +52,7 @@ from .metrics4 import (
METRICS4_INPUT_TO_METRICS4_NAMES_MV,
METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV,
METRICS4_INPUT_TO_METRICS4_SAMPLES_MV,
+ METRICS4_INPUT_TO_METRICS4_SAMPLES_MV_SELECT,
METRICS4_INPUT_TO_METRICS4_SERIES_MV,
METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV,
METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT,
@@ -82,6 +83,7 @@ __all__ = [
"METRICS4_INPUT_TO_METRICS4_NAMES_MV",
"METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV",
"METRICS4_INPUT_TO_METRICS4_SAMPLES_MV",
+ "METRICS4_INPUT_TO_METRICS4_SAMPLES_MV_SELECT",
"METRICS4_INPUT_TO_METRICS4_SERIES_MV",
"METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV",
"METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT",
diff --git a/posthog/clickhouse/metrics/metrics4.py b/posthog/clickhouse/metrics/metrics4.py
index 90531e03a668..dcd4c043378d 100644
--- a/posthog/clickhouse/metrics/metrics4.py
+++ b/posthog/clickhouse/metrics/metrics4.py
@@ -141,6 +141,7 @@ def WRITABLE_METRICS4_SAMPLES_TABLE_SQL() -> str:
`has_labels` SimpleAggregateFunction(max, UInt8),
`instrumentation_scope` SimpleAggregateFunction(any, String),
`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),
+ `histogram_bounds_hash` UInt64,
`_topic` SimpleAggregateFunction(any, LowCardinality(String)),
{arrays}""",
)
@@ -234,6 +235,7 @@ CREATE TABLE IF NOT EXISTS {_db()}.{METRICS4_SAMPLES_TABLE_NAME}
`has_labels` SimpleAggregateFunction(max, UInt8),
`instrumentation_scope` SimpleAggregateFunction(any, String),
`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),
+ `histogram_bounds_hash` UInt64,
`_topic` SimpleAggregateFunction(any, LowCardinality(String)),
{arrays},
`timestamp_min` DateTime64(6) ALIAS arrayMin(timestamp_arr),
@@ -255,7 +257,8 @@ CREATE TABLE IF NOT EXISTS {_db()}.{METRICS4_SAMPLES_TABLE_NAME}
)
ENGINE = {AggregatingMergeTree(METRICS4_SAMPLES_TABLE_NAME, replication_scheme=ReplicationScheme.REPLICATED)}
PARTITION BY original_expiry_date
-ORDER BY (team_id, metric_name, time_bucket, series_fingerprint)
+PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint)
+ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash)
TTL original_expiry_date
SETTINGS
index_granularity = 128,
@@ -362,20 +365,22 @@ SETTINGS
"""
-def METRICS4_INPUT_TO_METRICS4_SAMPLES_MV() -> str:
- db = _db()
+def METRICS4_INPUT_TO_METRICS4_SAMPLES_MV_SELECT() -> str:
group_arrays = ",\n".join(
f" groupArray({METRICS4_MAX_SAMPLES_PER_SERIES_HOUR})({name}) AS {name}_arr"
for name, _ in METRICS4_POINT_ARRAY_COLUMNS
if name not in METRICS4_UNWRITTEN_POINT_ARRAY_COLUMNS
)
- return f"""
-CREATE MATERIALIZED VIEW IF NOT EXISTS {db}.{METRICS4_INPUT_TABLE_NAME}_to_{METRICS4_SAMPLES_TABLE_NAME} TO {db}.{WRITABLE_METRICS4_SAMPLES_TABLE_NAME}
-AS SELECT
+ # A series can change its histogram bucket layout inside one hour. Each layout gets its own row,
+ # so the row's histogram_bounds are correct for every point in it. Points without bounds use hash 0,
+ # so their rows merge with the rows that were written before the hash column existed.
+ # The input. prefix reads the point column, not the series-hour aggregate of the same name.
+ return f"""SELECT
team_id,
metric_name,
toDateTime(toStartOfHour(timestamp)) AS time_bucket,
series_fingerprint,
+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,
toDate32(original_expiry_timestamp) AS original_expiry_date,
any(resource_fingerprint) AS resource_fingerprint,
any(service_name) AS service_name,
@@ -388,13 +393,21 @@ AS SELECT
anyLast(histogram_bounds) AS histogram_bounds,
any(_topic) AS _topic,
{group_arrays}
-FROM {db}.{METRICS4_INPUT_TABLE_NAME}
+FROM {_db()}.{METRICS4_INPUT_TABLE_NAME} AS input
GROUP BY
team_id,
metric_name,
time_bucket,
series_fingerprint,
- original_expiry_date
+ histogram_bounds_hash,
+ original_expiry_date"""
+
+
+def METRICS4_INPUT_TO_METRICS4_SAMPLES_MV() -> str:
+ db = _db()
+ return f"""
+CREATE MATERIALIZED VIEW IF NOT EXISTS {db}.{METRICS4_INPUT_TABLE_NAME}_to_{METRICS4_SAMPLES_TABLE_NAME} TO {db}.{WRITABLE_METRICS4_SAMPLES_TABLE_NAME}
+AS {METRICS4_INPUT_TO_METRICS4_SAMPLES_MV_SELECT()}
"""
diff --git a/posthog/clickhouse/migrations/0372_metrics4_samples_histogram_bounds_hash.py b/posthog/clickhouse/migrations/0372_metrics4_samples_histogram_bounds_hash.py
new file mode 100644
index 000000000000..3757597e970f
--- /dev/null
+++ b/posthog/clickhouse/migrations/0372_metrics4_samples_histogram_bounds_hash.py
@@ -0,0 +1,36 @@
+from django.conf import settings
+
+from posthog.clickhouse.client.connection import NodeRole
+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions
+from posthog.clickhouse.metrics import METRICS4_INPUT_TO_METRICS4_SAMPLES_MV_SELECT
+
+DB = settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE
+
+# The storage table and the writable table get the column before the view writes it.
+# ClickHouse can add a new column to the end of the sorting key only in the ALTER that adds the column.
+# The primary key does not change, so the primary index and the existing parts stay valid.
+# Existing rows read the column as 0, which is the hash the view writes for points without bounds.
+operations = [
+ run_sql_with_exceptions(
+ f"ALTER TABLE {DB}.metrics4_samples "
+ "ADD COLUMN IF NOT EXISTS histogram_bounds_hash UInt64 AFTER histogram_bounds, "
+ "MODIFY ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash)",
+ node_roles=[NodeRole.LOGS],
+ sharded=False,
+ is_alter_on_replicated_table=True,
+ ),
+ run_sql_with_exceptions(
+ f"ALTER TABLE {DB}.writable_metrics4_samples "
+ "ADD COLUMN IF NOT EXISTS histogram_bounds_hash UInt64 AFTER histogram_bounds",
+ node_roles=[NodeRole.APM],
+ sharded=False,
+ is_alter_on_replicated_table=False,
+ ),
+ run_sql_with_exceptions(
+ f"ALTER TABLE {DB}.metrics4_input_to_metrics4_samples MODIFY QUERY\n"
+ f"{METRICS4_INPUT_TO_METRICS4_SAMPLES_MV_SELECT()}",
+ node_roles=[NodeRole.APM],
+ sharded=False,
+ is_alter_on_replicated_table=False,
+ ),
+]
diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txt
index 7fb6c6b56b75..b210d2d0f9ce 100644
--- a/posthog/clickhouse/migrations/max_migration.txt
+++ b/posthog/clickhouse/migrations/max_migration.txt
@@ -1 +1 @@
-0371_metrics4_timestamp_delta_codec
+0372_metrics4_samples_histogram_bounds_hash
diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambr
index 0874fe13a141..818dd9392ab1 100644
--- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr
+++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr
@@ -5346,6 +5346,7 @@
metric_name,
toDateTime(toStartOfHour(timestamp)) AS time_bucket,
series_fingerprint,
+ if(empty(input.histogram_bounds), 0, cityHash64(input.histogram_bounds)) AS histogram_bounds_hash,
toDate32(original_expiry_timestamp) AS original_expiry_date,
any(resource_fingerprint) AS resource_fingerprint,
any(service_name) AS service_name,
@@ -5364,12 +5365,13 @@
groupArray(10000)(trace_id) AS trace_id_arr,
groupArray(10000)(span_id) AS span_id_arr,
groupArray(10000)(trace_flags) AS trace_flags_arr
- FROM posthog_test.metrics4_input
+ FROM posthog_test.metrics4_input AS input
GROUP BY
team_id,
metric_name,
time_bucket,
series_fingerprint,
+ histogram_bounds_hash,
original_expiry_date
'''
@@ -5463,6 +5465,7 @@
`has_labels` SimpleAggregateFunction(max, UInt8),
`instrumentation_scope` SimpleAggregateFunction(any, String),
`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),
+ `histogram_bounds_hash` UInt64,
`_topic` SimpleAggregateFunction(any, LowCardinality(String)),
`timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta, Default),
`observed_timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
@@ -5491,7 +5494,8 @@
)
ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.metrics4_samples', '{replica}-{shard}')
PARTITION BY original_expiry_date
- ORDER BY (team_id, metric_name, time_bucket, series_fingerprint)
+ PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint)
+ ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash)
TTL original_expiry_date
SETTINGS
index_granularity = 128,
@@ -11245,6 +11249,7 @@
`has_labels` SimpleAggregateFunction(max, UInt8),
`instrumentation_scope` SimpleAggregateFunction(any, String),
`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),
+ `histogram_bounds_hash` UInt64,
`_topic` SimpleAggregateFunction(any, LowCardinality(String)),
`timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),
`observed_timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))),
@@ -13335,6 +13340,7 @@
`has_labels` SimpleAggregateFunction(max, UInt8),
`instrumentation_scope` SimpleAggregateFunction(any, String),
`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),
+ `histogram_bounds_hash` UInt64,
`_topic` SimpleAggregateFunction(any, LowCardinality(String)),
`timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta, Default),
`observed_timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
@@ -13363,7 +13369,8 @@
)
ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.metrics4_samples', '{replica}-{shard}')
PARTITION BY original_expiry_date
- ORDER BY (team_id, metric_name, time_bucket, series_fingerprint)
+ PRIMARY KEY (team_id, metric_name, time_bucket, series_fingerprint)
+ ORDER BY (team_id, metric_name, time_bucket, series_fingerprint, histogram_bounds_hash)
TTL original_expiry_date
SETTINGS
index_granularity = 128,
diff --git a/products/metrics/backend/tests/test_metric_query_runner.py b/products/metrics/backend/tests/test_metric_query_runner.py
index 806c778a51c3..399f63bc5b19 100644
--- a/products/metrics/backend/tests/test_metric_query_runner.py
+++ b/products/metrics/backend/tests/test_metric_query_runner.py
@@ -1639,15 +1639,23 @@ class TestHistogramQuantileRunner(ClickhouseTestMixin, APIBaseTest):
earliest = dt.datetime.fromisoformat(rows[0]["time"]).astimezone(dt.UTC)
self.assertEqual(earliest, self.anchor)
- def test_mismatched_bounds_merge_onto_one_layout(self):
+ @parameterized.expand(
+ [
+ ("two_series", {"resource_labels": {"k8s.pod.name": "other"}}),
+ # One series changes its buckets inside one hour. The merge must keep each point on its own layout.
+ ("one_series_after_merge", {}),
+ ]
+ )
+ def test_mismatched_bounds_merge_onto_one_layout(self, _name, second_series_labels):
# Two endpoints with their own buckets chart together instead of failing.
self._seed_histogram([(self.anchor + dt.timedelta(seconds=0), [10, 0, 0, 0])], temporality="delta")
self._seed_histogram(
[(self.anchor + dt.timedelta(seconds=10), [0, 10, 0, 0])],
temporality="delta",
bounds=[0.2, 0.6, 2.0],
- resource_labels={"k8s.pod.name": "other"},
+ **second_series_labels,
)
+ sync_execute("OPTIMIZE TABLE metrics4_samples FINAL")
rows = self._run(0.75)
self.assertEqual(len(rows), 1)
# Union edges [0.1, 0.2, 0.5, 0.6, 1.0, 2.0] get counts [10, 0, 7.5, 2.5, 0, 0, 0], so p75 is 2/3 into (0.2, 0.5].