Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(metrics): use delta codec for metrics4 timestamp arrays (#130803)

Frank Hamand25bd1c62e306
diff --git a/posthog/clickhouse/hcl/golden/dev/logs.hcl b/posthog/clickhouse/hcl/golden/dev/logs.hcl
index 06345bc6cccf..c3c2b130ead0 100644
--- a/posthog/clickhouse/hcl/golden/dev/logs.hcl
+++ b/posthog/clickhouse/hcl/golden/dev/logs.hcl
@@ -1986,7 +1986,7 @@ SQL
}
column "timestamp_arr" {
type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"
- codec = "DoubleDelta, Default"
+ codec = "Delta(8), Default"
}
column "observed_timestamp_arr" {
type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"
diff --git a/posthog/clickhouse/hcl/golden/local-multi/logs.hcl b/posthog/clickhouse/hcl/golden/local-multi/logs.hcl
index fcdd418a6401..a1ae75beea23 100644
--- a/posthog/clickhouse/hcl/golden/local-multi/logs.hcl
+++ b/posthog/clickhouse/hcl/golden/local-multi/logs.hcl
@@ -2487,7 +2487,7 @@ SQL
}
column "timestamp_arr" {
type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"
- codec = "DoubleDelta, Default"
+ codec = "Delta(8), Default"
}
column "observed_timestamp_arr" {
type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"
diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hcl
index 4ee171f381ea..24d199abe7bb 100644
--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl
+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl
@@ -7970,7 +7970,7 @@ SQL
}
column "timestamp_arr" {
type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"
- codec = "DoubleDelta, Default"
+ codec = "Delta(8), Default"
}
column "observed_timestamp_arr" {
type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"
diff --git a/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl b/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl
index 25507c27906c..4dee554e8c7d 100644
--- a/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl
@@ -2073,7 +2073,7 @@ SQL
}
column "timestamp_arr" {
type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"
- codec = "DoubleDelta, Default"
+ codec = "Delta(8), Default"
}
column "observed_timestamp_arr" {
type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"
diff --git a/posthog/clickhouse/hcl/golden/prod-us/logs.hcl b/posthog/clickhouse/hcl/golden/prod-us/logs.hcl
index a22c718b2636..8917634c3724 100644
--- a/posthog/clickhouse/hcl/golden/prod-us/logs.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-us/logs.hcl
@@ -2073,7 +2073,7 @@ SQL
}
column "timestamp_arr" {
type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"
- codec = "DoubleDelta, Default"
+ codec = "Delta(8), Default"
}
column "observed_timestamp_arr" {
type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"
diff --git a/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl b/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl
index c5b1175ccf52..11601df19120 100644
--- a/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl
+++ b/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl
@@ -165,7 +165,7 @@ database "posthog" {
}
column "timestamp_arr" {
type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"
- codec = "DoubleDelta, Default"
+ codec = "Delta(8), Default"
}
column "observed_timestamp_arr" {
type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"
diff --git a/posthog/clickhouse/hcl/sql/dev/logs.sql b/posthog/clickhouse/hcl/sql/dev/logs.sql
index 3baa04ba4202..09b13baf542b 100644
--- a/posthog/clickhouse/hcl/sql/dev/logs.sql
+++ b/posthog/clickhouse/hcl/sql/dev/logs.sql
@@ -499,7 +499,7 @@ CREATE TABLE posthog.metrics4_samples (
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
_topic SimpleAggregateFunction(any, LowCardinality(String)),
- timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
+ timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
value_arr SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla(8), Default),
count_arr SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),
diff --git a/posthog/clickhouse/hcl/sql/local-multi/logs.sql b/posthog/clickhouse/hcl/sql/local-multi/logs.sql
index ee793346cfb4..7184ec2b9873 100644
--- a/posthog/clickhouse/hcl/sql/local-multi/logs.sql
+++ b/posthog/clickhouse/hcl/sql/local-multi/logs.sql
@@ -591,7 +591,7 @@ CREATE TABLE posthog.metrics4_samples (
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
_topic SimpleAggregateFunction(any, LowCardinality(String)),
- timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
+ timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
value_arr SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla(8), Default),
count_arr SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),
diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sql
index d84cc77e6953..ada095d71903 100644
--- a/posthog/clickhouse/hcl/sql/local-single/all.sql
+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql
@@ -1611,7 +1611,7 @@ CREATE TABLE posthog.metrics4_samples (
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
_topic SimpleAggregateFunction(any, LowCardinality(String)),
- timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
+ timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
value_arr SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla(8), Default),
count_arr SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),
diff --git a/posthog/clickhouse/hcl/sql/prod-eu/logs.sql b/posthog/clickhouse/hcl/sql/prod-eu/logs.sql
index 54d69e182b89..f90cfcb9b9b5 100644
--- a/posthog/clickhouse/hcl/sql/prod-eu/logs.sql
+++ b/posthog/clickhouse/hcl/sql/prod-eu/logs.sql
@@ -523,7 +523,7 @@ CREATE TABLE posthog.metrics4_samples (
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
_topic SimpleAggregateFunction(any, LowCardinality(String)),
- timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
+ timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
value_arr SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla(8), Default),
count_arr SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),
diff --git a/posthog/clickhouse/hcl/sql/prod-us/logs.sql b/posthog/clickhouse/hcl/sql/prod-us/logs.sql
index 9da449575a4a..107a7f66d046 100644
--- a/posthog/clickhouse/hcl/sql/prod-us/logs.sql
+++ b/posthog/clickhouse/hcl/sql/prod-us/logs.sql
@@ -523,7 +523,7 @@ CREATE TABLE posthog.metrics4_samples (
instrumentation_scope SimpleAggregateFunction(any, String),
histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),
_topic SimpleAggregateFunction(any, LowCardinality(String)),
- timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
+ timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),
observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
value_arr SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla(8), Default),
count_arr SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),
diff --git a/posthog/clickhouse/metrics/metrics4.py b/posthog/clickhouse/metrics/metrics4.py
index 373a20d56d05..90531e03a668 100644
--- a/posthog/clickhouse/metrics/metrics4.py
+++ b/posthog/clickhouse/metrics/metrics4.py
@@ -69,8 +69,10 @@ METRICS4_POINT_ARRAY_COLUMNS: tuple[tuple[str, str], ...] = (
# ClickHouse applies each codec to the continuous stream of array elements.
# Each array keeps its insert order. The delta codecs compress adjacent values.
# T64 and Gorilla can process values without a sort.
+# A merge appends point arrays in part order, so a merged timestamp array is not always in time order.
+# Delta encodes each out-of-order step as one large value. DoubleDelta encodes it as two.
_ARRAY_CODECS: dict[str, str] = {
- "timestamp": " CODEC(DoubleDelta, Default)",
+ "timestamp": " CODEC(Delta, Default)",
"observed_timestamp": " CODEC(DoubleDelta, Default)",
"value": " CODEC(Gorilla, Default)",
"count": " CODEC(T64, Default)",
diff --git a/posthog/clickhouse/migrations/0371_metrics4_timestamp_delta_codec.py b/posthog/clickhouse/migrations/0371_metrics4_timestamp_delta_codec.py
new file mode 100644
index 000000000000..c0381fd7f2b8
--- /dev/null
+++ b/posthog/clickhouse/migrations/0371_metrics4_timestamp_delta_codec.py
@@ -0,0 +1,18 @@
+from django.conf import settings
+
+from posthog.clickhouse.client.connection import NodeRole
+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions
+
+DB = settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE
+
+# A codec change only updates the table metadata. New parts and merges write timestamp_arr with Delta.
+# Existing parts keep DoubleDelta until a merge rewrites them.
+operations = [
+ run_sql_with_exceptions(
+ f"ALTER TABLE {DB}.metrics4_samples MODIFY COLUMN IF EXISTS timestamp_arr "
+ "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta, Default)",
+ node_roles=[NodeRole.LOGS],
+ sharded=False,
+ is_alter_on_replicated_table=True,
+ ),
+]
diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txt
index 754af8720b55..7fb6c6b56b75 100644
--- a/posthog/clickhouse/migrations/max_migration.txt
+++ b/posthog/clickhouse/migrations/max_migration.txt
@@ -1 +1 @@
-0370_metrics4_kafka_metrics_checkpoint
+0371_metrics4_timestamp_delta_codec
diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambr
index f4ddd900dc70..0874fe13a141 100644
--- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr
+++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr
@@ -5464,7 +5464,7 @@
`instrumentation_scope` SimpleAggregateFunction(any, String),
`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),
`_topic` SimpleAggregateFunction(any, LowCardinality(String)),
- `timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
+ `timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta, Default),
`observed_timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
`value_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla, Default),
`count_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),
@@ -13336,7 +13336,7 @@
`instrumentation_scope` SimpleAggregateFunction(any, String),
`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),
`_topic` SimpleAggregateFunction(any, LowCardinality(String)),
- `timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
+ `timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta, Default),
`observed_timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),
`value_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla, Default),
`count_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),