:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(metrics): use delta codec for metrics4 timestamp arrays (#130803)
Frank Hamand25bd1c62e306
diff --git a/posthog/clickhouse/hcl/golden/dev/logs.hcl b/posthog/clickhouse/hcl/golden/dev/logs.hclindex 06345bc6cccf..c3c2b130ead0 100644--- a/posthog/clickhouse/hcl/golden/dev/logs.hcl+++ b/posthog/clickhouse/hcl/golden/dev/logs.hcl@@ -1986,7 +1986,7 @@ SQL}column "timestamp_arr" {type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"- codec = "DoubleDelta, Default"+ codec = "Delta(8), Default"}column "observed_timestamp_arr" {type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"diff --git a/posthog/clickhouse/hcl/golden/local-multi/logs.hcl b/posthog/clickhouse/hcl/golden/local-multi/logs.hclindex fcdd418a6401..a1ae75beea23 100644--- a/posthog/clickhouse/hcl/golden/local-multi/logs.hcl+++ b/posthog/clickhouse/hcl/golden/local-multi/logs.hcl@@ -2487,7 +2487,7 @@ SQL}column "timestamp_arr" {type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"- codec = "DoubleDelta, Default"+ codec = "Delta(8), Default"}column "observed_timestamp_arr" {type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hclindex 4ee171f381ea..24d199abe7bb 100644--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl@@ -7970,7 +7970,7 @@ SQL}column "timestamp_arr" {type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"- codec = "DoubleDelta, Default"+ codec = "Delta(8), Default"}column "observed_timestamp_arr" {type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"diff --git a/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl b/posthog/clickhouse/hcl/golden/prod-eu/logs.hclindex 25507c27906c..4dee554e8c7d 100644--- a/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl+++ b/posthog/clickhouse/hcl/golden/prod-eu/logs.hcl@@ -2073,7 +2073,7 @@ SQL}column "timestamp_arr" {type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"- codec = "DoubleDelta, Default"+ codec = "Delta(8), Default"}column "observed_timestamp_arr" {type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"diff --git a/posthog/clickhouse/hcl/golden/prod-us/logs.hcl b/posthog/clickhouse/hcl/golden/prod-us/logs.hclindex a22c718b2636..8917634c3724 100644--- a/posthog/clickhouse/hcl/golden/prod-us/logs.hcl+++ b/posthog/clickhouse/hcl/golden/prod-us/logs.hcl@@ -2073,7 +2073,7 @@ SQL}column "timestamp_arr" {type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"- codec = "DoubleDelta, Default"+ codec = "Delta(8), Default"}column "observed_timestamp_arr" {type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"diff --git a/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl b/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hclindex c5b1175ccf52..11601df19120 100644--- a/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl+++ b/posthog/clickhouse/hcl/roles/logs/metrics/metrics4.hcl@@ -165,7 +165,7 @@ database "posthog" {}column "timestamp_arr" {type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"- codec = "DoubleDelta, Default"+ codec = "Delta(8), Default"}column "observed_timestamp_arr" {type = "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6)))"diff --git a/posthog/clickhouse/hcl/sql/dev/logs.sql b/posthog/clickhouse/hcl/sql/dev/logs.sqlindex 3baa04ba4202..09b13baf542b 100644--- a/posthog/clickhouse/hcl/sql/dev/logs.sql+++ b/posthog/clickhouse/hcl/sql/dev/logs.sql@@ -499,7 +499,7 @@ CREATE TABLE posthog.metrics4_samples (instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),_topic SimpleAggregateFunction(any, LowCardinality(String)),- timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),+ timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),value_arr SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla(8), Default),count_arr SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),diff --git a/posthog/clickhouse/hcl/sql/local-multi/logs.sql b/posthog/clickhouse/hcl/sql/local-multi/logs.sqlindex ee793346cfb4..7184ec2b9873 100644--- a/posthog/clickhouse/hcl/sql/local-multi/logs.sql+++ b/posthog/clickhouse/hcl/sql/local-multi/logs.sql@@ -591,7 +591,7 @@ CREATE TABLE posthog.metrics4_samples (instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),_topic SimpleAggregateFunction(any, LowCardinality(String)),- timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),+ timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),value_arr SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla(8), Default),count_arr SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sqlindex d84cc77e6953..ada095d71903 100644--- a/posthog/clickhouse/hcl/sql/local-single/all.sql+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql@@ -1611,7 +1611,7 @@ CREATE TABLE posthog.metrics4_samples (instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),_topic SimpleAggregateFunction(any, LowCardinality(String)),- timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),+ timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),value_arr SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla(8), Default),count_arr SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),diff --git a/posthog/clickhouse/hcl/sql/prod-eu/logs.sql b/posthog/clickhouse/hcl/sql/prod-eu/logs.sqlindex 54d69e182b89..f90cfcb9b9b5 100644--- a/posthog/clickhouse/hcl/sql/prod-eu/logs.sql+++ b/posthog/clickhouse/hcl/sql/prod-eu/logs.sql@@ -523,7 +523,7 @@ CREATE TABLE posthog.metrics4_samples (instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),_topic SimpleAggregateFunction(any, LowCardinality(String)),- timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),+ timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),value_arr SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla(8), Default),count_arr SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),diff --git a/posthog/clickhouse/hcl/sql/prod-us/logs.sql b/posthog/clickhouse/hcl/sql/prod-us/logs.sqlindex 9da449575a4a..107a7f66d046 100644--- a/posthog/clickhouse/hcl/sql/prod-us/logs.sql+++ b/posthog/clickhouse/hcl/sql/prod-us/logs.sql@@ -523,7 +523,7 @@ CREATE TABLE posthog.metrics4_samples (instrumentation_scope SimpleAggregateFunction(any, String),histogram_bounds SimpleAggregateFunction(anyLast, Array(Float64)),_topic SimpleAggregateFunction(any, LowCardinality(String)),- timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),+ timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta(8), Default),observed_timestamp_arr SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),value_arr SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla(8), Default),count_arr SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),diff --git a/posthog/clickhouse/metrics/metrics4.py b/posthog/clickhouse/metrics/metrics4.pyindex 373a20d56d05..90531e03a668 100644--- a/posthog/clickhouse/metrics/metrics4.py+++ b/posthog/clickhouse/metrics/metrics4.py@@ -69,8 +69,10 @@ METRICS4_POINT_ARRAY_COLUMNS: tuple[tuple[str, str], ...] = (# ClickHouse applies each codec to the continuous stream of array elements.# Each array keeps its insert order. The delta codecs compress adjacent values.# T64 and Gorilla can process values without a sort.+# A merge appends point arrays in part order, so a merged timestamp array is not always in time order.+# Delta encodes each out-of-order step as one large value. DoubleDelta encodes it as two._ARRAY_CODECS: dict[str, str] = {- "timestamp": " CODEC(DoubleDelta, Default)",+ "timestamp": " CODEC(Delta, Default)","observed_timestamp": " CODEC(DoubleDelta, Default)","value": " CODEC(Gorilla, Default)","count": " CODEC(T64, Default)",diff --git a/posthog/clickhouse/migrations/0371_metrics4_timestamp_delta_codec.py b/posthog/clickhouse/migrations/0371_metrics4_timestamp_delta_codec.pynew file mode 100644index 000000000000..c0381fd7f2b8--- /dev/null+++ b/posthog/clickhouse/migrations/0371_metrics4_timestamp_delta_codec.py@@ -0,0 +1,18 @@+from django.conf import settings++from posthog.clickhouse.client.connection import NodeRole+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions++DB = settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE++# A codec change only updates the table metadata. New parts and merges write timestamp_arr with Delta.+# Existing parts keep DoubleDelta until a merge rewrites them.+operations = [+ run_sql_with_exceptions(+ f"ALTER TABLE {DB}.metrics4_samples MODIFY COLUMN IF EXISTS timestamp_arr "+ "SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta, Default)",+ node_roles=[NodeRole.LOGS],+ sharded=False,+ is_alter_on_replicated_table=True,+ ),+]diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txtindex 754af8720b55..7fb6c6b56b75 100644--- a/posthog/clickhouse/migrations/max_migration.txt+++ b/posthog/clickhouse/migrations/max_migration.txt@@ -1 +1 @@-0370_metrics4_kafka_metrics_checkpoint+0371_metrics4_timestamp_delta_codecdiff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambrindex f4ddd900dc70..0874fe13a141 100644--- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr+++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr@@ -5464,7 +5464,7 @@`instrumentation_scope` SimpleAggregateFunction(any, String),`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),`_topic` SimpleAggregateFunction(any, LowCardinality(String)),- `timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),+ `timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta, Default),`observed_timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),`value_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla, Default),`count_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),@@ -13336,7 +13336,7 @@`instrumentation_scope` SimpleAggregateFunction(any, String),`histogram_bounds` SimpleAggregateFunction(anyLast, Array(Float64)),`_topic` SimpleAggregateFunction(any, LowCardinality(String)),- `timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),+ `timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(Delta, Default),`observed_timestamp_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(DateTime64(6))) CODEC(DoubleDelta, Default),`value_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(Float64)) CODEC(Gorilla, Default),`count_arr` SimpleAggregateFunction(groupArrayArray(10000), Array(UInt64)) CODEC(T64, Default),