:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(clickhouse): add minmax index on sharded_events inserted_at (#120115)
Ross2ce0a8b1cc25
diff --git a/posthog/clickhouse/hcl/golden/dev/data.hcl b/posthog/clickhouse/hcl/golden/dev/data.hclindex 753e88e51aac..b0cac6ba7592 100644--- a/posthog/clickhouse/hcl/golden/dev/data.hcl+++ b/posthog/clickhouse/hcl/golden/dev/data.hcl@@ -5118,6 +5118,11 @@ database "posthog" {type = "minmax"granularity = 1}+ index "minmax_sharded_events_inserted_at" {+ expr = "inserted_at"+ type = "minmax"+ granularity = 1+ }engine "replicated_replacing_merge_tree" {zoo_path = "/clickhouse/tables/{shard}/posthog.events"replica_name = "{replica}"diff --git a/posthog/clickhouse/hcl/golden/local-multi/data.hcl b/posthog/clickhouse/hcl/golden/local-multi/data.hclindex 103c85e478fa..6d1fcf799797 100644--- a/posthog/clickhouse/hcl/golden/local-multi/data.hcl+++ b/posthog/clickhouse/hcl/golden/local-multi/data.hcl@@ -5344,6 +5344,11 @@ database "posthog" {type = "bloom_filter"granularity = 1}+ index "minmax_sharded_events_inserted_at" {+ expr = "inserted_at"+ type = "minmax"+ granularity = 1+ }index "minmax_$group_0" {expr = "`$group_0`"type = "minmax"diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hclindex 0267100a0d06..3aab051ed27a 100644--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl@@ -11234,6 +11234,11 @@ SQLtype = "bloom_filter"granularity = 1}+ index "minmax_sharded_events_inserted_at" {+ expr = "inserted_at"+ type = "minmax"+ granularity = 1+ }index "minmax_$group_0" {expr = "`$group_0`"type = "minmax"diff --git a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl b/posthog/clickhouse/hcl/golden/prod-eu/data.hclindex 3d7a22f1c902..0eeaee62f7ae 100644--- a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl+++ b/posthog/clickhouse/hcl/golden/prod-eu/data.hcl@@ -5633,6 +5633,11 @@ database "posthog" {type = "bloom_filter"granularity = 1}+ index "minmax_sharded_events_inserted_at" {+ expr = "inserted_at"+ type = "minmax"+ granularity = 1+ }index "minmax_mat_metadata_backend" {expr = "mat_metadata_backend"type = "minmax"diff --git a/posthog/clickhouse/hcl/golden/prod-us/data.hcl b/posthog/clickhouse/hcl/golden/prod-us/data.hclindex c9bcb6d30149..f1519e148c17 100644--- a/posthog/clickhouse/hcl/golden/prod-us/data.hcl+++ b/posthog/clickhouse/hcl/golden/prod-us/data.hcl@@ -5461,6 +5461,11 @@ database "posthog" {type = "bloom_filter"granularity = 1}+ index "minmax_sharded_events_inserted_at" {+ expr = "inserted_at"+ type = "minmax"+ granularity = 1+ }engine "replicated_replacing_merge_tree" {zoo_path = "/clickhouse/tables/reshard3/{shard}/posthog.sharded_events"replica_name = "{replica}"diff --git a/posthog/clickhouse/hcl/roles/data/prod-us/tables.hcl b/posthog/clickhouse/hcl/roles/data/prod-us/tables.hclindex e5b310f1c25b..0fbcb00fa84e 100644--- a/posthog/clickhouse/hcl/roles/data/prod-us/tables.hcl+++ b/posthog/clickhouse/hcl/roles/data/prod-us/tables.hcl@@ -607,6 +607,11 @@ database "posthog" {type = "bloom_filter"granularity = 1}+ index "minmax_sharded_events_inserted_at" {+ expr = "inserted_at"+ type = "minmax"+ granularity = 1+ }engine "replicated_replacing_merge_tree" {zoo_path = "/clickhouse/tables/reshard3/{shard}/posthog.sharded_events"replica_name = "{replica}"diff --git a/posthog/clickhouse/hcl/roles/data/shared/tables.hcl b/posthog/clickhouse/hcl/roles/data/shared/tables.hclindex c4787ca2f3c8..42dbbd33bd12 100644--- a/posthog/clickhouse/hcl/roles/data/shared/tables.hcl+++ b/posthog/clickhouse/hcl/roles/data/shared/tables.hcl@@ -215,6 +215,11 @@ database "posthog" {type = "bloom_filter"granularity = 1}+ index "minmax_sharded_events_inserted_at" {+ expr = "inserted_at"+ type = "minmax"+ granularity = 1+ }engine "replicated_replacing_merge_tree" {zoo_path = "/clickhouse/tables/{shard}/posthog.events"replica_name = "{replica}"diff --git a/posthog/clickhouse/hcl/sql/dev/data.sql b/posthog/clickhouse/hcl/sql/dev/data.sqlindex 7d4c4ccf701d..8095ad5d8960 100644--- a/posthog/clickhouse/hcl/sql/dev/data.sql+++ b/posthog/clickhouse/hcl/sql/dev/data.sql@@ -1032,7 +1032,8 @@ CREATE TABLE posthog.sharded_events (INDEX minmax_$ai_prompt_name `mat_$ai_prompt_name` TYPE minmax GRANULARITY 1,INDEX bloom_filter_$ai_experiment_id `mat_$ai_experiment_id` TYPE bloom_filter GRANULARITY 1,INDEX minmax_$ai_experiment_id `mat_$ai_experiment_id` TYPE minmax GRANULARITY 1,- INDEX minmax_$session_id_uuid `$session_id_uuid` TYPE minmax GRANULARITY 1+ INDEX minmax_$session_id_uuid `$session_id_uuid` TYPE minmax GRANULARITY 1,+ INDEX minmax_sharded_events_inserted_at inserted_at TYPE minmax GRANULARITY 1) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.events', '{replica}', _timestamp) ORDER BY (team_id, toDate(timestamp), event, cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) SAMPLE BY cityHash64(distinct_id) SETTINGS index_granularity = 8192, replicated_fetches_min_part_level = 1, replicated_fetches_min_part_level_timeout_seconds = 300;CREATE TABLE posthog.sharded_events_codec_test (uuid UUID,diff --git a/posthog/clickhouse/hcl/sql/local-multi/data.sql b/posthog/clickhouse/hcl/sql/local-multi/data.sqlindex 56ee55c8f997..4cec1bc8320e 100644--- a/posthog/clickhouse/hcl/sql/local-multi/data.sql+++ b/posthog/clickhouse/hcl/sql/local-multi/data.sql@@ -1017,6 +1017,7 @@ CREATE TABLE posthog.sharded_events (INDEX minmax_$ai_experiment_id `mat_$ai_experiment_id` TYPE minmax GRANULARITY 1,INDEX minmax_$session_id_uuid `$session_id_uuid` TYPE minmax GRANULARITY 1,INDEX bloom_filter_$session_id nullIf(nullIf(`$session_id`, ''), 'null') TYPE bloom_filter GRANULARITY 1,+ INDEX minmax_sharded_events_inserted_at inserted_at TYPE minmax GRANULARITY 1,INDEX minmax_$group_0 `$group_0` TYPE minmax GRANULARITY 1,INDEX minmax_$group_1 `$group_1` TYPE minmax GRANULARITY 1,INDEX minmax_$group_2 `$group_2` TYPE minmax GRANULARITY 1,diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sqlindex 0055ecbf3ace..b81269a30ee7 100644--- a/posthog/clickhouse/hcl/sql/local-single/all.sql+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql@@ -2011,6 +2011,7 @@ CREATE TABLE posthog.sharded_events (INDEX minmax_$ai_experiment_id `mat_$ai_experiment_id` TYPE minmax GRANULARITY 1,INDEX minmax_$session_id_uuid `$session_id_uuid` TYPE minmax GRANULARITY 1,INDEX bloom_filter_$session_id nullIf(nullIf(`$session_id`, ''), 'null') TYPE bloom_filter GRANULARITY 1,+ INDEX minmax_sharded_events_inserted_at inserted_at TYPE minmax GRANULARITY 1,INDEX minmax_$group_0 `$group_0` TYPE minmax GRANULARITY 1,INDEX minmax_$group_1 `$group_1` TYPE minmax GRANULARITY 1,INDEX minmax_$group_2 `$group_2` TYPE minmax GRANULARITY 1,diff --git a/posthog/clickhouse/hcl/sql/prod-eu/data.sql b/posthog/clickhouse/hcl/sql/prod-eu/data.sqlindex 54935412da5f..ec37a35a227a 100644--- a/posthog/clickhouse/hcl/sql/prod-eu/data.sql+++ b/posthog/clickhouse/hcl/sql/prod-eu/data.sql@@ -1394,6 +1394,7 @@ CREATE TABLE posthog.sharded_events (INDEX minmax_$ai_experiment_id `mat_$ai_experiment_id` TYPE minmax GRANULARITY 1,INDEX minmax_$session_id_uuid `$session_id_uuid` TYPE minmax GRANULARITY 1,INDEX bloom_filter_$session_id nullIf(nullIf(`$session_id`, ''), 'null') TYPE bloom_filter GRANULARITY 1,+ INDEX minmax_sharded_events_inserted_at inserted_at TYPE minmax GRANULARITY 1,INDEX minmax_mat_metadata_backend mat_metadata_backend TYPE minmax GRANULARITY 1,INDEX minmax_mat_metadata_loggedIn mat_metadata_loggedIn TYPE minmax GRANULARITY 1,INDEX minmax_mat_$ai_trace_id `mat_$ai_trace_id` TYPE minmax GRANULARITY 1diff --git a/posthog/clickhouse/hcl/sql/prod-us/data.sql b/posthog/clickhouse/hcl/sql/prod-us/data.sqlindex 97a43021644f..a47f998b8575 100644--- a/posthog/clickhouse/hcl/sql/prod-us/data.sql+++ b/posthog/clickhouse/hcl/sql/prod-us/data.sql@@ -1238,7 +1238,8 @@ CREATE TABLE posthog.sharded_events (INDEX bloom_filter_$ai_experiment_id `mat_$ai_experiment_id` TYPE bloom_filter GRANULARITY 1,INDEX minmax_$ai_experiment_id `mat_$ai_experiment_id` TYPE minmax GRANULARITY 1,INDEX minmax_$session_id_uuid `$session_id_uuid` TYPE minmax GRANULARITY 1,- INDEX bloom_filter_$session_id nullIf(nullIf(`$session_id`, ''), 'null') TYPE bloom_filter GRANULARITY 1+ INDEX bloom_filter_$session_id nullIf(nullIf(`$session_id`, ''), 'null') TYPE bloom_filter GRANULARITY 1,+ INDEX minmax_sharded_events_inserted_at inserted_at TYPE minmax GRANULARITY 1) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/reshard3/{shard}/posthog.sharded_events', '{replica}', _timestamp) ORDER BY (team_id, toDate(timestamp), event, cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) SAMPLE BY cityHash64(distinct_id) SETTINGS index_granularity = 8192, max_avg_part_size_for_too_many_parts = 0;CREATE TABLE posthog.sharded_experiment_exposures_preaggregated (team_id Int64,diff --git a/posthog/clickhouse/migrations/0357_sharded_events_inserted_at_minmax_index.py b/posthog/clickhouse/migrations/0357_sharded_events_inserted_at_minmax_index.pynew file mode 100644index 000000000000..357c3f94f52e--- /dev/null+++ b/posthog/clickhouse/migrations/0357_sharded_events_inserted_at_minmax_index.py@@ -0,0 +1,23 @@+"""AUTO-GENERATED from the declarative HCL by posthog/clickhouse/hcl/codegen/gen_migration.py.+Placement (node_roles) is derived from the node composition manifest; review before committing.+"""++from django.conf import settings++from posthog.clickhouse.client.connection import NodeRole+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions++DB = settings.CLICKHOUSE_DATABASE++# The index is not materialized here. MATERIALIZE INDEX on sharded_events reads every part of the+# table, so it runs as a separate manual operation. Until it runs, only parts written or merged+# after this migration carry the index.+operations = [+ run_sql_with_exceptions(+ f"ALTER TABLE {DB}.sharded_events "+ "ADD INDEX IF NOT EXISTS minmax_sharded_events_inserted_at inserted_at TYPE minmax GRANULARITY 1",+ node_roles=[NodeRole.DATA],+ sharded=True,+ is_alter_on_replicated_table=True,+ ),+]diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txtindex 5bc212e557c7..9617b5833727 100644--- a/posthog/clickhouse/migrations/max_migration.txt+++ b/posthog/clickhouse/migrations/max_migration.txt@@ -1 +1 @@-0356_flag_evaluations_property_groups+0357_sharded_events_inserted_at_minmax_index