Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(clickhouse): add minmax index on sharded_events inserted_at (#120115)

Ross2ce0a8b1cc25
diff --git a/posthog/clickhouse/hcl/golden/dev/data.hcl b/posthog/clickhouse/hcl/golden/dev/data.hcl
index 753e88e51aac..b0cac6ba7592 100644
--- a/posthog/clickhouse/hcl/golden/dev/data.hcl
+++ b/posthog/clickhouse/hcl/golden/dev/data.hcl
@@ -5118,6 +5118,11 @@ database "posthog" {
type = "minmax"
granularity = 1
}
+ index "minmax_sharded_events_inserted_at" {
+ expr = "inserted_at"
+ type = "minmax"
+ granularity = 1
+ }
engine "replicated_replacing_merge_tree" {
zoo_path = "/clickhouse/tables/{shard}/posthog.events"
replica_name = "{replica}"
diff --git a/posthog/clickhouse/hcl/golden/local-multi/data.hcl b/posthog/clickhouse/hcl/golden/local-multi/data.hcl
index 103c85e478fa..6d1fcf799797 100644
--- a/posthog/clickhouse/hcl/golden/local-multi/data.hcl
+++ b/posthog/clickhouse/hcl/golden/local-multi/data.hcl
@@ -5344,6 +5344,11 @@ database "posthog" {
type = "bloom_filter"
granularity = 1
}
+ index "minmax_sharded_events_inserted_at" {
+ expr = "inserted_at"
+ type = "minmax"
+ granularity = 1
+ }
index "minmax_$group_0" {
expr = "`$group_0`"
type = "minmax"
diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hcl
index 0267100a0d06..3aab051ed27a 100644
--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl
+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl
@@ -11234,6 +11234,11 @@ SQL
type = "bloom_filter"
granularity = 1
}
+ index "minmax_sharded_events_inserted_at" {
+ expr = "inserted_at"
+ type = "minmax"
+ granularity = 1
+ }
index "minmax_$group_0" {
expr = "`$group_0`"
type = "minmax"
diff --git a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl b/posthog/clickhouse/hcl/golden/prod-eu/data.hcl
index 3d7a22f1c902..0eeaee62f7ae 100644
--- a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-eu/data.hcl
@@ -5633,6 +5633,11 @@ database "posthog" {
type = "bloom_filter"
granularity = 1
}
+ index "minmax_sharded_events_inserted_at" {
+ expr = "inserted_at"
+ type = "minmax"
+ granularity = 1
+ }
index "minmax_mat_metadata_backend" {
expr = "mat_metadata_backend"
type = "minmax"
diff --git a/posthog/clickhouse/hcl/golden/prod-us/data.hcl b/posthog/clickhouse/hcl/golden/prod-us/data.hcl
index c9bcb6d30149..f1519e148c17 100644
--- a/posthog/clickhouse/hcl/golden/prod-us/data.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-us/data.hcl
@@ -5461,6 +5461,11 @@ database "posthog" {
type = "bloom_filter"
granularity = 1
}
+ index "minmax_sharded_events_inserted_at" {
+ expr = "inserted_at"
+ type = "minmax"
+ granularity = 1
+ }
engine "replicated_replacing_merge_tree" {
zoo_path = "/clickhouse/tables/reshard3/{shard}/posthog.sharded_events"
replica_name = "{replica}"
diff --git a/posthog/clickhouse/hcl/roles/data/prod-us/tables.hcl b/posthog/clickhouse/hcl/roles/data/prod-us/tables.hcl
index e5b310f1c25b..0fbcb00fa84e 100644
--- a/posthog/clickhouse/hcl/roles/data/prod-us/tables.hcl
+++ b/posthog/clickhouse/hcl/roles/data/prod-us/tables.hcl
@@ -607,6 +607,11 @@ database "posthog" {
type = "bloom_filter"
granularity = 1
}
+ index "minmax_sharded_events_inserted_at" {
+ expr = "inserted_at"
+ type = "minmax"
+ granularity = 1
+ }
engine "replicated_replacing_merge_tree" {
zoo_path = "/clickhouse/tables/reshard3/{shard}/posthog.sharded_events"
replica_name = "{replica}"
diff --git a/posthog/clickhouse/hcl/roles/data/shared/tables.hcl b/posthog/clickhouse/hcl/roles/data/shared/tables.hcl
index c4787ca2f3c8..42dbbd33bd12 100644
--- a/posthog/clickhouse/hcl/roles/data/shared/tables.hcl
+++ b/posthog/clickhouse/hcl/roles/data/shared/tables.hcl
@@ -215,6 +215,11 @@ database "posthog" {
type = "bloom_filter"
granularity = 1
}
+ index "minmax_sharded_events_inserted_at" {
+ expr = "inserted_at"
+ type = "minmax"
+ granularity = 1
+ }
engine "replicated_replacing_merge_tree" {
zoo_path = "/clickhouse/tables/{shard}/posthog.events"
replica_name = "{replica}"
diff --git a/posthog/clickhouse/hcl/sql/dev/data.sql b/posthog/clickhouse/hcl/sql/dev/data.sql
index 7d4c4ccf701d..8095ad5d8960 100644
--- a/posthog/clickhouse/hcl/sql/dev/data.sql
+++ b/posthog/clickhouse/hcl/sql/dev/data.sql
@@ -1032,7 +1032,8 @@ CREATE TABLE posthog.sharded_events (
INDEX minmax_$ai_prompt_name `mat_$ai_prompt_name` TYPE minmax GRANULARITY 1,
INDEX bloom_filter_$ai_experiment_id `mat_$ai_experiment_id` TYPE bloom_filter GRANULARITY 1,
INDEX minmax_$ai_experiment_id `mat_$ai_experiment_id` TYPE minmax GRANULARITY 1,
- INDEX minmax_$session_id_uuid `$session_id_uuid` TYPE minmax GRANULARITY 1
+ INDEX minmax_$session_id_uuid `$session_id_uuid` TYPE minmax GRANULARITY 1,
+ INDEX minmax_sharded_events_inserted_at inserted_at TYPE minmax GRANULARITY 1
) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.events', '{replica}', _timestamp) ORDER BY (team_id, toDate(timestamp), event, cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) SAMPLE BY cityHash64(distinct_id) SETTINGS index_granularity = 8192, replicated_fetches_min_part_level = 1, replicated_fetches_min_part_level_timeout_seconds = 300;
CREATE TABLE posthog.sharded_events_codec_test (
uuid UUID,
diff --git a/posthog/clickhouse/hcl/sql/local-multi/data.sql b/posthog/clickhouse/hcl/sql/local-multi/data.sql
index 56ee55c8f997..4cec1bc8320e 100644
--- a/posthog/clickhouse/hcl/sql/local-multi/data.sql
+++ b/posthog/clickhouse/hcl/sql/local-multi/data.sql
@@ -1017,6 +1017,7 @@ CREATE TABLE posthog.sharded_events (
INDEX minmax_$ai_experiment_id `mat_$ai_experiment_id` TYPE minmax GRANULARITY 1,
INDEX minmax_$session_id_uuid `$session_id_uuid` TYPE minmax GRANULARITY 1,
INDEX bloom_filter_$session_id nullIf(nullIf(`$session_id`, ''), 'null') TYPE bloom_filter GRANULARITY 1,
+ INDEX minmax_sharded_events_inserted_at inserted_at TYPE minmax GRANULARITY 1,
INDEX minmax_$group_0 `$group_0` TYPE minmax GRANULARITY 1,
INDEX minmax_$group_1 `$group_1` TYPE minmax GRANULARITY 1,
INDEX minmax_$group_2 `$group_2` TYPE minmax GRANULARITY 1,
diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sql
index 0055ecbf3ace..b81269a30ee7 100644
--- a/posthog/clickhouse/hcl/sql/local-single/all.sql
+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql
@@ -2011,6 +2011,7 @@ CREATE TABLE posthog.sharded_events (
INDEX minmax_$ai_experiment_id `mat_$ai_experiment_id` TYPE minmax GRANULARITY 1,
INDEX minmax_$session_id_uuid `$session_id_uuid` TYPE minmax GRANULARITY 1,
INDEX bloom_filter_$session_id nullIf(nullIf(`$session_id`, ''), 'null') TYPE bloom_filter GRANULARITY 1,
+ INDEX minmax_sharded_events_inserted_at inserted_at TYPE minmax GRANULARITY 1,
INDEX minmax_$group_0 `$group_0` TYPE minmax GRANULARITY 1,
INDEX minmax_$group_1 `$group_1` TYPE minmax GRANULARITY 1,
INDEX minmax_$group_2 `$group_2` TYPE minmax GRANULARITY 1,
diff --git a/posthog/clickhouse/hcl/sql/prod-eu/data.sql b/posthog/clickhouse/hcl/sql/prod-eu/data.sql
index 54935412da5f..ec37a35a227a 100644
--- a/posthog/clickhouse/hcl/sql/prod-eu/data.sql
+++ b/posthog/clickhouse/hcl/sql/prod-eu/data.sql
@@ -1394,6 +1394,7 @@ CREATE TABLE posthog.sharded_events (
INDEX minmax_$ai_experiment_id `mat_$ai_experiment_id` TYPE minmax GRANULARITY 1,
INDEX minmax_$session_id_uuid `$session_id_uuid` TYPE minmax GRANULARITY 1,
INDEX bloom_filter_$session_id nullIf(nullIf(`$session_id`, ''), 'null') TYPE bloom_filter GRANULARITY 1,
+ INDEX minmax_sharded_events_inserted_at inserted_at TYPE minmax GRANULARITY 1,
INDEX minmax_mat_metadata_backend mat_metadata_backend TYPE minmax GRANULARITY 1,
INDEX minmax_mat_metadata_loggedIn mat_metadata_loggedIn TYPE minmax GRANULARITY 1,
INDEX minmax_mat_$ai_trace_id `mat_$ai_trace_id` TYPE minmax GRANULARITY 1
diff --git a/posthog/clickhouse/hcl/sql/prod-us/data.sql b/posthog/clickhouse/hcl/sql/prod-us/data.sql
index 97a43021644f..a47f998b8575 100644
--- a/posthog/clickhouse/hcl/sql/prod-us/data.sql
+++ b/posthog/clickhouse/hcl/sql/prod-us/data.sql
@@ -1238,7 +1238,8 @@ CREATE TABLE posthog.sharded_events (
INDEX bloom_filter_$ai_experiment_id `mat_$ai_experiment_id` TYPE bloom_filter GRANULARITY 1,
INDEX minmax_$ai_experiment_id `mat_$ai_experiment_id` TYPE minmax GRANULARITY 1,
INDEX minmax_$session_id_uuid `$session_id_uuid` TYPE minmax GRANULARITY 1,
- INDEX bloom_filter_$session_id nullIf(nullIf(`$session_id`, ''), 'null') TYPE bloom_filter GRANULARITY 1
+ INDEX bloom_filter_$session_id nullIf(nullIf(`$session_id`, ''), 'null') TYPE bloom_filter GRANULARITY 1,
+ INDEX minmax_sharded_events_inserted_at inserted_at TYPE minmax GRANULARITY 1
) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/reshard3/{shard}/posthog.sharded_events', '{replica}', _timestamp) ORDER BY (team_id, toDate(timestamp), event, cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) SAMPLE BY cityHash64(distinct_id) SETTINGS index_granularity = 8192, max_avg_part_size_for_too_many_parts = 0;
CREATE TABLE posthog.sharded_experiment_exposures_preaggregated (
team_id Int64,
diff --git a/posthog/clickhouse/migrations/0357_sharded_events_inserted_at_minmax_index.py b/posthog/clickhouse/migrations/0357_sharded_events_inserted_at_minmax_index.py
new file mode 100644
index 000000000000..357c3f94f52e
--- /dev/null
+++ b/posthog/clickhouse/migrations/0357_sharded_events_inserted_at_minmax_index.py
@@ -0,0 +1,23 @@
+"""AUTO-GENERATED from the declarative HCL by posthog/clickhouse/hcl/codegen/gen_migration.py.
+Placement (node_roles) is derived from the node composition manifest; review before committing.
+"""
+
+from django.conf import settings
+
+from posthog.clickhouse.client.connection import NodeRole
+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions
+
+DB = settings.CLICKHOUSE_DATABASE
+
+# The index is not materialized here. MATERIALIZE INDEX on sharded_events reads every part of the
+# table, so it runs as a separate manual operation. Until it runs, only parts written or merged
+# after this migration carry the index.
+operations = [
+ run_sql_with_exceptions(
+ f"ALTER TABLE {DB}.sharded_events "
+ "ADD INDEX IF NOT EXISTS minmax_sharded_events_inserted_at inserted_at TYPE minmax GRANULARITY 1",
+ node_roles=[NodeRole.DATA],
+ sharded=True,
+ is_alter_on_replicated_table=True,
+ ),
+]
diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txt
index 5bc212e557c7..9617b5833727 100644
--- a/posthog/clickhouse/migrations/max_migration.txt
+++ b/posthog/clickhouse/migrations/max_migration.txt
@@ -1 +1 @@
-0356_flag_evaluations_property_groups
+0357_sharded_events_inserted_at_minmax_index