:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(flags): raise flag_evaluations TTL to 92 days (#124485)
Phil Haack00987354c796
diff --git a/posthog/clickhouse/hcl/golden/dev/data.hcl b/posthog/clickhouse/hcl/golden/dev/data.hclindex b93250599d6e..15b6bbfe2688 100644--- a/posthog/clickhouse/hcl/golden/dev/data.hcl+++ b/posthog/clickhouse/hcl/golden/dev/data.hcl@@ -5541,10 +5541,11 @@ database "posthog" {table "sharded_flag_evaluations" {order_by = ["team_id", "flag_key", "toDate(timestamp)", "cityHash64(distinct_id)", "cityHash64(uuid)"]partition_by = "toYYYYMM(timestamp)"- ttl = "toDate(timestamp) + toIntervalDay(90)"+ ttl = "toDate(timestamp) + toIntervalDay(92)"settings = {- index_granularity = "8192"- ttl_only_drop_parts = "1"+ index_granularity = "8192"+ materialize_ttl_recalculate_only = "1"+ ttl_only_drop_parts = "1"}column "uuid" {type = "UUID"diff --git a/posthog/clickhouse/hcl/golden/local-multi/data.hcl b/posthog/clickhouse/hcl/golden/local-multi/data.hclindex 105dbffa2b7c..58c0c7a4a87e 100644--- a/posthog/clickhouse/hcl/golden/local-multi/data.hcl+++ b/posthog/clickhouse/hcl/golden/local-multi/data.hcl@@ -5762,10 +5762,11 @@ database "posthog" {table "sharded_flag_evaluations" {order_by = ["team_id", "flag_key", "toDate(timestamp)", "cityHash64(distinct_id)", "cityHash64(uuid)"]partition_by = "toYYYYMM(timestamp)"- ttl = "toDate(timestamp) + toIntervalDay(90)"+ ttl = "toDate(timestamp) + toIntervalDay(92)"settings = {- index_granularity = "8192"- ttl_only_drop_parts = "1"+ index_granularity = "8192"+ materialize_ttl_recalculate_only = "1"+ ttl_only_drop_parts = "1"}column "uuid" {type = "UUID"diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hclindex 4763c79d8fe0..885606e02a99 100644--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl@@ -11731,10 +11731,11 @@ SQLtable "sharded_flag_evaluations" {order_by = ["team_id", "flag_key", "toDate(timestamp)", "cityHash64(distinct_id)", "cityHash64(uuid)"]partition_by = "toYYYYMM(timestamp)"- ttl = "toDate(timestamp) + toIntervalDay(90)"+ ttl = "toDate(timestamp) + toIntervalDay(92)"settings = {- index_granularity = "8192"- ttl_only_drop_parts = "1"+ index_granularity = "8192"+ materialize_ttl_recalculate_only = "1"+ ttl_only_drop_parts = "1"}column "uuid" {type = "UUID"diff --git a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl b/posthog/clickhouse/hcl/golden/prod-eu/data.hclindex 1c11d1eb7086..8d890d4e0d6a 100644--- a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl+++ b/posthog/clickhouse/hcl/golden/prod-eu/data.hcl@@ -5716,10 +5716,11 @@ database "posthog" {table "sharded_flag_evaluations" {order_by = ["team_id", "flag_key", "toDate(timestamp)", "cityHash64(distinct_id)", "cityHash64(uuid)"]partition_by = "toYYYYMM(timestamp)"- ttl = "toDate(timestamp) + toIntervalDay(90)"+ ttl = "toDate(timestamp) + toIntervalDay(92)"settings = {- index_granularity = "8192"- ttl_only_drop_parts = "1"+ index_granularity = "8192"+ materialize_ttl_recalculate_only = "1"+ ttl_only_drop_parts = "1"}column "uuid" {type = "UUID"diff --git a/posthog/clickhouse/hcl/golden/prod-us/data.hcl b/posthog/clickhouse/hcl/golden/prod-us/data.hclindex 3797607f6b09..0fd5d1e90fde 100644--- a/posthog/clickhouse/hcl/golden/prod-us/data.hcl+++ b/posthog/clickhouse/hcl/golden/prod-us/data.hcl@@ -5529,10 +5529,11 @@ database "posthog" {table "sharded_flag_evaluations" {order_by = ["team_id", "flag_key", "toDate(timestamp)", "cityHash64(distinct_id)", "cityHash64(uuid)"]partition_by = "toYYYYMM(timestamp)"- ttl = "toDate(timestamp) + toIntervalDay(90)"+ ttl = "toDate(timestamp) + toIntervalDay(92)"settings = {- index_granularity = "8192"- ttl_only_drop_parts = "1"+ index_granularity = "8192"+ materialize_ttl_recalculate_only = "1"+ ttl_only_drop_parts = "1"}column "uuid" {type = "UUID"diff --git a/posthog/clickhouse/hcl/roles/data/shared/flag_evaluations.hcl b/posthog/clickhouse/hcl/roles/data/shared/flag_evaluations.hclindex 2b16f5475128..e9709fee4246 100644--- a/posthog/clickhouse/hcl/roles/data/shared/flag_evaluations.hcl+++ b/posthog/clickhouse/hcl/roles/data/shared/flag_evaluations.hcl@@ -113,10 +113,11 @@ database "posthog" {table "sharded_flag_evaluations" {order_by = ["team_id", "flag_key", "toDate(timestamp)", "cityHash64(distinct_id)", "cityHash64(uuid)"]partition_by = "toYYYYMM(timestamp)"- ttl = "toDate(timestamp) + toIntervalDay(90)"+ ttl = "toDate(timestamp) + toIntervalDay(92)"settings = {- index_granularity = "8192"- ttl_only_drop_parts = "1"+ index_granularity = "8192"+ materialize_ttl_recalculate_only = "1"+ ttl_only_drop_parts = "1"}extend = "_flag_evaluations_columns"patch_column "$group_0" {diff --git a/posthog/clickhouse/hcl/sql/dev/data.sql b/posthog/clickhouse/hcl/sql/dev/data.sqlindex 1001d6eff399..dfa23c56419f 100644--- a/posthog/clickhouse/hcl/sql/dev/data.sql+++ b/posthog/clickhouse/hcl/sql/dev/data.sql@@ -1172,7 +1172,7 @@ CREATE TABLE posthog.sharded_flag_evaluations (INDEX properties_group_custom_values_bf mapValues(properties_group_custom) TYPE bloom_filter GRANULARITY 1,INDEX person_properties_map_custom_keys_bf mapKeys(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1,INDEX person_properties_map_custom_values_bf mapValues(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1-) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(90) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(92) SETTINGS index_granularity = 8192, materialize_ttl_recalculate_only = 1, ttl_only_drop_parts = 1;CREATE TABLE posthog.sharded_heatmaps (session_id String,team_id Int64,diff --git a/posthog/clickhouse/hcl/sql/local-multi/data.sql b/posthog/clickhouse/hcl/sql/local-multi/data.sqlindex ffcf570a0518..3cc9ef918295 100644--- a/posthog/clickhouse/hcl/sql/local-multi/data.sql+++ b/posthog/clickhouse/hcl/sql/local-multi/data.sql@@ -1156,7 +1156,7 @@ CREATE TABLE posthog.sharded_flag_evaluations (INDEX properties_group_custom_values_bf mapValues(properties_group_custom) TYPE bloom_filter GRANULARITY 1,INDEX person_properties_map_custom_keys_bf mapKeys(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1,INDEX person_properties_map_custom_values_bf mapValues(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1-) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(90) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(92) SETTINGS index_granularity = 8192, materialize_ttl_recalculate_only = 1, ttl_only_drop_parts = 1;CREATE TABLE posthog.sharded_heatmaps (session_id String,team_id Int64,diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sqlindex 33578a6fa027..40eac1e062f8 100644--- a/posthog/clickhouse/hcl/sql/local-single/all.sql+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql@@ -2170,7 +2170,7 @@ CREATE TABLE posthog.sharded_flag_evaluations (INDEX properties_group_custom_values_bf mapValues(properties_group_custom) TYPE bloom_filter GRANULARITY 1,INDEX person_properties_map_custom_keys_bf mapKeys(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1,INDEX person_properties_map_custom_values_bf mapValues(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1-) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(90) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(92) SETTINGS index_granularity = 8192, materialize_ttl_recalculate_only = 1, ttl_only_drop_parts = 1;CREATE TABLE posthog.sharded_heatmaps (session_id String,team_id Int64,diff --git a/posthog/clickhouse/hcl/sql/prod-eu/data.sql b/posthog/clickhouse/hcl/sql/prod-eu/data.sqlindex ffed8bbe7e8f..9f05bb5663e6 100644--- a/posthog/clickhouse/hcl/sql/prod-eu/data.sql+++ b/posthog/clickhouse/hcl/sql/prod-eu/data.sql@@ -1449,7 +1449,7 @@ CREATE TABLE posthog.sharded_flag_evaluations (INDEX properties_group_custom_values_bf mapValues(properties_group_custom) TYPE bloom_filter GRANULARITY 1,INDEX person_properties_map_custom_keys_bf mapKeys(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1,INDEX person_properties_map_custom_values_bf mapValues(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1-) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(90) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(92) SETTINGS index_granularity = 8192, materialize_ttl_recalculate_only = 1, ttl_only_drop_parts = 1;CREATE TABLE posthog.sharded_heatmaps (session_id String,team_id Int64,diff --git a/posthog/clickhouse/hcl/sql/prod-us/data.sql b/posthog/clickhouse/hcl/sql/prod-us/data.sqlindex d1633d606d7d..a1c7688c4451 100644--- a/posthog/clickhouse/hcl/sql/prod-us/data.sql+++ b/posthog/clickhouse/hcl/sql/prod-us/data.sql@@ -1291,7 +1291,7 @@ CREATE TABLE posthog.sharded_flag_evaluations (INDEX properties_group_custom_values_bf mapValues(properties_group_custom) TYPE bloom_filter GRANULARITY 1,INDEX person_properties_map_custom_keys_bf mapKeys(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1,INDEX person_properties_map_custom_values_bf mapValues(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1-) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(90) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(92) SETTINGS index_granularity = 8192, materialize_ttl_recalculate_only = 1, ttl_only_drop_parts = 1;CREATE TABLE posthog.sharded_heatmaps (session_id String,team_id Int64,diff --git a/posthog/clickhouse/migrations/0362_flag_evaluations_ttl_92_days.py b/posthog/clickhouse/migrations/0362_flag_evaluations_ttl_92_days.pynew file mode 100644index 000000000000..621ed5dea9c2--- /dev/null+++ b/posthog/clickhouse/migrations/0362_flag_evaluations_ttl_92_days.py@@ -0,0 +1,41 @@+from django.conf import settings++from posthog.clickhouse.client.connection import NodeRole+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions+from posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_DATA_TABLE++# Raises the TTL from 90 to 92 days. A row dated day D expires when day D + TTL starts in UTC. A "Last 90 days"+# range starts at midnight 90 days back in the team's timezone. That midnight can fall on the UTC day 91 days+# back. A 92-day TTL keeps every UTC day that such a range covers.+#+# MODIFY TTL starts a mutation that recalculates the TTL of every existing part. Without it, a part+# written before this migration keeps expiring on the 90-day schedule. The table has only a row TTL and+# ttl_only_drop_parts = 1, so the mutation reads only `timestamp` and rewrites only each part's TTL metadata.+#+# Tables in tests have no TTL. See ttl_period().+operations = (+ [+ # A settings-only ALTER changes only the replica that runs it, so this statement runs on every host. Each+ # replica runs the TTL mutation on its own parts. The setting keeps a TTL mutation from rewriting columns+ # if the table later gains a column TTL or loses ttl_only_drop_parts.+ run_sql_with_exceptions(+ f"ALTER TABLE {FLAG_EVALUATIONS_DATA_TABLE} MODIFY SETTING materialize_ttl_recalculate_only = 1",+ node_roles=[NodeRole.DATA],+ sharded=True,+ is_alter_on_replicated_table=False,+ ),+ # With alter_sync = 0, the statement returns once the ALTER is in Keeper. Each replica switches to the new+ # TTL when its replication queue runs the ALTER_METADATA entry. The mutation then recalculates the TTL of+ # each existing part. With the default alter_sync = 1, the deploy would wait for both steps on this replica.+ # system.tables and system.mutations on each replica show when each step is done.+ run_sql_with_exceptions(+ f"ALTER TABLE {FLAG_EVALUATIONS_DATA_TABLE} MODIFY TTL toDate(timestamp) + INTERVAL 92 DAY "+ "SETTINGS materialize_ttl_after_modify = 1, alter_sync = 0",+ node_roles=[NodeRole.DATA],+ sharded=True,+ is_alter_on_replicated_table=True,+ ),+ ]+ if not settings.TEST+ else []+)diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txtindex e3122604a450..e7da1c40d93a 100644--- a/posthog/clickhouse/migrations/max_migration.txt+++ b/posthog/clickhouse/migrations/max_migration.txt@@ -1 +1 @@-0361_logs34_internal_source+0362_flag_evaluations_ttl_92_daysdiff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambrindex 635b9c6ebabd..aa76937df1f7 100644--- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr+++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr@@ -8189,7 +8189,7 @@PARTITION BY toYYYYMM(timestamp)ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid))- SETTINGS ttl_only_drop_parts = 1+ SETTINGS ttl_only_drop_parts = 1, materialize_ttl_recalculate_only = 1'''# ---@@ -14054,7 +14054,7 @@PARTITION BY toYYYYMM(timestamp)ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid))- SETTINGS ttl_only_drop_parts = 1+ SETTINGS ttl_only_drop_parts = 1, materialize_ttl_recalculate_only = 1'''# ---diff --git a/posthog/models/flag_evaluations/sql.py b/posthog/models/flag_evaluations/sql.pyindex 3f0a3d5188a2..b990a21d3ba4 100644--- a/posthog/models/flag_evaluations/sql.py+++ b/posthog/models/flag_evaluations/sql.py@@ -245,7 +245,7 @@ ENGINE = {FLAG_EVALUATIONS_DATA_TABLE_ENGINE()}PARTITION BY toYYYYMM(timestamp)ORDER BY {FLAG_EVALUATIONS_ORDER_BY}{ttl_period("timestamp", FLAG_EVALUATIONS_TTL_DAYS, unit="DAY")}-SETTINGS ttl_only_drop_parts = 1+SETTINGS ttl_only_drop_parts = 1, materialize_ttl_recalculate_only = 1""")