Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): raise flag_evaluations TTL to 92 days (#124485)

Phil Haack00987354c796
diff --git a/posthog/clickhouse/hcl/golden/dev/data.hcl b/posthog/clickhouse/hcl/golden/dev/data.hcl
index b93250599d6e..15b6bbfe2688 100644
--- a/posthog/clickhouse/hcl/golden/dev/data.hcl
+++ b/posthog/clickhouse/hcl/golden/dev/data.hcl
@@ -5541,10 +5541,11 @@ database "posthog" {
table "sharded_flag_evaluations" {
order_by = ["team_id", "flag_key", "toDate(timestamp)", "cityHash64(distinct_id)", "cityHash64(uuid)"]
partition_by = "toYYYYMM(timestamp)"
- ttl = "toDate(timestamp) + toIntervalDay(90)"
+ ttl = "toDate(timestamp) + toIntervalDay(92)"
settings = {
- index_granularity = "8192"
- ttl_only_drop_parts = "1"
+ index_granularity = "8192"
+ materialize_ttl_recalculate_only = "1"
+ ttl_only_drop_parts = "1"
}
column "uuid" {
type = "UUID"
diff --git a/posthog/clickhouse/hcl/golden/local-multi/data.hcl b/posthog/clickhouse/hcl/golden/local-multi/data.hcl
index 105dbffa2b7c..58c0c7a4a87e 100644
--- a/posthog/clickhouse/hcl/golden/local-multi/data.hcl
+++ b/posthog/clickhouse/hcl/golden/local-multi/data.hcl
@@ -5762,10 +5762,11 @@ database "posthog" {
table "sharded_flag_evaluations" {
order_by = ["team_id", "flag_key", "toDate(timestamp)", "cityHash64(distinct_id)", "cityHash64(uuid)"]
partition_by = "toYYYYMM(timestamp)"
- ttl = "toDate(timestamp) + toIntervalDay(90)"
+ ttl = "toDate(timestamp) + toIntervalDay(92)"
settings = {
- index_granularity = "8192"
- ttl_only_drop_parts = "1"
+ index_granularity = "8192"
+ materialize_ttl_recalculate_only = "1"
+ ttl_only_drop_parts = "1"
}
column "uuid" {
type = "UUID"
diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hcl
index 4763c79d8fe0..885606e02a99 100644
--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl
+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl
@@ -11731,10 +11731,11 @@ SQL
table "sharded_flag_evaluations" {
order_by = ["team_id", "flag_key", "toDate(timestamp)", "cityHash64(distinct_id)", "cityHash64(uuid)"]
partition_by = "toYYYYMM(timestamp)"
- ttl = "toDate(timestamp) + toIntervalDay(90)"
+ ttl = "toDate(timestamp) + toIntervalDay(92)"
settings = {
- index_granularity = "8192"
- ttl_only_drop_parts = "1"
+ index_granularity = "8192"
+ materialize_ttl_recalculate_only = "1"
+ ttl_only_drop_parts = "1"
}
column "uuid" {
type = "UUID"
diff --git a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl b/posthog/clickhouse/hcl/golden/prod-eu/data.hcl
index 1c11d1eb7086..8d890d4e0d6a 100644
--- a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-eu/data.hcl
@@ -5716,10 +5716,11 @@ database "posthog" {
table "sharded_flag_evaluations" {
order_by = ["team_id", "flag_key", "toDate(timestamp)", "cityHash64(distinct_id)", "cityHash64(uuid)"]
partition_by = "toYYYYMM(timestamp)"
- ttl = "toDate(timestamp) + toIntervalDay(90)"
+ ttl = "toDate(timestamp) + toIntervalDay(92)"
settings = {
- index_granularity = "8192"
- ttl_only_drop_parts = "1"
+ index_granularity = "8192"
+ materialize_ttl_recalculate_only = "1"
+ ttl_only_drop_parts = "1"
}
column "uuid" {
type = "UUID"
diff --git a/posthog/clickhouse/hcl/golden/prod-us/data.hcl b/posthog/clickhouse/hcl/golden/prod-us/data.hcl
index 3797607f6b09..0fd5d1e90fde 100644
--- a/posthog/clickhouse/hcl/golden/prod-us/data.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-us/data.hcl
@@ -5529,10 +5529,11 @@ database "posthog" {
table "sharded_flag_evaluations" {
order_by = ["team_id", "flag_key", "toDate(timestamp)", "cityHash64(distinct_id)", "cityHash64(uuid)"]
partition_by = "toYYYYMM(timestamp)"
- ttl = "toDate(timestamp) + toIntervalDay(90)"
+ ttl = "toDate(timestamp) + toIntervalDay(92)"
settings = {
- index_granularity = "8192"
- ttl_only_drop_parts = "1"
+ index_granularity = "8192"
+ materialize_ttl_recalculate_only = "1"
+ ttl_only_drop_parts = "1"
}
column "uuid" {
type = "UUID"
diff --git a/posthog/clickhouse/hcl/roles/data/shared/flag_evaluations.hcl b/posthog/clickhouse/hcl/roles/data/shared/flag_evaluations.hcl
index 2b16f5475128..e9709fee4246 100644
--- a/posthog/clickhouse/hcl/roles/data/shared/flag_evaluations.hcl
+++ b/posthog/clickhouse/hcl/roles/data/shared/flag_evaluations.hcl
@@ -113,10 +113,11 @@ database "posthog" {
table "sharded_flag_evaluations" {
order_by = ["team_id", "flag_key", "toDate(timestamp)", "cityHash64(distinct_id)", "cityHash64(uuid)"]
partition_by = "toYYYYMM(timestamp)"
- ttl = "toDate(timestamp) + toIntervalDay(90)"
+ ttl = "toDate(timestamp) + toIntervalDay(92)"
settings = {
- index_granularity = "8192"
- ttl_only_drop_parts = "1"
+ index_granularity = "8192"
+ materialize_ttl_recalculate_only = "1"
+ ttl_only_drop_parts = "1"
}
extend = "_flag_evaluations_columns"
patch_column "$group_0" {
diff --git a/posthog/clickhouse/hcl/sql/dev/data.sql b/posthog/clickhouse/hcl/sql/dev/data.sql
index 1001d6eff399..dfa23c56419f 100644
--- a/posthog/clickhouse/hcl/sql/dev/data.sql
+++ b/posthog/clickhouse/hcl/sql/dev/data.sql
@@ -1172,7 +1172,7 @@ CREATE TABLE posthog.sharded_flag_evaluations (
INDEX properties_group_custom_values_bf mapValues(properties_group_custom) TYPE bloom_filter GRANULARITY 1,
INDEX person_properties_map_custom_keys_bf mapKeys(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1,
INDEX person_properties_map_custom_values_bf mapValues(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1
-) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(90) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(92) SETTINGS index_granularity = 8192, materialize_ttl_recalculate_only = 1, ttl_only_drop_parts = 1;
CREATE TABLE posthog.sharded_heatmaps (
session_id String,
team_id Int64,
diff --git a/posthog/clickhouse/hcl/sql/local-multi/data.sql b/posthog/clickhouse/hcl/sql/local-multi/data.sql
index ffcf570a0518..3cc9ef918295 100644
--- a/posthog/clickhouse/hcl/sql/local-multi/data.sql
+++ b/posthog/clickhouse/hcl/sql/local-multi/data.sql
@@ -1156,7 +1156,7 @@ CREATE TABLE posthog.sharded_flag_evaluations (
INDEX properties_group_custom_values_bf mapValues(properties_group_custom) TYPE bloom_filter GRANULARITY 1,
INDEX person_properties_map_custom_keys_bf mapKeys(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1,
INDEX person_properties_map_custom_values_bf mapValues(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1
-) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(90) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(92) SETTINGS index_granularity = 8192, materialize_ttl_recalculate_only = 1, ttl_only_drop_parts = 1;
CREATE TABLE posthog.sharded_heatmaps (
session_id String,
team_id Int64,
diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sql
index 33578a6fa027..40eac1e062f8 100644
--- a/posthog/clickhouse/hcl/sql/local-single/all.sql
+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql
@@ -2170,7 +2170,7 @@ CREATE TABLE posthog.sharded_flag_evaluations (
INDEX properties_group_custom_values_bf mapValues(properties_group_custom) TYPE bloom_filter GRANULARITY 1,
INDEX person_properties_map_custom_keys_bf mapKeys(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1,
INDEX person_properties_map_custom_values_bf mapValues(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1
-) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(90) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(92) SETTINGS index_granularity = 8192, materialize_ttl_recalculate_only = 1, ttl_only_drop_parts = 1;
CREATE TABLE posthog.sharded_heatmaps (
session_id String,
team_id Int64,
diff --git a/posthog/clickhouse/hcl/sql/prod-eu/data.sql b/posthog/clickhouse/hcl/sql/prod-eu/data.sql
index ffed8bbe7e8f..9f05bb5663e6 100644
--- a/posthog/clickhouse/hcl/sql/prod-eu/data.sql
+++ b/posthog/clickhouse/hcl/sql/prod-eu/data.sql
@@ -1449,7 +1449,7 @@ CREATE TABLE posthog.sharded_flag_evaluations (
INDEX properties_group_custom_values_bf mapValues(properties_group_custom) TYPE bloom_filter GRANULARITY 1,
INDEX person_properties_map_custom_keys_bf mapKeys(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1,
INDEX person_properties_map_custom_values_bf mapValues(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1
-) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(90) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(92) SETTINGS index_granularity = 8192, materialize_ttl_recalculate_only = 1, ttl_only_drop_parts = 1;
CREATE TABLE posthog.sharded_heatmaps (
session_id String,
team_id Int64,
diff --git a/posthog/clickhouse/hcl/sql/prod-us/data.sql b/posthog/clickhouse/hcl/sql/prod-us/data.sql
index d1633d606d7d..a1c7688c4451 100644
--- a/posthog/clickhouse/hcl/sql/prod-us/data.sql
+++ b/posthog/clickhouse/hcl/sql/prod-us/data.sql
@@ -1291,7 +1291,7 @@ CREATE TABLE posthog.sharded_flag_evaluations (
INDEX properties_group_custom_values_bf mapValues(properties_group_custom) TYPE bloom_filter GRANULARITY 1,
INDEX person_properties_map_custom_keys_bf mapKeys(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1,
INDEX person_properties_map_custom_values_bf mapValues(person_properties_map_custom) TYPE bloom_filter GRANULARITY 1
-) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(90) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/posthog.flag_evaluations', '{replica}', _timestamp) ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid)) PARTITION BY toYYYYMM(timestamp) TTL toDate(timestamp) + toIntervalDay(92) SETTINGS index_granularity = 8192, materialize_ttl_recalculate_only = 1, ttl_only_drop_parts = 1;
CREATE TABLE posthog.sharded_heatmaps (
session_id String,
team_id Int64,
diff --git a/posthog/clickhouse/migrations/0362_flag_evaluations_ttl_92_days.py b/posthog/clickhouse/migrations/0362_flag_evaluations_ttl_92_days.py
new file mode 100644
index 000000000000..621ed5dea9c2
--- /dev/null
+++ b/posthog/clickhouse/migrations/0362_flag_evaluations_ttl_92_days.py
@@ -0,0 +1,41 @@
+from django.conf import settings
+
+from posthog.clickhouse.client.connection import NodeRole
+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions
+from posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_DATA_TABLE
+
+# Raises the TTL from 90 to 92 days. A row dated day D expires when day D + TTL starts in UTC. A "Last 90 days"
+# range starts at midnight 90 days back in the team's timezone. That midnight can fall on the UTC day 91 days
+# back. A 92-day TTL keeps every UTC day that such a range covers.
+#
+# MODIFY TTL starts a mutation that recalculates the TTL of every existing part. Without it, a part
+# written before this migration keeps expiring on the 90-day schedule. The table has only a row TTL and
+# ttl_only_drop_parts = 1, so the mutation reads only `timestamp` and rewrites only each part's TTL metadata.
+#
+# Tables in tests have no TTL. See ttl_period().
+operations = (
+ [
+ # A settings-only ALTER changes only the replica that runs it, so this statement runs on every host. Each
+ # replica runs the TTL mutation on its own parts. The setting keeps a TTL mutation from rewriting columns
+ # if the table later gains a column TTL or loses ttl_only_drop_parts.
+ run_sql_with_exceptions(
+ f"ALTER TABLE {FLAG_EVALUATIONS_DATA_TABLE} MODIFY SETTING materialize_ttl_recalculate_only = 1",
+ node_roles=[NodeRole.DATA],
+ sharded=True,
+ is_alter_on_replicated_table=False,
+ ),
+ # With alter_sync = 0, the statement returns once the ALTER is in Keeper. Each replica switches to the new
+ # TTL when its replication queue runs the ALTER_METADATA entry. The mutation then recalculates the TTL of
+ # each existing part. With the default alter_sync = 1, the deploy would wait for both steps on this replica.
+ # system.tables and system.mutations on each replica show when each step is done.
+ run_sql_with_exceptions(
+ f"ALTER TABLE {FLAG_EVALUATIONS_DATA_TABLE} MODIFY TTL toDate(timestamp) + INTERVAL 92 DAY "
+ "SETTINGS materialize_ttl_after_modify = 1, alter_sync = 0",
+ node_roles=[NodeRole.DATA],
+ sharded=True,
+ is_alter_on_replicated_table=True,
+ ),
+ ]
+ if not settings.TEST
+ else []
+)
diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txt
index e3122604a450..e7da1c40d93a 100644
--- a/posthog/clickhouse/migrations/max_migration.txt
+++ b/posthog/clickhouse/migrations/max_migration.txt
@@ -1 +1 @@
-0361_logs34_internal_source
+0362_flag_evaluations_ttl_92_days
diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambr
index 635b9c6ebabd..aa76937df1f7 100644
--- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr
+++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr
@@ -8189,7 +8189,7 @@
PARTITION BY toYYYYMM(timestamp)
ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid))
- SETTINGS ttl_only_drop_parts = 1
+ SETTINGS ttl_only_drop_parts = 1, materialize_ttl_recalculate_only = 1
'''
# ---
@@ -14054,7 +14054,7 @@
PARTITION BY toYYYYMM(timestamp)
ORDER BY (team_id, flag_key, toDate(timestamp), cityHash64(distinct_id), cityHash64(uuid))
- SETTINGS ttl_only_drop_parts = 1
+ SETTINGS ttl_only_drop_parts = 1, materialize_ttl_recalculate_only = 1
'''
# ---
diff --git a/posthog/models/flag_evaluations/sql.py b/posthog/models/flag_evaluations/sql.py
index 3f0a3d5188a2..b990a21d3ba4 100644
--- a/posthog/models/flag_evaluations/sql.py
+++ b/posthog/models/flag_evaluations/sql.py
@@ -245,7 +245,7 @@ ENGINE = {FLAG_EVALUATIONS_DATA_TABLE_ENGINE()}
PARTITION BY toYYYYMM(timestamp)
ORDER BY {FLAG_EVALUATIONS_ORDER_BY}
{ttl_period("timestamp", FLAG_EVALUATIONS_TTL_DAYS, unit="DAY")}
-SETTINGS ttl_only_drop_parts = 1
+SETTINGS ttl_only_drop_parts = 1, materialize_ttl_recalculate_only = 1
"""
)