:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(workflows): keep every transactional email status (#127744)
Daniel Marchukd86e11a43a07
diff --git a/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl b/posthog/clickhouse/hcl/golden/dev/auxiliary.hclindex 0d1a9e4d155e..c6e16571456c 100644--- a/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl+++ b/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl@@ -195,6 +195,106 @@ database "posthog" {}}+ table "email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_events_data"+ }+ }++ table "email_status_events_data" {+ order_by = ["team_id", "created_at", "message_id", "status_at"]+ partition_by = "toYYYYMMDD(created_at)"+ ttl = "toDate(created_at) + toIntervalDay(31)"+ settings = {+ index_granularity = "1024"+ ttl_only_drop_parts = "1"+ }+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "replicated_merge_tree" {+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_events_data"+ replica_name = "{replica}-{shard}"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -922,6 +1022,50 @@ database "posthog" {}}+ table "kafka_email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ engine "kafka" {+ collection = "warpstream_cyclotron"+ topic_list = "clickhouse_email_status"+ group_name = "clickhouse_email_status_events"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 100000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ thread_per_consumer = true+ }+ }+table "kafka_hog_invocation_results" {column "team_id" {type = "Int64"@@ -5345,6 +5489,67 @@ database "posthog" {}}+ materialized_view "email_status_events_mv" {+ to_table = "posthog.email_status_events_data"+ query = <<SQL+SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ recipient,+ function_id,+ sending_domain,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status_events+SQL++ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "Nullable(DateTime)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ }+materialized_view "email_status_mv" {to_table = "posthog.email_status_data"query = <<SQL@@ -5360,6 +5565,7 @@ SELECT_offset,_partitionFROM posthog.kafka_email_status+WHERE status NOT IN ('opened', 'clicked')SQLcolumn "team_id" {diff --git a/posthog/clickhouse/hcl/golden/dev/data.hcl b/posthog/clickhouse/hcl/golden/dev/data.hclindex 44836f6f34e1..5a223f1568ac 100644--- a/posthog/clickhouse/hcl/golden/dev/data.hcl+++ b/posthog/clickhouse/hcl/golden/dev/data.hcl@@ -1252,6 +1252,53 @@ database "posthog" {}}+ table "email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_events_data"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"diff --git a/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl b/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hclindex f08276d31c6f..2c8b3b741f6d 100644--- a/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl+++ b/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl@@ -195,6 +195,106 @@ database "posthog" {}}+ table "email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_events_data"+ }+ }++ table "email_status_events_data" {+ order_by = ["team_id", "created_at", "message_id", "status_at"]+ partition_by = "toYYYYMMDD(created_at)"+ ttl = "toDate(created_at) + toIntervalDay(31)"+ settings = {+ index_granularity = "1024"+ ttl_only_drop_parts = "1"+ }+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "replicated_merge_tree" {+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_events_data"+ replica_name = "{replica}-{shard}"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -921,6 +1021,50 @@ database "posthog" {}}+ table "kafka_email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ engine "kafka" {+ collection = "warpstream_cyclotron"+ topic_list = "clickhouse_email_status"+ group_name = "clickhouse_email_status_events"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 100000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ thread_per_consumer = true+ }+ }+table "kafka_error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -5352,6 +5496,67 @@ database "posthog" {}}+ materialized_view "email_status_events_mv" {+ to_table = "posthog.email_status_events_data"+ query = <<SQL+SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ recipient,+ function_id,+ sending_domain,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status_events+SQL++ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "Nullable(DateTime)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ }+materialized_view "email_status_mv" {to_table = "posthog.email_status_data"query = <<SQL@@ -5367,6 +5572,7 @@ SELECT_offset,_partitionFROM posthog.kafka_email_status+WHERE status NOT IN ('opened', 'clicked')SQLcolumn "team_id" {diff --git a/posthog/clickhouse/hcl/golden/local-multi/data.hcl b/posthog/clickhouse/hcl/golden/local-multi/data.hclindex 737e15785045..cc3fb9362333 100644--- a/posthog/clickhouse/hcl/golden/local-multi/data.hcl+++ b/posthog/clickhouse/hcl/golden/local-multi/data.hcl@@ -1198,6 +1198,53 @@ database "posthog" {}}+ table "email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_events_data"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hclindex 415381254058..a45ad09f78a4 100644--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl@@ -1270,6 +1270,106 @@ database "posthog" {}}+ table "email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_events_data"+ }+ }++ table "email_status_events_data" {+ order_by = ["team_id", "created_at", "message_id", "status_at"]+ partition_by = "toYYYYMMDD(created_at)"+ ttl = "toDate(created_at) + toIntervalDay(31)"+ settings = {+ index_granularity = "1024"+ ttl_only_drop_parts = "1"+ }+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "replicated_merge_tree" {+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_events_data"+ replica_name = "{replica}-{shard}"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -2918,6 +3018,50 @@ database "posthog" {}}+ table "kafka_email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ engine "kafka" {+ collection = "warpstream_cyclotron"+ topic_list = "clickhouse_email_status"+ group_name = "clickhouse_email_status_events"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 100000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ thread_per_consumer = true+ }+ }+table "kafka_error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -20579,6 +20723,67 @@ SQL}}+ materialized_view "email_status_events_mv" {+ to_table = "posthog.email_status_events_data"+ query = <<SQL+SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ recipient,+ function_id,+ sending_domain,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status_events+SQL++ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "Nullable(DateTime)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ }+materialized_view "email_status_mv" {to_table = "posthog.email_status_data"query = <<SQL@@ -20594,6 +20799,7 @@ SELECT_offset,_partitionFROM posthog.kafka_email_status+WHERE status NOT IN ('opened', 'clicked')SQLcolumn "team_id" {diff --git a/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl b/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hclindex ec51438b5762..011c0b0dc0c0 100644--- a/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl+++ b/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl@@ -195,6 +195,106 @@ database "posthog" {}}+ table "email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_events_data"+ }+ }++ table "email_status_events_data" {+ order_by = ["team_id", "created_at", "message_id", "status_at"]+ partition_by = "toYYYYMMDD(created_at)"+ ttl = "toDate(created_at) + toIntervalDay(31)"+ settings = {+ index_granularity = "1024"+ ttl_only_drop_parts = "1"+ }+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "replicated_merge_tree" {+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_events_data"+ replica_name = "{replica}-{shard}"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -922,6 +1022,50 @@ database "posthog" {}}+ table "kafka_email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ engine "kafka" {+ collection = "warpstream_cyclotron"+ topic_list = "clickhouse_email_status"+ group_name = "clickhouse_email_status_events"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 100000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ thread_per_consumer = true+ }+ }+table "kafka_hog_invocation_results" {column "team_id" {type = "Int64"@@ -5345,6 +5489,67 @@ database "posthog" {}}+ materialized_view "email_status_events_mv" {+ to_table = "posthog.email_status_events_data"+ query = <<SQL+SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ recipient,+ function_id,+ sending_domain,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status_events+SQL++ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "Nullable(DateTime)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ }+materialized_view "email_status_mv" {to_table = "posthog.email_status_data"query = <<SQL@@ -5360,6 +5565,7 @@ SELECT_offset,_partitionFROM posthog.kafka_email_status+WHERE status NOT IN ('opened', 'clicked')SQLcolumn "team_id" {diff --git a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl b/posthog/clickhouse/hcl/golden/prod-eu/data.hclindex 41a754bfd235..a4de920c82ff 100644--- a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl+++ b/posthog/clickhouse/hcl/golden/prod-eu/data.hcl@@ -1190,6 +1190,53 @@ database "posthog" {}}+ table "email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_events_data"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"diff --git a/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl b/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hclindex 6a10cb79c02c..dfc1bd77758d 100644--- a/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl+++ b/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl@@ -195,6 +195,106 @@ database "posthog" {}}+ table "email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_events_data"+ }+ }++ table "email_status_events_data" {+ order_by = ["team_id", "created_at", "message_id", "status_at"]+ partition_by = "toYYYYMMDD(created_at)"+ ttl = "toDate(created_at) + toIntervalDay(31)"+ settings = {+ index_granularity = "1024"+ ttl_only_drop_parts = "1"+ }+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "replicated_merge_tree" {+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_events_data"+ replica_name = "{replica}-{shard}"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -956,6 +1056,50 @@ database "posthog" {}}+ table "kafka_email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ engine "kafka" {+ collection = "warpstream_cyclotron"+ topic_list = "clickhouse_email_status"+ group_name = "clickhouse_email_status_events"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 100000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ thread_per_consumer = true+ }+ }+table "kafka_hog_invocation_results" {column "team_id" {type = "Int64"@@ -5383,6 +5527,67 @@ database "posthog" {}}+ materialized_view "email_status_events_mv" {+ to_table = "posthog.email_status_events_data"+ query = <<SQL+SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ recipient,+ function_id,+ sending_domain,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status_events+SQL++ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "Nullable(DateTime)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ }+materialized_view "email_status_mv" {to_table = "posthog.email_status_data"query = <<SQL@@ -5398,6 +5603,7 @@ SELECT_offset,_partitionFROM posthog.kafka_email_status+WHERE status NOT IN ('opened', 'clicked')SQLcolumn "team_id" {diff --git a/posthog/clickhouse/hcl/golden/prod-us/data.hcl b/posthog/clickhouse/hcl/golden/prod-us/data.hclindex dabf7d8ac616..4d57267f8c30 100644--- a/posthog/clickhouse/hcl/golden/prod-us/data.hcl+++ b/posthog/clickhouse/hcl/golden/prod-us/data.hcl@@ -1284,6 +1284,53 @@ database "posthog" {}}+ table "email_status_events" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_events_data"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"diff --git a/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_events_mv.sql b/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_events_mv.sqlnew file mode 100644index 000000000000..c219f0d96376--- /dev/null+++ b/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_events_mv.sql@@ -0,0 +1,15 @@+SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ recipient,+ function_id,+ sending_domain,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status_eventsdiff --git a/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sql b/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sqlindex 8b56f47befa9..9a0bf3bdea46 100644--- a/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sql+++ b/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sql@@ -10,3 +10,4 @@ SELECT_offset,_partitionFROM posthog.kafka_email_status+WHERE status NOT IN ('opened', 'clicked')diff --git a/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl b/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hclindex 0be12c26ae72..4ba342156339 100644--- a/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl+++ b/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl@@ -1156,6 +1156,29 @@ database "posthog" {version_column = "version"}}+ table "email_status_events_data" {+ order_by = ["team_id", "created_at", "message_id", "status_at"]+ partition_by = "toYYYYMMDD(created_at)"+ ttl = "toDate(created_at) + toIntervalDay(31)"+ settings = {+ index_granularity = "1024"+ ttl_only_drop_parts = "1"+ }+ extend = "_email_status_events_columns"+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "replicated_merge_tree" {+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_events_data"+ replica_name = "{replica}-{shard}"+ }+ }table "kafka_email_status" {extend = "_email_status_columns"engine "kafka" {@@ -1170,6 +1193,20 @@ database "posthog" {thread_per_consumer = true}}+ table "kafka_email_status_events" {+ extend = "_email_status_events_columns"+ engine "kafka" {+ collection = "warpstream_cyclotron"+ topic_list = "clickhouse_email_status"+ group_name = "clickhouse_email_status_events"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 100000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ thread_per_consumer = true+ }+ }materialized_view "email_status_mv" {to_table = "posthog.email_status_data"query = file("sql/email_status_mv.sql")@@ -1205,4 +1242,48 @@ database "posthog" {type = "UInt64"}}+ materialized_view "email_status_events_mv" {+ to_table = "posthog.email_status_events_data"+ query = file("sql/email_status_events_mv.sql")++ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ column "_timestamp" {+ type = "Nullable(DateTime)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ }}diff --git a/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl b/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hclindex fb8396c68902..a62ab670da6a 100644--- a/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl+++ b/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl@@ -85,6 +85,39 @@ database "posthog" {type = "String"}}+ table "_email_status_events_columns" {+ abstract = true+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "recipient" {+ type = "String"+ }+ column "function_id" {+ type = "String"+ }+ column "sending_domain" {+ type = "LowCardinality(String)"+ }+ }table "_experiment_metric_events_preaggregated_columns" {abstract = truecolumn "team_id" {@@ -1144,6 +1177,23 @@ database "posthog" {remote_table = "email_status_data"}}+ table "email_status_events" {+ extend = "_email_status_events_columns"+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_events_data"+ }+ }table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"diff --git a/posthog/clickhouse/hcl/sql/dev/auxiliary.sql b/posthog/clickhouse/hcl/sql/dev/auxiliary.sqlindex 017a00fe11e4..e934b55818a7 100644--- a/posthog/clickhouse/hcl/sql/dev/auxiliary.sql+++ b/posthog/clickhouse/hcl/sql/dev/auxiliary.sql@@ -27,6 +27,21 @@ CREATE TABLE posthog.email_status_data (_offset UInt64,_partition UInt64) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;+CREATE TABLE posthog.email_status_events_data (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.email_status_events_data', '{replica}-{shard}') ORDER BY (team_id, created_at, message_id, status_at) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;CREATE TABLE posthog.events (uuid UUID,event String,@@ -182,6 +197,18 @@ CREATE TABLE posthog.kafka_email_status (status_at DateTime64(6, 'UTC'),reason String) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';+CREATE TABLE posthog.kafka_email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String)+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status_events', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';CREATE TABLE posthog.kafka_hog_invocation_results (team_id Int64,function_kind LowCardinality(String),@@ -1337,6 +1364,21 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (lc_estimated_rows SimpleAggregateFunction(max, Int64),lc_estimated_bytes SimpleAggregateFunction(max, Int64)) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');+CREATE MATERIALIZED VIEW posthog.email_status_events_mv TO posthog.email_status_events_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, recipient String, function_id String, sending_domain LowCardinality(String), _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ recipient,+ function_id,+ sending_domain,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status_events;CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,message_id,@@ -1348,7 +1390,8 @@ CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (t_timestamp,_offset,_partition-FROM posthog.kafka_email_status;+FROM posthog.kafka_email_status+WHERE status NOT IN ('opened', 'clicked');CREATE MATERIALIZED VIEW posthog.hog_invocation_results_mv TO posthog.hog_invocation_results_data (team_id Int64, function_kind LowCardinality(String), function_id String, invocation_id String, parent_run_id String, status LowCardinality(String), attempts UInt8, is_retry UInt8, scheduled_at DateTime64(6, 'UTC'), first_scheduled_at DateTime64(6, 'UTC'), started_at Nullable(DateTime64(6, 'UTC')), finished_at Nullable(DateTime64(6, 'UTC')), duration_ms Nullable(UInt32), error_kind LowCardinality(String), error_message String, event_uuid String, distinct_id String, person_id String, invocation_globals String, version UInt64, is_deleted UInt8, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,function_kind,@@ -1682,6 +1725,21 @@ CREATE TABLE posthog.email_status (_offset UInt64,_partition UInt64) ENGINE = Distributed('aux', 'posthog', 'email_status_data');+CREATE TABLE posthog.email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/dev/data.sql b/posthog/clickhouse/hcl/sql/dev/data.sqlindex 0492bb145cf6..be9586c03c5f 100644--- a/posthog/clickhouse/hcl/sql/dev/data.sql+++ b/posthog/clickhouse/hcl/sql/dev/data.sql@@ -288,6 +288,21 @@ CREATE TABLE posthog.email_status (_offset UInt64,_partition UInt64) ENGINE = Distributed('aux', 'posthog', 'email_status_data');+CREATE TABLE posthog.email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql b/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sqlindex 0abc88af38d9..2797a1dad6a9 100644--- a/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql+++ b/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql@@ -27,6 +27,21 @@ CREATE TABLE posthog.email_status_data (_offset UInt64,_partition UInt64) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;+CREATE TABLE posthog.email_status_events_data (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.email_status_events_data', '{replica}-{shard}') ORDER BY (team_id, created_at, message_id, status_at) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;CREATE TABLE posthog.events (uuid UUID,event String,@@ -182,6 +197,18 @@ CREATE TABLE posthog.kafka_email_status (status_at DateTime64(6, 'UTC'),reason String) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';+CREATE TABLE posthog.kafka_email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String)+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status_events', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';CREATE TABLE posthog.kafka_error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,@@ -1341,6 +1368,21 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (lc_estimated_rows SimpleAggregateFunction(max, Int64),lc_estimated_bytes SimpleAggregateFunction(max, Int64)) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');+CREATE MATERIALIZED VIEW posthog.email_status_events_mv TO posthog.email_status_events_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, recipient String, function_id String, sending_domain LowCardinality(String), _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ recipient,+ function_id,+ sending_domain,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status_events;CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,message_id,@@ -1352,7 +1394,8 @@ CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (t_timestamp,_offset,_partition-FROM posthog.kafka_email_status;+FROM posthog.kafka_email_status+WHERE status NOT IN ('opened', 'clicked');CREATE MATERIALIZED VIEW posthog.error_tracking_fingerprint_issue_state_mv TO posthog.writable_error_tracking_fingerprint_issue_state (team_id Int64, fingerprint String, issue_id UUID, issue_name Nullable(String), issue_description Nullable(String), issue_status String, issue_severity Nullable(String), assigned_user_id Nullable(Int64), assigned_role_id Nullable(UUID), first_seen DateTime64(3, 'UTC'), is_deleted Int8, version Int64, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,fingerprint,@@ -1703,6 +1746,21 @@ CREATE TABLE posthog.email_status (_offset UInt64,_partition UInt64) ENGINE = Distributed('aux', 'posthog', 'email_status_data');+CREATE TABLE posthog.email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/local-multi/data.sql b/posthog/clickhouse/hcl/sql/local-multi/data.sqlindex 862db991efeb..48c637358af5 100644--- a/posthog/clickhouse/hcl/sql/local-multi/data.sql+++ b/posthog/clickhouse/hcl/sql/local-multi/data.sql@@ -245,6 +245,21 @@ CREATE TABLE posthog.email_status (_offset UInt64,_partition UInt64) ENGINE = Distributed('aux', 'posthog', 'email_status_data');+CREATE TABLE posthog.email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sqlindex 74203b0b5181..1d0014a9c545 100644--- a/posthog/clickhouse/hcl/sql/local-single/all.sql+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql@@ -142,6 +142,21 @@ CREATE TABLE posthog.email_status_data (_offset UInt64,_partition UInt64) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;+CREATE TABLE posthog.email_status_events_data (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.email_status_events_data', '{replica}-{shard}') ORDER BY (team_id, created_at, message_id, status_at) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;CREATE TABLE posthog.error_tracking_issue_fingerprint_overrides (team_id Int64,fingerprint String,@@ -369,6 +384,18 @@ CREATE TABLE posthog.kafka_email_status (status_at DateTime64(6, 'UTC'),reason String) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';+CREATE TABLE posthog.kafka_email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String)+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status_events', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';CREATE TABLE posthog.kafka_error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,@@ -4557,6 +4584,21 @@ CREATE MATERIALIZED VIEW posthog.duplicate_events_mv TO posthog.writable_duplica_offset,_partitionFROM posthog.kafka_duplicate_events;+CREATE MATERIALIZED VIEW posthog.email_status_events_mv TO posthog.email_status_events_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, recipient String, function_id String, sending_domain LowCardinality(String), _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ recipient,+ function_id,+ sending_domain,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status_events;CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,message_id,@@ -4568,7 +4610,8 @@ CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (t_timestamp,_offset,_partition-FROM posthog.kafka_email_status;+FROM posthog.kafka_email_status+WHERE status NOT IN ('opened', 'clicked');CREATE MATERIALIZED VIEW posthog.error_tracking_fingerprint_issue_state_mv TO posthog.writable_error_tracking_fingerprint_issue_state (team_id Int64, fingerprint String, issue_id UUID, issue_name Nullable(String), issue_description Nullable(String), issue_status String, issue_severity Nullable(String), assigned_user_id Nullable(Int64), assigned_role_id Nullable(UUID), first_seen DateTime64(3, 'UTC'), is_deleted Int8, version Int64, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,fingerprint,@@ -6611,6 +6654,21 @@ CREATE TABLE posthog.email_status (_offset UInt64,_partition UInt64) ENGINE = Distributed('aux', 'posthog', 'email_status_data');+CREATE TABLE posthog.email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql b/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sqlindex 6bb9f0de70e1..f08cffdc21df 100644--- a/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql+++ b/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql@@ -27,6 +27,21 @@ CREATE TABLE posthog.email_status_data (_offset UInt64,_partition UInt64) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;+CREATE TABLE posthog.email_status_events_data (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.email_status_events_data', '{replica}-{shard}') ORDER BY (team_id, created_at, message_id, status_at) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;CREATE TABLE posthog.events (uuid UUID,event String,@@ -182,6 +197,18 @@ CREATE TABLE posthog.kafka_email_status (status_at DateTime64(6, 'UTC'),reason String) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';+CREATE TABLE posthog.kafka_email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String)+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status_events', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';CREATE TABLE posthog.kafka_hog_invocation_results (team_id Int64,function_kind LowCardinality(String),@@ -1337,6 +1364,21 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (lc_estimated_rows SimpleAggregateFunction(max, Int64),lc_estimated_bytes SimpleAggregateFunction(max, Int64)) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');+CREATE MATERIALIZED VIEW posthog.email_status_events_mv TO posthog.email_status_events_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, recipient String, function_id String, sending_domain LowCardinality(String), _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ recipient,+ function_id,+ sending_domain,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status_events;CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,message_id,@@ -1348,7 +1390,8 @@ CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (t_timestamp,_offset,_partition-FROM posthog.kafka_email_status;+FROM posthog.kafka_email_status+WHERE status NOT IN ('opened', 'clicked');CREATE MATERIALIZED VIEW posthog.hog_invocation_results_mv TO posthog.hog_invocation_results_data (team_id Int64, function_kind LowCardinality(String), function_id String, invocation_id String, parent_run_id String, status LowCardinality(String), attempts UInt8, is_retry UInt8, scheduled_at DateTime64(6, 'UTC'), first_scheduled_at DateTime64(6, 'UTC'), started_at Nullable(DateTime64(6, 'UTC')), finished_at Nullable(DateTime64(6, 'UTC')), duration_ms Nullable(UInt32), error_kind LowCardinality(String), error_message String, event_uuid String, distinct_id String, person_id String, invocation_globals String, version UInt64, is_deleted UInt8, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,function_kind,@@ -1682,6 +1725,21 @@ CREATE TABLE posthog.email_status (_offset UInt64,_partition UInt64) ENGINE = Distributed('aux', 'posthog', 'email_status_data');+CREATE TABLE posthog.email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/prod-eu/data.sql b/posthog/clickhouse/hcl/sql/prod-eu/data.sqlindex 6a46b3f226ec..5d22950423f4 100644--- a/posthog/clickhouse/hcl/sql/prod-eu/data.sql+++ b/posthog/clickhouse/hcl/sql/prod-eu/data.sql@@ -301,6 +301,21 @@ CREATE TABLE posthog.email_status (_offset UInt64,_partition UInt64) ENGINE = Distributed('aux', 'posthog', 'email_status_data');+CREATE TABLE posthog.email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql b/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sqlindex 515531b2c911..16ac5cdd77da 100644--- a/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql+++ b/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql@@ -27,6 +27,21 @@ CREATE TABLE posthog.email_status_data (_offset UInt64,_partition UInt64) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;+CREATE TABLE posthog.email_status_events_data (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.email_status_events_data', '{replica}-{shard}') ORDER BY (team_id, created_at, message_id, status_at) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;CREATE TABLE posthog.events (uuid UUID,event String,@@ -182,6 +197,18 @@ CREATE TABLE posthog.kafka_email_status (status_at DateTime64(6, 'UTC'),reason String) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';+CREATE TABLE posthog.kafka_email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String)+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status_events', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';CREATE TABLE posthog.kafka_hog_invocation_results (team_id Int64,function_kind LowCardinality(String),@@ -1337,6 +1364,21 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (lc_estimated_rows SimpleAggregateFunction(max, Int64),lc_estimated_bytes SimpleAggregateFunction(max, Int64)) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');+CREATE MATERIALIZED VIEW posthog.email_status_events_mv TO posthog.email_status_events_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, recipient String, function_id String, sending_domain LowCardinality(String), _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ recipient,+ function_id,+ sending_domain,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status_events;CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,message_id,@@ -1348,7 +1390,8 @@ CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (t_timestamp,_offset,_partition-FROM posthog.kafka_email_status;+FROM posthog.kafka_email_status+WHERE status NOT IN ('opened', 'clicked');CREATE MATERIALIZED VIEW posthog.hog_invocation_results_mv TO posthog.hog_invocation_results_data (team_id Int64, function_kind LowCardinality(String), function_id String, invocation_id String, parent_run_id String, status LowCardinality(String), attempts UInt8, is_retry UInt8, scheduled_at DateTime64(6, 'UTC'), first_scheduled_at DateTime64(6, 'UTC'), started_at Nullable(DateTime64(6, 'UTC')), finished_at Nullable(DateTime64(6, 'UTC')), duration_ms Nullable(UInt32), error_kind LowCardinality(String), error_message String, event_uuid String, distinct_id String, person_id String, invocation_globals String, version UInt64, is_deleted UInt8, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,function_kind,@@ -1682,6 +1725,21 @@ CREATE TABLE posthog.email_status (_offset UInt64,_partition UInt64) ENGINE = Distributed('aux', 'posthog', 'email_status_data');+CREATE TABLE posthog.email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/prod-us/data.sql b/posthog/clickhouse/hcl/sql/prod-us/data.sqlindex 2f9c75229c9c..93603385e76b 100644--- a/posthog/clickhouse/hcl/sql/prod-us/data.sql+++ b/posthog/clickhouse/hcl/sql/prod-us/data.sql@@ -311,6 +311,21 @@ CREATE TABLE posthog.email_status (_offset UInt64,_partition UInt64) ENGINE = Distributed('aux', 'posthog', 'email_status_data');+CREATE TABLE posthog.email_status_events (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String),+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/kafka_engine.py b/posthog/clickhouse/kafka_engine.pyindex 71175da3498c..1d4c1ed7fd15 100644--- a/posthog/clickhouse/kafka_engine.py+++ b/posthog/clickhouse/kafka_engine.py@@ -22,6 +22,7 @@ CONSUMER_GROUP_LOG_ENTRIES = "clickhouse_log_entries_v3" if _US else "clickhouseCONSUMER_GROUP_HOG_INVOCATION_RESULTS = "clickhouse_hog_invocation_results"CONSUMER_GROUP_MESSAGE_ASSETS = "clickhouse_message_assets"CONSUMER_GROUP_EMAIL_STATUS = "clickhouse_email_status"+CONSUMER_GROUP_EMAIL_STATUS_EVENTS = "clickhouse_email_status_events"CONSUMER_GROUP_DOCUMENT_EMBEDDINGS = "clickhouse_document_embeddings2" if _US else "clickhouse_document_embeddings"CONSUMER_GROUP_HEATMAPS = "clickhouse_heatmaps" if _US else "group1"CONSUMER_GROUP_PRECALCULATED_EVENTS = "clickhouse_precalculated_events2" if _US else "clickhouse_prefiltered_events"diff --git a/posthog/clickhouse/migrations/0368_email_status_events.py b/posthog/clickhouse/migrations/0368_email_status_events.pynew file mode 100644index 000000000000..52e5e0a967ac--- /dev/null+++ b/posthog/clickhouse/migrations/0368_email_status_events.py@@ -0,0 +1,25 @@+from posthog.clickhouse.client.connection import NodeRole+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions++from products.messaging.backend.facade.clickhouse import (+ ALTER_EMAIL_STATUS_MV_QUERY_SQL,+ DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL,+ EMAIL_STATUS_EVENTS_DATA_TABLE_SQL,+ EMAIL_STATUS_EVENTS_MV_SQL,+ KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL,+)++# Only adds objects: the new table reads the topic through its own Kafka table and consumer group, and the+# existing view keeps consuming while its query changes in place, so the email_status feed never pauses.+operations = [+ run_sql_with_exceptions(EMAIL_STATUS_EVENTS_DATA_TABLE_SQL(), node_roles=[NodeRole.AUX]),+ run_sql_with_exceptions(KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL(), node_roles=[NodeRole.AUX]),+ run_sql_with_exceptions(EMAIL_STATUS_EVENTS_MV_SQL(), node_roles=[NodeRole.AUX]),+ run_sql_with_exceptions(+ ALTER_EMAIL_STATUS_MV_QUERY_SQL(),+ node_roles=[NodeRole.AUX],+ sharded=False,+ is_alter_on_replicated_table=False,+ ),+ run_sql_with_exceptions(DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL(), node_roles=[NodeRole.AUX, NodeRole.DATA]),+]diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txtindex 1f048aa29546..e0822ce38c94 100644--- a/posthog/clickhouse/migrations/max_migration.txt+++ b/posthog/clickhouse/migrations/max_migration.txt@@ -1 +1 @@-0367_metrics4_view_drop_metrics2+0368_email_status_eventsdiff --git a/posthog/clickhouse/schema.py b/posthog/clickhouse/schema.pyindex be8c8fbf2eb8..bce58280a253 100644--- a/posthog/clickhouse/schema.py+++ b/posthog/clickhouse/schema.py@@ -410,9 +410,13 @@ from products.error_tracking.backend.sql import ()from products.event_definitions.backend.models.property_definition import PROPERTY_DEFINITIONS_TABLE_SQLfrom products.messaging.backend.facade.clickhouse import (+ DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL,DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,EMAIL_STATUS_DATA_TABLE_SQL,+ EMAIL_STATUS_EVENTS_DATA_TABLE_SQL,+ EMAIL_STATUS_EVENTS_MV_SQL,EMAIL_STATUS_MV_SQL,+ KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL,KAFKA_EMAIL_STATUS_TABLE_SQL,)from products.workflows.backend.facade.clickhouse import (@@ -453,6 +457,7 @@ CREATE_MERGETREE_TABLE_QUERIES = (HOG_INVOCATION_RESULTS_DATA_TABLE_SQL,MESSAGE_ASSETS_DATA_TABLE_SQL,EMAIL_STATUS_DATA_TABLE_SQL,+ EMAIL_STATUS_EVENTS_DATA_TABLE_SQL,PERFORMANCE_EVENTS_TABLE_SQL,SESSION_REPLAY_EVENTS_TABLE_SQL,WRITABLE_SESSION_REPLAY_EVENTS_TABLE_SQL,@@ -548,6 +553,7 @@ CREATE_DISTRIBUTED_TABLE_QUERIES = (DISTRIBUTED_HOG_INVOCATION_RESULTS_TABLE_SQL,DISTRIBUTED_MESSAGE_ASSETS_TABLE_SQL,DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,+ DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL,WRITABLE_PERFORMANCE_EVENTS_TABLE_SQL,DISTRIBUTED_PERFORMANCE_EVENTS_TABLE_SQL,DISTRIBUTED_SESSION_REPLAY_EVENTS_TABLE_SQL,@@ -655,6 +661,7 @@ CREATE_KAFKA_TABLE_QUERIES = (KAFKA_HOG_INVOCATION_RESULTS_TABLE_SQL,KAFKA_MESSAGE_ASSETS_TABLE_SQL,KAFKA_EMAIL_STATUS_TABLE_SQL,+ KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL,KAFKA_PERFORMANCE_EVENTS_TABLE_SQL,KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL,KAFKA_SESSION_REPLAY_FEATURES_TABLE_SQL,@@ -694,6 +701,7 @@ CREATE_MV_TABLE_QUERIES = (HOG_INVOCATION_RESULTS_MV_SQL,MESSAGE_ASSETS_MV_SQL,EMAIL_STATUS_MV_SQL,+ EMAIL_STATUS_EVENTS_MV_SQL,PERFORMANCE_EVENTS_TABLE_MV_SQL,SESSION_REPLAY_EVENTS_TABLE_MV_SQL,SESSION_REPLAY_FEATURES_TABLE_MV_SQL,diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambrindex 0276a0d0bd3e..fec26e9b4e92 100644--- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr+++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr@@ -176,6 +176,31 @@'''# ---+# name: test_create_kafka_table_with_different_kafka_host[kafka_email_status_events]+ '''++ CREATE TABLE IF NOT EXISTS kafka_email_status_events+ (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String)+ )+ ENGINE = Kafka(warpstream_cyclotron, kafka_topic_list = 'clickhouse_email_status_test', kafka_group_name = 'clickhouse_email_status_events', kafka_format = 'JSONEachRow')+ SETTINGS kafka_skip_broken_messages = 100,+ kafka_num_consumers = 1,+ kafka_thread_per_consumer = 1,+ kafka_poll_timeout_ms = 10000,+ kafka_max_block_size = 100000++ '''+# ---# name: test_create_kafka_table_with_different_kafka_host[kafka_error_tracking_fingerprint_issue_state]'''@@ -1613,12 +1638,90 @@'''# ---+# name: test_create_table_query[email_status_events]+ '''++ CREATE TABLE IF NOT EXISTS email_status_events+ (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String)++ , _timestamp DateTime+ , _offset UInt64+ , _partition UInt64++ )+ ENGINE = Distributed('aux', 'posthog_test', 'email_status_events_data')++ '''+# ---+# name: test_create_table_query[email_status_events_data]+ '''++ CREATE TABLE IF NOT EXISTS email_status_events_data+ (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String)++ , _timestamp DateTime+ , _offset UInt64+ , _partition UInt64++ )+ ENGINE = ReplicatedMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.email_status_events_data', '{replica}-{shard}')+ PARTITION BY toYYYYMMDD(created_at)+ ORDER BY (team_id, created_at, message_id, status_at)++ SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1++ '''+# ---+# name: test_create_table_query[email_status_events_mv]+ '''++ CREATE MATERIALIZED VIEW IF NOT EXISTS email_status_events_mv+ TO email_status_events_data+ AS SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ recipient,+ function_id,+ sending_domain,+ _timestamp,+ _offset,+ _partition+ FROM kafka_email_status_events++ '''+# ---# name: test_create_table_query[email_status_mv]'''CREATE MATERIALIZED VIEW IF NOT EXISTS email_status_mvTO email_status_data- AS SELECT+ AS+ SELECTteam_id,message_id,status,@@ -1630,6 +1733,7 @@_offset,_partitionFROM kafka_email_status+ WHERE status NOT IN ('opened', 'clicked')'''# ---@@ -2706,6 +2810,31 @@'''# ---+# name: test_create_table_query[kafka_email_status_events]+ '''++ CREATE TABLE IF NOT EXISTS kafka_email_status_events+ (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String)+ )+ ENGINE = Kafka(warpstream_cyclotron, kafka_topic_list = 'clickhouse_email_status_test', kafka_group_name = 'clickhouse_email_status_events', kafka_format = 'JSONEachRow')+ SETTINGS kafka_skip_broken_messages = 100,+ kafka_num_consumers = 1,+ kafka_thread_per_consumer = 1,+ kafka_poll_timeout_ms = 10000,+ kafka_max_block_size = 100000++ '''+# ---# name: test_create_table_query[kafka_error_tracking_fingerprint_issue_state]'''@@ -12018,6 +12147,35 @@'''# ---+# name: test_create_table_query_replicated_and_storage[email_status_events_data]+ '''++ CREATE TABLE IF NOT EXISTS email_status_events_data+ (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ recipient String,+ function_id String,+ sending_domain LowCardinality(String)++ , _timestamp DateTime+ , _offset UInt64+ , _partition UInt64++ )+ ENGINE = ReplicatedMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.email_status_events_data', '{replica}-{shard}')+ PARTITION BY toYYYYMMDD(created_at)+ ORDER BY (team_id, created_at, message_id, status_at)++ SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1++ '''+# ---# name: test_create_table_query_replicated_and_storage[error_tracking_issue_fingerprint_overrides]'''diff --git a/products/messaging/backend/facade/clickhouse.py b/products/messaging/backend/facade/clickhouse.pyindex ade6d5e1aad2..7791f486ae1b 100644--- a/products/messaging/backend/facade/clickhouse.py+++ b/products/messaging/backend/facade/clickhouse.py@@ -5,8 +5,13 @@ the migration that created them applies the same definitions to a cluster."""from products.messaging.backend.models.email_status_sql import (+ ALTER_EMAIL_STATUS_MV_QUERY_SQL as ALTER_EMAIL_STATUS_MV_QUERY_SQL,+ DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL as DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL,DISTRIBUTED_EMAIL_STATUS_TABLE_SQL as DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,EMAIL_STATUS_DATA_TABLE_SQL as EMAIL_STATUS_DATA_TABLE_SQL,+ EMAIL_STATUS_EVENTS_DATA_TABLE_SQL as EMAIL_STATUS_EVENTS_DATA_TABLE_SQL,+ EMAIL_STATUS_EVENTS_MV_SQL as EMAIL_STATUS_EVENTS_MV_SQL,EMAIL_STATUS_MV_SQL as EMAIL_STATUS_MV_SQL,+ KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL as KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL,KAFKA_EMAIL_STATUS_TABLE_SQL as KAFKA_EMAIL_STATUS_TABLE_SQL,)diff --git a/products/messaging/backend/models/email_status_sql.py b/products/messaging/backend/models/email_status_sql.pyindex d3abc64a3ba3..9f5d020ec86e 100644--- a/products/messaging/backend/models/email_status_sql.py+++ b/products/messaging/backend/models/email_status_sql.py@@ -2,11 +2,12 @@ from django.conf import settingsfrom posthog.clickhouse.kafka_engine import (CONSUMER_GROUP_EMAIL_STATUS,+ CONSUMER_GROUP_EMAIL_STATUS_EVENTS,KAFKA_COLUMNS_WITH_PARTITION,kafka_engine,ttl_period,)-from posthog.clickhouse.table_engines import Distributed, ReplacingMergeTree, ReplicationScheme+from posthog.clickhouse.table_engines import Distributed, MergeTreeEngine, ReplacingMergeTree, ReplicationSchemefrom posthog.kafka_client.topics import KAFKA_EMAIL_STATUSEMAIL_STATUS_RETENTION_DAYS = 30@@ -22,6 +23,17 @@ EMAIL_STATUS_DATA_TABLE = f"{EMAIL_STATUS_TABLE}_data"KAFKA_EMAIL_STATUS_TABLE = f"kafka_{EMAIL_STATUS_TABLE}"EMAIL_STATUS_MV_TABLE = f"{EMAIL_STATUS_TABLE}_mv"+# Keeps every status row of a message, where `email_status` keeps only the latest. The email log reads+# the timeline, the opens and clicks, and the recipient and sending domain from it.+EMAIL_STATUS_EVENTS_TABLE = "email_status_events"+EMAIL_STATUS_EVENTS_DATA_TABLE = f"{EMAIL_STATUS_EVENTS_TABLE}_data"+EMAIL_STATUS_EVENTS_MV_TABLE = f"{EMAIL_STATUS_EVENTS_TABLE}_mv"+# Its own Kafka table and consumer group on the same topic, so adding it leaves the `email_status` feed untouched.+KAFKA_EMAIL_STATUS_EVENTS_TABLE = f"kafka_{EMAIL_STATUS_EVENTS_TABLE}"++# Rows that record what the recipient did with the email. They never change its delivery status.+EMAIL_ENGAGEMENT_STATUSES = ("opened", "clicked")+EMAIL_STATUS_KAFKA_COLUMNS = """team_id Int64,@@ -33,6 +45,15 @@ EMAIL_STATUS_KAFKA_COLUMNS = """reason String""".strip()+# The queued row carries the recipient and the workflow, and the sent row the sending domain. Other rows+# leave them empty.+EMAIL_STATUS_EVENTS_KAFKA_COLUMNS = f"""+ {EMAIL_STATUS_KAFKA_COLUMNS},+ recipient String,+ function_id String,+ sending_domain LowCardinality(String)+""".strip()+# Every row of a message carries the same `created_at`, so the rows share a partition and the# ReplacingMergeTree can collapse them.@@ -65,18 +86,16 @@ ENGINE = {Distributed(data_table=EMAIL_STATUS_DATA_TABLE, cluster=settings.CLICK)-# The CDP producer writes to the cyclotron WarpStream cluster, so the Kafka table reads from its-# named collection.-KAFKA_EMAIL_STATUS_TABLE_SQL = lambda: (- f"""-CREATE TABLE IF NOT EXISTS {KAFKA_EMAIL_STATUS_TABLE}+def _kafka_email_status_table_sql(table: str, columns: str, group: str) -> str:+ return f"""+CREATE TABLE IF NOT EXISTS {table}(- {EMAIL_STATUS_KAFKA_COLUMNS}+ {columns})ENGINE = {kafka_engine(topic=KAFKA_EMAIL_STATUS,- group=CONSUMER_GROUP_EMAIL_STATUS,+ group=group,named_collection=settings.CLICKHOUSE_KAFKA_WARPSTREAM_CYCLOTRON_NAMED_COLLECTION,)}@@ -86,13 +105,83 @@ SETTINGS kafka_skip_broken_messages = 100,kafka_poll_timeout_ms = 10000,kafka_max_block_size = 100000"""+++# The CDP producer writes to the cyclotron WarpStream cluster, so the Kafka tables read from its+# named collection.+KAFKA_EMAIL_STATUS_TABLE_SQL = lambda: _kafka_email_status_table_sql(+ KAFKA_EMAIL_STATUS_TABLE, EMAIL_STATUS_KAFKA_COLUMNS, CONSUMER_GROUP_EMAIL_STATUS)+KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL = lambda: _kafka_email_status_table_sql(+ KAFKA_EMAIL_STATUS_EVENTS_TABLE, EMAIL_STATUS_EVENTS_KAFKA_COLUMNS, CONSUMER_GROUP_EMAIL_STATUS_EVENTS+)+++# Opens and clicks rank above delivered, so in this ReplacingMergeTree they would replace the delivery status.+EMAIL_STATUS_MV_QUERY = lambda: (+ f"""+SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+FROM {KAFKA_EMAIL_STATUS_TABLE}+WHERE status NOT IN {EMAIL_ENGAGEMENT_STATUSES}+"""+)EMAIL_STATUS_MV_SQL = lambda: (f"""CREATE MATERIALIZED VIEW IF NOT EXISTS {EMAIL_STATUS_MV_TABLE}TO {EMAIL_STATUS_DATA_TABLE}+AS {EMAIL_STATUS_MV_QUERY()}"""+)++# Changes the query of an existing view in place, so it keeps consuming with no gap.+ALTER_EMAIL_STATUS_MV_QUERY_SQL = lambda: f"ALTER TABLE {EMAIL_STATUS_MV_TABLE} MODIFY QUERY {EMAIL_STATUS_MV_QUERY()}"+++# A plain MergeTree, so no row replaces another. `created_at` leads the key because ClickHouse doesn't sort a+# UUID by time. All rows of a message share a `created_at`, so they stay together.+EMAIL_STATUS_EVENTS_DATA_TABLE_SQL = lambda: (+ f"""+CREATE TABLE IF NOT EXISTS {EMAIL_STATUS_EVENTS_DATA_TABLE}+(+ {EMAIL_STATUS_EVENTS_KAFKA_COLUMNS}+ {KAFKA_COLUMNS_WITH_PARTITION}+)+ENGINE = {MergeTreeEngine(EMAIL_STATUS_EVENTS_DATA_TABLE, replication_scheme=ReplicationScheme.REPLICATED)}+PARTITION BY toYYYYMMDD(created_at)+ORDER BY (team_id, created_at, message_id, status_at)+{ttl_period("created_at", EMAIL_STATUS_TTL_DAYS, unit="DAY")}+SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1+"""+)+++DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL = lambda: (+ f"""+CREATE TABLE IF NOT EXISTS {EMAIL_STATUS_EVENTS_TABLE}+(+ {EMAIL_STATUS_EVENTS_KAFKA_COLUMNS}+ {KAFKA_COLUMNS_WITH_PARTITION}+)+ENGINE = {Distributed(data_table=EMAIL_STATUS_EVENTS_DATA_TABLE, cluster=settings.CLICKHOUSE_AUX_CLUSTER)}+"""+)+++EMAIL_STATUS_EVENTS_MV_SQL = lambda: (+ f"""+CREATE MATERIALIZED VIEW IF NOT EXISTS {EMAIL_STATUS_EVENTS_MV_TABLE}+TO {EMAIL_STATUS_EVENTS_DATA_TABLE}AS SELECTteam_id,message_id,@@ -101,9 +190,12 @@ AS SELECTcreated_at,status_at,reason,+ recipient,+ function_id,+ sending_domain,_timestamp,_offset,_partition-FROM {KAFKA_EMAIL_STATUS_TABLE}+FROM {KAFKA_EMAIL_STATUS_EVENTS_TABLE}""")