:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(messaging): add the email_status clickhouse table (#124696)
Harley Alexanderf68c97401ee2
diff --git a/docker/kafka/topics.txt b/docker/kafka/topics.txtindex 40380d9fcb86..87771516e35b 100644--- a/docker/kafka/topics.txt+++ b/docker/kafka/topics.txt@@ -16,6 +16,7 @@ clickhouse_app_metricsclickhouse_app_metrics2clickhouse_billing_usage_recordsclickhouse_document_embeddings+clickhouse_email_statusclickhouse_error_tracking_fingerprint_issue_stateclickhouse_error_tracking_issue_fingerprintclickhouse_events_jsondiff --git a/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl b/posthog/clickhouse/hcl/golden/dev/auxiliary.hclindex e96c12c64e2a..a90ed9b39d84 100644--- a/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl+++ b/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl@@ -112,6 +112,89 @@ database "posthog" {}}+ table "email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_data"+ }+ }++ table "email_status_data" {+ order_by = ["team_id", "message_id"]+ partition_by = "toYYYYMMDD(created_at)"+ ttl = "toDate(created_at) + toIntervalDay(31)"+ settings = {+ index_granularity = "1024"+ ttl_only_drop_parts = "1"+ }+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "replicated_replacing_merge_tree" {+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_data"+ replica_name = "{replica}-{shard}"+ version_column = "version"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -797,6 +880,41 @@ database "posthog" {}}+ table "kafka_email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ engine "kafka" {+ collection = "warpstream_cyclotron"+ topic_list = "clickhouse_email_status"+ group_name = "clickhouse_email_status"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 100000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ thread_per_consumer = true+ }+ }+table "kafka_hog_invocation_results" {column "team_id" {type = "Int64"@@ -5213,6 +5331,55 @@ database "posthog" {}}+ materialized_view "email_status_mv" {+ to_table = "posthog.email_status_data"+ query = <<SQL+SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status+SQL++ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "Nullable(DateTime)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ }+materialized_view "hog_invocation_results_mv" {to_table = "posthog.hog_invocation_results_data"query = <<SQLdiff --git a/posthog/clickhouse/hcl/golden/dev/data.hcl b/posthog/clickhouse/hcl/golden/dev/data.hclindex 15b6bbfe2688..436c085a09a4 100644--- a/posthog/clickhouse/hcl/golden/dev/data.hcl+++ b/posthog/clickhouse/hcl/golden/dev/data.hcl@@ -1214,6 +1214,44 @@ database "posthog" {}}+ table "email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_data"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"diff --git a/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl b/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hclindex 4f13b043f4ec..5f44cbb02d30 100644--- a/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl+++ b/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl@@ -112,6 +112,89 @@ database "posthog" {}}+ table "email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_data"+ }+ }++ table "email_status_data" {+ order_by = ["team_id", "message_id"]+ partition_by = "toYYYYMMDD(created_at)"+ ttl = "toDate(created_at) + toIntervalDay(31)"+ settings = {+ index_granularity = "1024"+ ttl_only_drop_parts = "1"+ }+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "replicated_replacing_merge_tree" {+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_data"+ replica_name = "{replica}-{shard}"+ version_column = "version"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -796,6 +879,41 @@ database "posthog" {}}+ table "kafka_email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ engine "kafka" {+ collection = "warpstream_cyclotron"+ topic_list = "clickhouse_email_status"+ group_name = "clickhouse_email_status"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 100000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ thread_per_consumer = true+ }+ }+table "kafka_error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -5220,6 +5338,55 @@ database "posthog" {}}+ materialized_view "email_status_mv" {+ to_table = "posthog.email_status_data"+ query = <<SQL+SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status+SQL++ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "Nullable(DateTime)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ }+materialized_view "error_tracking_fingerprint_issue_state_mv" {to_table = "posthog.writable_error_tracking_fingerprint_issue_state"query = <<SQLdiff --git a/posthog/clickhouse/hcl/golden/local-multi/data.hcl b/posthog/clickhouse/hcl/golden/local-multi/data.hclindex 58c0c7a4a87e..8dc5fb3408e6 100644--- a/posthog/clickhouse/hcl/golden/local-multi/data.hcl+++ b/posthog/clickhouse/hcl/golden/local-multi/data.hcl@@ -1160,6 +1160,44 @@ database "posthog" {}}+ table "email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_data"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hclindex 885606e02a99..a144006d9c37 100644--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl@@ -1187,6 +1187,89 @@ database "posthog" {}}+ table "email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_data"+ }+ }++ table "email_status_data" {+ order_by = ["team_id", "message_id"]+ partition_by = "toYYYYMMDD(created_at)"+ ttl = "toDate(created_at) + toIntervalDay(31)"+ settings = {+ index_granularity = "1024"+ ttl_only_drop_parts = "1"+ }+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "replicated_replacing_merge_tree" {+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_data"+ replica_name = "{replica}-{shard}"+ version_column = "version"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -2786,6 +2869,41 @@ database "posthog" {}}+ table "kafka_email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ engine "kafka" {+ collection = "warpstream_cyclotron"+ topic_list = "clickhouse_email_status"+ group_name = "clickhouse_email_status"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 100000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ thread_per_consumer = true+ }+ }+table "kafka_error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -20456,6 +20574,55 @@ SQL}}+ materialized_view "email_status_mv" {+ to_table = "posthog.email_status_data"+ query = <<SQL+SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status+SQL++ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "Nullable(DateTime)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ }+materialized_view "error_tracking_fingerprint_issue_state_mv" {to_table = "posthog.writable_error_tracking_fingerprint_issue_state"query = <<SQLdiff --git a/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl b/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hclindex b51afa6a1e03..5f225b60fe66 100644--- a/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl+++ b/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl@@ -112,6 +112,89 @@ database "posthog" {}}+ table "email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_data"+ }+ }++ table "email_status_data" {+ order_by = ["team_id", "message_id"]+ partition_by = "toYYYYMMDD(created_at)"+ ttl = "toDate(created_at) + toIntervalDay(31)"+ settings = {+ index_granularity = "1024"+ ttl_only_drop_parts = "1"+ }+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "replicated_replacing_merge_tree" {+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_data"+ replica_name = "{replica}-{shard}"+ version_column = "version"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -797,6 +880,41 @@ database "posthog" {}}+ table "kafka_email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ engine "kafka" {+ collection = "warpstream_cyclotron"+ topic_list = "clickhouse_email_status"+ group_name = "clickhouse_email_status"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 100000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ thread_per_consumer = true+ }+ }+table "kafka_hog_invocation_results" {column "team_id" {type = "Int64"@@ -5213,6 +5331,55 @@ database "posthog" {}}+ materialized_view "email_status_mv" {+ to_table = "posthog.email_status_data"+ query = <<SQL+SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status+SQL++ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "Nullable(DateTime)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ }+materialized_view "hog_invocation_results_mv" {to_table = "posthog.hog_invocation_results_data"query = <<SQLdiff --git a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl b/posthog/clickhouse/hcl/golden/prod-eu/data.hclindex 8d890d4e0d6a..b434e375ebe2 100644--- a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl+++ b/posthog/clickhouse/hcl/golden/prod-eu/data.hcl@@ -1152,6 +1152,44 @@ database "posthog" {}}+ table "email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_data"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"diff --git a/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl b/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hclindex cea262562a94..93d692cd3ba3 100644--- a/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl+++ b/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl@@ -112,6 +112,89 @@ database "posthog" {}}+ table "email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_data"+ }+ }++ table "email_status_data" {+ order_by = ["team_id", "message_id"]+ partition_by = "toYYYYMMDD(created_at)"+ ttl = "toDate(created_at) + toIntervalDay(31)"+ settings = {+ index_granularity = "1024"+ ttl_only_drop_parts = "1"+ }+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "replicated_replacing_merge_tree" {+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_data"+ replica_name = "{replica}-{shard}"+ version_column = "version"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"@@ -831,6 +914,41 @@ database "posthog" {}}+ table "kafka_email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ engine "kafka" {+ collection = "warpstream_cyclotron"+ topic_list = "clickhouse_email_status"+ group_name = "clickhouse_email_status"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 100000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ thread_per_consumer = true+ }+ }+table "kafka_hog_invocation_results" {column "team_id" {type = "Int64"@@ -5251,6 +5369,55 @@ database "posthog" {}}+ materialized_view "email_status_mv" {+ to_table = "posthog.email_status_data"+ query = <<SQL+SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status+SQL++ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "Nullable(DateTime)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ }+materialized_view "hog_invocation_results_mv" {to_table = "posthog.hog_invocation_results_data"query = <<SQLdiff --git a/posthog/clickhouse/hcl/golden/prod-us/data.hcl b/posthog/clickhouse/hcl/golden/prod-us/data.hclindex 0fd5d1e90fde..0a03c13f8977 100644--- a/posthog/clickhouse/hcl/golden/prod-us/data.hcl+++ b/posthog/clickhouse/hcl/golden/prod-us/data.hcl@@ -1246,6 +1246,44 @@ database "posthog" {}}+ table "email_status" {+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_data"+ }+ }+table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"diff --git a/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sql b/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sqlnew file mode 100644index 000000000000..8b56f47befa9--- /dev/null+++ b/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sql@@ -0,0 +1,12 @@+SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_statusdiff --git a/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl b/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hclindex 12981f14e134..0be12c26ae72 100644--- a/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl+++ b/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl@@ -1131,4 +1131,78 @@ database "posthog" {replica_name = "{replica}-{shard}"}}++ table "email_status_data" {+ order_by = ["team_id", "message_id"]+ partition_by = "toYYYYMMDD(created_at)"+ ttl = "toDate(created_at) + toIntervalDay(31)"+ settings = {+ index_granularity = "1024"+ ttl_only_drop_parts = "1"+ }+ extend = "_email_status_columns"+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "replicated_replacing_merge_tree" {+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_data"+ replica_name = "{replica}-{shard}"+ version_column = "version"+ }+ }+ table "kafka_email_status" {+ extend = "_email_status_columns"+ engine "kafka" {+ collection = "warpstream_cyclotron"+ topic_list = "clickhouse_email_status"+ group_name = "clickhouse_email_status"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 100000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ thread_per_consumer = true+ }+ }+ materialized_view "email_status_mv" {+ to_table = "posthog.email_status_data"+ query = file("sql/email_status_mv.sql")++ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ column "_timestamp" {+ type = "Nullable(DateTime)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ }}diff --git a/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl b/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hclindex 715c23703fa1..a177a37b0bba 100644--- a/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl+++ b/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl@@ -61,6 +61,30 @@ database "posthog" {default = "today() + toIntervalDay(7)"}}+ table "_email_status_columns" {+ abstract = true+ column "team_id" {+ type = "Int64"+ }+ column "message_id" {+ type = "UUID"+ }+ column "status" {+ type = "LowCardinality(String)"+ }+ column "version" {+ type = "UInt64"+ }+ column "created_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "status_at" {+ type = "DateTime64(6, 'UTC')"+ }+ column "reason" {+ type = "String"+ }+ }table "_experiment_metric_events_preaggregated_columns" {abstract = truecolumn "team_id" {@@ -1096,6 +1120,23 @@ database "posthog" {sharding_key = "cityHash64(person_id)"}}+ table "email_status" {+ extend = "_email_status_columns"+ column "_timestamp" {+ type = "DateTime"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_partition" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "email_status_data"+ }+ }table "error_tracking_fingerprint_issue_state" {column "team_id" {type = "Int64"diff --git a/posthog/clickhouse/hcl/sql/dev/auxiliary.sql b/posthog/clickhouse/hcl/sql/dev/auxiliary.sqlindex c1f22726c62b..eb95ad514b45 100644--- a/posthog/clickhouse/hcl/sql/dev/auxiliary.sql+++ b/posthog/clickhouse/hcl/sql/dev/auxiliary.sql@@ -15,6 +15,18 @@ CREATE TABLE posthog.cohortpeople (sign Int8,version UInt64) ENGINE = Distributed('posthog_single_shard', 'posthog', 'cohortpeople', rand());+CREATE TABLE posthog.email_status_data (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;CREATE TABLE posthog.events (uuid UUID,event String,@@ -161,6 +173,15 @@ CREATE TABLE posthog.ingestion_warnings_v2_distributed (_offset UInt64,_partition UInt64) ENGINE = Distributed('aux', 'posthog', 'ingestion_warnings_v2');+CREATE TABLE posthog.kafka_email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';CREATE TABLE posthog.kafka_hog_invocation_results (team_id Int64,function_kind LowCardinality(String),@@ -1314,6 +1335,18 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (lc_estimated_rows SimpleAggregateFunction(max, Int64),lc_estimated_bytes SimpleAggregateFunction(max, Int64)) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');+CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status;CREATE MATERIALIZED VIEW posthog.hog_invocation_results_mv TO posthog.hog_invocation_results_data (team_id Int64, function_kind LowCardinality(String), function_id String, invocation_id String, parent_run_id String, status LowCardinality(String), attempts UInt8, is_retry UInt8, scheduled_at DateTime64(6, 'UTC'), first_scheduled_at DateTime64(6, 'UTC'), started_at Nullable(DateTime64(6, 'UTC')), finished_at Nullable(DateTime64(6, 'UTC')), duration_ms Nullable(UInt32), error_kind LowCardinality(String), error_message String, event_uuid String, distinct_id String, person_id String, invocation_globals String, version UInt64, is_deleted UInt8, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,function_kind,@@ -1635,6 +1668,18 @@ CREATE TABLE posthog.conversion_goal_attributed_preaggregated (computed_at DateTime64(6, 'UTC') DEFAULT now(),expires_at Date DEFAULT today() + toIntervalDay(7)) ENGINE = Distributed('aux', 'posthog', 'sharded_conversion_goal_attributed_preaggregated', cityHash64(person_id));+CREATE TABLE posthog.email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/dev/data.sql b/posthog/clickhouse/hcl/sql/dev/data.sqlindex dfa23c56419f..45d354945d44 100644--- a/posthog/clickhouse/hcl/sql/dev/data.sql+++ b/posthog/clickhouse/hcl/sql/dev/data.sql@@ -276,6 +276,18 @@ CREATE TABLE posthog.duplicate_events (_partition UInt64,INDEX kafka_timestamp_minmax_duplicate_events _timestamp TYPE minmax GRANULARITY 3) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.duplicate_events', '{replica}-{shard}') ORDER BY (team_id, distinct_id, event, inserted_at) PARTITION BY toYYYYMMDD(inserted_at) TTL inserted_at + toIntervalDay(7) SETTINGS index_granularity = 512;+CREATE TABLE posthog.email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql b/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sqlindex 32aa1b140308..89fa3324b427 100644--- a/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql+++ b/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql@@ -15,6 +15,18 @@ CREATE TABLE posthog.cohortpeople (sign Int8,version UInt64) ENGINE = Distributed('posthog_single_shard', 'posthog', 'cohortpeople', rand());+CREATE TABLE posthog.email_status_data (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;CREATE TABLE posthog.events (uuid UUID,event String,@@ -161,6 +173,15 @@ CREATE TABLE posthog.ingestion_warnings_v2_distributed (_offset UInt64,_partition UInt64) ENGINE = Distributed('aux', 'posthog', 'ingestion_warnings_v2');+CREATE TABLE posthog.kafka_email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';CREATE TABLE posthog.kafka_error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,@@ -1318,6 +1339,18 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (lc_estimated_rows SimpleAggregateFunction(max, Int64),lc_estimated_bytes SimpleAggregateFunction(max, Int64)) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');+CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status;CREATE MATERIALIZED VIEW posthog.error_tracking_fingerprint_issue_state_mv TO posthog.writable_error_tracking_fingerprint_issue_state (team_id Int64, fingerprint String, issue_id UUID, issue_name Nullable(String), issue_description Nullable(String), issue_status String, issue_severity Nullable(String), assigned_user_id Nullable(Int64), assigned_role_id Nullable(UUID), first_seen DateTime64(3, 'UTC'), is_deleted Int8, version Int64, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,fingerprint,@@ -1656,6 +1689,18 @@ CREATE TABLE posthog.conversion_goal_attributed_preaggregated (computed_at DateTime64(6, 'UTC') DEFAULT now(),expires_at Date DEFAULT today() + toIntervalDay(7)) ENGINE = Distributed('aux', 'posthog', 'sharded_conversion_goal_attributed_preaggregated', cityHash64(person_id));+CREATE TABLE posthog.email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/local-multi/data.sql b/posthog/clickhouse/hcl/sql/local-multi/data.sqlindex 3cc9ef918295..5bce7827aabb 100644--- a/posthog/clickhouse/hcl/sql/local-multi/data.sql+++ b/posthog/clickhouse/hcl/sql/local-multi/data.sql@@ -233,6 +233,18 @@ CREATE TABLE posthog.duplicate_events (_partition UInt64,INDEX kafka_timestamp_minmax_duplicate_events _timestamp TYPE minmax GRANULARITY 3) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.duplicate_events', '{replica}-{shard}') ORDER BY (team_id, distinct_id, event, inserted_at) PARTITION BY toYYYYMMDD(inserted_at) TTL inserted_at + toIntervalDay(7) SETTINGS index_granularity = 512;+CREATE TABLE posthog.email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sqlindex 40eac1e062f8..87260bcb07db 100644--- a/posthog/clickhouse/hcl/sql/local-single/all.sql+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql@@ -130,6 +130,18 @@ CREATE TABLE posthog.duplicate_events (_partition UInt64,INDEX kafka_timestamp_minmax_duplicate_events _timestamp TYPE minmax GRANULARITY 3) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.duplicate_events', '{replica}-{shard}') ORDER BY (team_id, distinct_id, event, inserted_at) PARTITION BY toYYYYMMDD(inserted_at) TTL inserted_at + toIntervalDay(7) SETTINGS index_granularity = 512;+CREATE TABLE posthog.email_status_data (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;CREATE TABLE posthog.error_tracking_issue_fingerprint_overrides (team_id Int64,fingerprint String,@@ -348,6 +360,15 @@ CREATE TABLE posthog.kafka_duplicate_events (distinct_fields String,inserted_at DateTime64(3, 'UTC')) ENGINE = Kafka(msk_cluster) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_duplicate_events', kafka_topic_list = 'clickhouse_ingestion_events_duplicates';+CREATE TABLE posthog.kafka_email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';CREATE TABLE posthog.kafka_error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,@@ -4537,6 +4558,18 @@ CREATE MATERIALIZED VIEW posthog.duplicate_events_mv TO posthog.writable_duplica_offset,_partitionFROM posthog.kafka_duplicate_events;+CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status;CREATE MATERIALIZED VIEW posthog.error_tracking_fingerprint_issue_state_mv TO posthog.writable_error_tracking_fingerprint_issue_state (team_id Int64, fingerprint String, issue_id UUID, issue_name Nullable(String), issue_description Nullable(String), issue_status String, issue_severity Nullable(String), assigned_user_id Nullable(Int64), assigned_role_id Nullable(UUID), first_seen DateTime64(3, 'UTC'), is_deleted Int8, version Int64, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,fingerprint,@@ -6394,6 +6427,18 @@ CREATE TABLE posthog.distributed_posthog_document_embeddings_text_embedding_3_sm_offset UInt64,_partition UInt64) ENGINE = Distributed('posthog', 'posthog', 'sharded_posthog_document_embeddings_text_embedding_3_small_1536', cityHash64(document_id));+CREATE TABLE posthog.email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql b/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sqlindex 950b2f0d12bc..1854007c14e8 100644--- a/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql+++ b/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql@@ -15,6 +15,18 @@ CREATE TABLE posthog.cohortpeople (sign Int8,version UInt64) ENGINE = Distributed('posthog_single_shard', 'posthog', 'cohortpeople', rand());+CREATE TABLE posthog.email_status_data (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;CREATE TABLE posthog.events (uuid UUID,event String,@@ -161,6 +173,15 @@ CREATE TABLE posthog.ingestion_warnings_v2_distributed (_offset UInt64,_partition UInt64) ENGINE = Distributed('aux', 'posthog', 'ingestion_warnings_v2');+CREATE TABLE posthog.kafka_email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';CREATE TABLE posthog.kafka_hog_invocation_results (team_id Int64,function_kind LowCardinality(String),@@ -1314,6 +1335,18 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (lc_estimated_rows SimpleAggregateFunction(max, Int64),lc_estimated_bytes SimpleAggregateFunction(max, Int64)) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');+CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status;CREATE MATERIALIZED VIEW posthog.hog_invocation_results_mv TO posthog.hog_invocation_results_data (team_id Int64, function_kind LowCardinality(String), function_id String, invocation_id String, parent_run_id String, status LowCardinality(String), attempts UInt8, is_retry UInt8, scheduled_at DateTime64(6, 'UTC'), first_scheduled_at DateTime64(6, 'UTC'), started_at Nullable(DateTime64(6, 'UTC')), finished_at Nullable(DateTime64(6, 'UTC')), duration_ms Nullable(UInt32), error_kind LowCardinality(String), error_message String, event_uuid String, distinct_id String, person_id String, invocation_globals String, version UInt64, is_deleted UInt8, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,function_kind,@@ -1635,6 +1668,18 @@ CREATE TABLE posthog.conversion_goal_attributed_preaggregated (computed_at DateTime64(6, 'UTC') DEFAULT now(),expires_at Date DEFAULT today() + toIntervalDay(7)) ENGINE = Distributed('aux', 'posthog', 'sharded_conversion_goal_attributed_preaggregated', cityHash64(person_id));+CREATE TABLE posthog.email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/prod-eu/data.sql b/posthog/clickhouse/hcl/sql/prod-eu/data.sqlindex 9f05bb5663e6..ce33ba61df44 100644--- a/posthog/clickhouse/hcl/sql/prod-eu/data.sql+++ b/posthog/clickhouse/hcl/sql/prod-eu/data.sql@@ -289,6 +289,18 @@ CREATE TABLE posthog.duplicate_events (_partition UInt64,INDEX kafka_timestamp_minmax_duplicate_events _timestamp TYPE minmax GRANULARITY 3) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.duplicate_events', '{replica}-{shard}') ORDER BY (team_id, distinct_id, event, inserted_at) PARTITION BY toYYYYMMDD(inserted_at) TTL inserted_at + toIntervalDay(7) SETTINGS index_granularity = 512;+CREATE TABLE posthog.email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql b/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sqlindex 829357461413..a4c004ae6549 100644--- a/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql+++ b/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql@@ -15,6 +15,18 @@ CREATE TABLE posthog.cohortpeople (sign Int8,version UInt64) ENGINE = Distributed('posthog_single_shard', 'posthog', 'cohortpeople', rand());+CREATE TABLE posthog.email_status_data (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;CREATE TABLE posthog.events (uuid UUID,event String,@@ -161,6 +173,15 @@ CREATE TABLE posthog.ingestion_warnings_v2_distributed (_offset UInt64,_partition UInt64) ENGINE = Distributed('aux', 'posthog', 'ingestion_warnings_v2');+CREATE TABLE posthog.kafka_email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';CREATE TABLE posthog.kafka_hog_invocation_results (team_id Int64,function_kind LowCardinality(String),@@ -1314,6 +1335,18 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (lc_estimated_rows SimpleAggregateFunction(max, Int64),lc_estimated_bytes SimpleAggregateFunction(max, Int64)) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');+CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+FROM posthog.kafka_email_status;CREATE MATERIALIZED VIEW posthog.hog_invocation_results_mv TO posthog.hog_invocation_results_data (team_id Int64, function_kind LowCardinality(String), function_id String, invocation_id String, parent_run_id String, status LowCardinality(String), attempts UInt8, is_retry UInt8, scheduled_at DateTime64(6, 'UTC'), first_scheduled_at DateTime64(6, 'UTC'), started_at Nullable(DateTime64(6, 'UTC')), finished_at Nullable(DateTime64(6, 'UTC')), duration_ms Nullable(UInt32), error_kind LowCardinality(String), error_message String, event_uuid String, distinct_id String, person_id String, invocation_globals String, version UInt64, is_deleted UInt8, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTteam_id,function_kind,@@ -1635,6 +1668,18 @@ CREATE TABLE posthog.conversion_goal_attributed_preaggregated (computed_at DateTime64(6, 'UTC') DEFAULT now(),expires_at Date DEFAULT today() + toIntervalDay(7)) ENGINE = Distributed('aux', 'posthog', 'sharded_conversion_goal_attributed_preaggregated', cityHash64(person_id));+CREATE TABLE posthog.email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/hcl/sql/prod-us/data.sql b/posthog/clickhouse/hcl/sql/prod-us/data.sqlindex a1c7688c4451..1ee516670807 100644--- a/posthog/clickhouse/hcl/sql/prod-us/data.sql+++ b/posthog/clickhouse/hcl/sql/prod-us/data.sql@@ -299,6 +299,18 @@ CREATE TABLE posthog.duplicate_events (_partition UInt64,INDEX kafka_timestamp_minmax_duplicate_events _timestamp TYPE minmax GRANULARITY 3) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.duplicate_events', '{replica}-{shard}') ORDER BY (team_id, distinct_id, event, inserted_at) PARTITION BY toYYYYMMDD(inserted_at) TTL inserted_at + toIntervalDay(7) SETTINGS default_compression_codec = 'lz4', index_granularity = 512;+CREATE TABLE posthog.email_status (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String,+ _timestamp DateTime,+ _offset UInt64,+ _partition UInt64+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');CREATE TABLE posthog.error_tracking_fingerprint_issue_state (team_id Int64,fingerprint String,diff --git a/posthog/clickhouse/kafka_engine.py b/posthog/clickhouse/kafka_engine.pyindex 4c15c47cc7b4..71175da3498c 100644--- a/posthog/clickhouse/kafka_engine.py+++ b/posthog/clickhouse/kafka_engine.py@@ -21,6 +21,7 @@ CONSUMER_GROUP_SESSION_REPLAY_FEATURES = "clickhouse_session_replay_features" ifCONSUMER_GROUP_LOG_ENTRIES = "clickhouse_log_entries_v3" if _US else "clickhouse_log_entries"CONSUMER_GROUP_HOG_INVOCATION_RESULTS = "clickhouse_hog_invocation_results"CONSUMER_GROUP_MESSAGE_ASSETS = "clickhouse_message_assets"+CONSUMER_GROUP_EMAIL_STATUS = "clickhouse_email_status"CONSUMER_GROUP_DOCUMENT_EMBEDDINGS = "clickhouse_document_embeddings2" if _US else "clickhouse_document_embeddings"CONSUMER_GROUP_HEATMAPS = "clickhouse_heatmaps" if _US else "group1"CONSUMER_GROUP_PRECALCULATED_EVENTS = "clickhouse_precalculated_events2" if _US else "clickhouse_prefiltered_events"diff --git a/posthog/clickhouse/migrations/0363_email_status.py b/posthog/clickhouse/migrations/0363_email_status.pynew file mode 100644index 000000000000..59755a563ab2--- /dev/null+++ b/posthog/clickhouse/migrations/0363_email_status.py@@ -0,0 +1,28 @@+from posthog.clickhouse.client.connection import NodeRole+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions++from products.messaging.backend.facade.clickhouse import (+ DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,+ EMAIL_STATUS_DATA_TABLE_SQL,+ EMAIL_STATUS_MV_SQL,+ KAFKA_EMAIL_STATUS_TABLE_SQL,+)++operations = [+ run_sql_with_exceptions(+ EMAIL_STATUS_DATA_TABLE_SQL(),+ node_roles=[NodeRole.AUX],+ ),+ run_sql_with_exceptions(+ KAFKA_EMAIL_STATUS_TABLE_SQL(),+ node_roles=[NodeRole.AUX],+ ),+ run_sql_with_exceptions(+ EMAIL_STATUS_MV_SQL(),+ node_roles=[NodeRole.AUX],+ ),+ run_sql_with_exceptions(+ DISTRIBUTED_EMAIL_STATUS_TABLE_SQL(),+ node_roles=[NodeRole.AUX, NodeRole.DATA],+ ),+]diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txtindex e7da1c40d93a..452e42cade07 100644--- a/posthog/clickhouse/migrations/max_migration.txt+++ b/posthog/clickhouse/migrations/max_migration.txt@@ -1 +1 @@-0362_flag_evaluations_ttl_92_days+0363_email_statusdiff --git a/posthog/clickhouse/schema.py b/posthog/clickhouse/schema.pyindex 1433d349dc72..be8c8fbf2eb8 100644--- a/posthog/clickhouse/schema.py+++ b/posthog/clickhouse/schema.py@@ -409,6 +409,12 @@ from products.error_tracking.backend.sql import (WRITABLE_ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL,)from products.event_definitions.backend.models.property_definition import PROPERTY_DEFINITIONS_TABLE_SQL+from products.messaging.backend.facade.clickhouse import (+ DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,+ EMAIL_STATUS_DATA_TABLE_SQL,+ EMAIL_STATUS_MV_SQL,+ KAFKA_EMAIL_STATUS_TABLE_SQL,+)from products.workflows.backend.facade.clickhouse import (DISTRIBUTED_MESSAGE_ASSETS_TABLE_SQL,KAFKA_MESSAGE_ASSETS_TABLE_SQL,@@ -446,6 +452,7 @@ CREATE_MERGETREE_TABLE_QUERIES = (APP_METRICS2_DATA_TABLE_SQL,HOG_INVOCATION_RESULTS_DATA_TABLE_SQL,MESSAGE_ASSETS_DATA_TABLE_SQL,+ EMAIL_STATUS_DATA_TABLE_SQL,PERFORMANCE_EVENTS_TABLE_SQL,SESSION_REPLAY_EVENTS_TABLE_SQL,WRITABLE_SESSION_REPLAY_EVENTS_TABLE_SQL,@@ -540,6 +547,7 @@ CREATE_DISTRIBUTED_TABLE_QUERIES = (DISTRIBUTED_APP_METRICS2_TABLE_SQL,DISTRIBUTED_HOG_INVOCATION_RESULTS_TABLE_SQL,DISTRIBUTED_MESSAGE_ASSETS_TABLE_SQL,+ DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,WRITABLE_PERFORMANCE_EVENTS_TABLE_SQL,DISTRIBUTED_PERFORMANCE_EVENTS_TABLE_SQL,DISTRIBUTED_SESSION_REPLAY_EVENTS_TABLE_SQL,@@ -646,6 +654,7 @@ CREATE_KAFKA_TABLE_QUERIES = (KAFKA_APP_METRICS2_TABLE_SQL,KAFKA_HOG_INVOCATION_RESULTS_TABLE_SQL,KAFKA_MESSAGE_ASSETS_TABLE_SQL,+ KAFKA_EMAIL_STATUS_TABLE_SQL,KAFKA_PERFORMANCE_EVENTS_TABLE_SQL,KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL,KAFKA_SESSION_REPLAY_FEATURES_TABLE_SQL,@@ -684,6 +693,7 @@ CREATE_MV_TABLE_QUERIES = (APP_METRICS2_MV_TABLE_SQL,HOG_INVOCATION_RESULTS_MV_SQL,MESSAGE_ASSETS_MV_SQL,+ EMAIL_STATUS_MV_SQL,PERFORMANCE_EVENTS_TABLE_MV_SQL,SESSION_REPLAY_EVENTS_TABLE_MV_SQL,SESSION_REPLAY_FEATURES_TABLE_MV_SQL,diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambrindex aa76937df1f7..ade87a65fc05 100644--- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr+++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr@@ -154,6 +154,28 @@'''# ---+# name: test_create_kafka_table_with_different_kafka_host[kafka_email_status]+ '''++ CREATE TABLE IF NOT EXISTS kafka_email_status+ (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String+ )+ ENGINE = Kafka(warpstream_cyclotron, kafka_topic_list = 'clickhouse_email_status_test', kafka_group_name = 'clickhouse_email_status', kafka_format = 'JSONEachRow')+ SETTINGS kafka_skip_broken_messages = 100,+ kafka_num_consumers = 1,+ kafka_thread_per_consumer = 1,+ kafka_poll_timeout_ms = 10000,+ kafka_max_block_size = 100000++ '''+# ---# name: test_create_kafka_table_with_different_kafka_host[kafka_error_tracking_fingerprint_issue_state]'''@@ -1543,6 +1565,74 @@'''# ---+# name: test_create_table_query[email_status]+ '''++ CREATE TABLE IF NOT EXISTS email_status+ (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String++ , _timestamp DateTime+ , _offset UInt64+ , _partition UInt64++ )+ ENGINE = Distributed('aux', 'posthog_test', 'email_status_data')++ '''+# ---+# name: test_create_table_query[email_status_data]+ '''++ CREATE TABLE IF NOT EXISTS email_status_data+ (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String++ , _timestamp DateTime+ , _offset UInt64+ , _partition UInt64++ )+ ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.email_status_data', '{replica}-{shard}', version)+ PARTITION BY toYYYYMMDD(created_at)+ ORDER BY (team_id, message_id)++ SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1++ '''+# ---+# name: test_create_table_query[email_status_mv]+ '''++ CREATE MATERIALIZED VIEW IF NOT EXISTS email_status_mv+ TO email_status_data+ AS SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+ FROM kafka_email_status++ '''+# ---# name: test_create_table_query[error_tracking_fingerprint_issue_state]'''@@ -2580,6 +2670,28 @@'''# ---+# name: test_create_table_query[kafka_email_status]+ '''++ CREATE TABLE IF NOT EXISTS kafka_email_status+ (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String+ )+ ENGINE = Kafka(warpstream_cyclotron, kafka_topic_list = 'clickhouse_email_status_test', kafka_group_name = 'clickhouse_email_status', kafka_format = 'JSONEachRow')+ SETTINGS kafka_skip_broken_messages = 100,+ kafka_num_consumers = 1,+ kafka_thread_per_consumer = 1,+ kafka_poll_timeout_ms = 10000,+ kafka_max_block_size = 100000++ '''+# ---# name: test_create_table_query[kafka_error_tracking_fingerprint_issue_state]'''@@ -11805,6 +11917,32 @@'''# ---+# name: test_create_table_query_replicated_and_storage[email_status_data]+ '''++ CREATE TABLE IF NOT EXISTS email_status_data+ (+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String++ , _timestamp DateTime+ , _offset UInt64+ , _partition UInt64++ )+ ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.email_status_data', '{replica}-{shard}', version)+ PARTITION BY toYYYYMMDD(created_at)+ ORDER BY (team_id, message_id)++ SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1++ '''+# ---# name: test_create_table_query_replicated_and_storage[error_tracking_issue_fingerprint_overrides]'''diff --git a/posthog/kafka_client/routing.py b/posthog/kafka_client/routing.pyindex f534707f93dc..49d2c5435dd1 100644--- a/posthog/kafka_client/routing.py+++ b/posthog/kafka_client/routing.py@@ -36,6 +36,7 @@ from posthog.kafka_client.topics import (KAFKA_DOCUMENT_EMBEDDINGS_INPUT_TOPIC,KAFKA_DOCUMENT_EMBEDDINGS_TOPIC,KAFKA_DWH_CDP_RAW_TABLE,+ KAFKA_EMAIL_STATUS,KAFKA_ERROR_TRACKING_FINGERPRINT_ISSUE_STATE,KAFKA_ERROR_TRACKING_ISSUE_FINGERPRINT,KAFKA_ERROR_TRACKING_ISSUE_FINGERPRINT_EMBEDDINGS,@@ -104,6 +105,7 @@ _DEFAULT_TOPIC_ROUTING: dict[str, KafkaClusterProfile] = {# warpstream-cyclotron named collection, so a Django producer has to target# the same cluster the CDP workers produce to or the rows never arrive.KAFKA_HOG_INVOCATION_RESULTS: KafkaClusterProfile.CYCLOTRON,+ KAFKA_EMAIL_STATUS: KafkaClusterProfile.CYCLOTRON,# --- CALCULATED_EVENTS (Warpstream calculated-events) ---KAFKA_COHORT_MEMBERSHIP_CHANGED: KafkaClusterProfile.CALCULATED_EVENTS,# --- REPLAY (Session replay) ---diff --git a/posthog/kafka_client/topics.py b/posthog/kafka_client/topics.pyindex 5a1bcd67d5b7..93908c9d18dc 100644--- a/posthog/kafka_client/topics.py+++ b/posthog/kafka_client/topics.py@@ -35,6 +35,7 @@ KAFKA_LOG_ENTRIES = f"{KAFKA_PREFIX}log_entries{SUFFIX}"KAFKA_LOG_ENTRIES_V2_TEST = f"{KAFKA_PREFIX}log_entries_v2_test{SUFFIX}"KAFKA_HOG_INVOCATION_RESULTS = f"{KAFKA_PREFIX}clickhouse_hog_invocation_results{SUFFIX}"KAFKA_MESSAGE_ASSETS = f"{KAFKA_PREFIX}clickhouse_message_assets{SUFFIX}"+KAFKA_EMAIL_STATUS = f"{KAFKA_PREFIX}clickhouse_email_status{SUFFIX}"KAFKA_CLICKHOUSE_HEATMAP_EVENTS = f"{KAFKA_PREFIX}clickhouse_heatmap_events{SUFFIX}"KAFKA_CLICKHOUSE_AI_EVENTS_JSON = f"{KAFKA_PREFIX}clickhouse_ai_events_json{SUFFIX}"diff --git a/products/messaging/backend/facade/clickhouse.py b/products/messaging/backend/facade/clickhouse.pynew file mode 100644index 000000000000..ade6d5e1aad2--- /dev/null+++ b/products/messaging/backend/facade/clickhouse.py@@ -0,0 +1,12 @@+"""The ClickHouse tables core creates and migrates for this product.++Wiring, not data: `posthog/clickhouse/schema.py` builds these in a local or test ClickHouse, and+the migration that created them applies the same definitions to a cluster.+"""++from products.messaging.backend.models.email_status_sql import (+ DISTRIBUTED_EMAIL_STATUS_TABLE_SQL as DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,+ EMAIL_STATUS_DATA_TABLE_SQL as EMAIL_STATUS_DATA_TABLE_SQL,+ EMAIL_STATUS_MV_SQL as EMAIL_STATUS_MV_SQL,+ KAFKA_EMAIL_STATUS_TABLE_SQL as KAFKA_EMAIL_STATUS_TABLE_SQL,+)diff --git a/products/messaging/backend/models/email_status_sql.py b/products/messaging/backend/models/email_status_sql.pynew file mode 100644index 000000000000..d3abc64a3ba3--- /dev/null+++ b/products/messaging/backend/models/email_status_sql.py@@ -0,0 +1,109 @@+from django.conf import settings++from posthog.clickhouse.kafka_engine import (+ CONSUMER_GROUP_EMAIL_STATUS,+ KAFKA_COLUMNS_WITH_PARTITION,+ kafka_engine,+ ttl_period,+)+from posthog.clickhouse.table_engines import Distributed, ReplacingMergeTree, ReplicationScheme+from posthog.kafka_client.topics import KAFKA_EMAIL_STATUS++EMAIL_STATUS_RETENTION_DAYS = 30+# Parts drop by whole day of `created_at`, so one more day keeps an email created late in a day+# for its full retention.+EMAIL_STATUS_TTL_DAYS = EMAIL_STATUS_RETENTION_DAYS + 1++# AUX-resident table family modelled on `hog_invocation_results`. Each status change of a+# transactional email is one row. A message's rows replace each other by `version`, which puts the+# status rank above the status time, so a late `sent` never replaces `delivered`.+EMAIL_STATUS_TABLE = "email_status"+EMAIL_STATUS_DATA_TABLE = f"{EMAIL_STATUS_TABLE}_data"+KAFKA_EMAIL_STATUS_TABLE = f"kafka_{EMAIL_STATUS_TABLE}"+EMAIL_STATUS_MV_TABLE = f"{EMAIL_STATUS_TABLE}_mv"+++EMAIL_STATUS_KAFKA_COLUMNS = """+ team_id Int64,+ message_id UUID,+ status LowCardinality(String),+ version UInt64,+ created_at DateTime64(6, 'UTC'),+ status_at DateTime64(6, 'UTC'),+ reason String+""".strip()+++# Every row of a message carries the same `created_at`, so the rows share a partition and the+# ReplacingMergeTree can collapse them.+EMAIL_STATUS_DATA_TABLE_SQL = lambda: (+ f"""+CREATE TABLE IF NOT EXISTS {EMAIL_STATUS_DATA_TABLE}+(+ {EMAIL_STATUS_KAFKA_COLUMNS}+ {KAFKA_COLUMNS_WITH_PARTITION}+)+ENGINE = {ReplacingMergeTree(EMAIL_STATUS_DATA_TABLE, ver="version", replication_scheme=ReplicationScheme.REPLICATED)}+PARTITION BY toYYYYMMDD(created_at)+ORDER BY (team_id, message_id)+{ttl_period("created_at", EMAIL_STATUS_TTL_DAYS, unit="DAY")}+SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1+"""+)+++# Created on both AUX and DATA, so that queries from either cluster reach the data.+DISTRIBUTED_EMAIL_STATUS_TABLE_SQL = lambda: (+ f"""+CREATE TABLE IF NOT EXISTS {EMAIL_STATUS_TABLE}+(+ {EMAIL_STATUS_KAFKA_COLUMNS}+ {KAFKA_COLUMNS_WITH_PARTITION}+)+ENGINE = {Distributed(data_table=EMAIL_STATUS_DATA_TABLE, cluster=settings.CLICKHOUSE_AUX_CLUSTER)}+"""+)+++# The CDP producer writes to the cyclotron WarpStream cluster, so the Kafka table reads from its+# named collection.+KAFKA_EMAIL_STATUS_TABLE_SQL = lambda: (+ f"""+CREATE TABLE IF NOT EXISTS {KAFKA_EMAIL_STATUS_TABLE}+(+ {EMAIL_STATUS_KAFKA_COLUMNS}+)+ENGINE = {+ kafka_engine(+ topic=KAFKA_EMAIL_STATUS,+ group=CONSUMER_GROUP_EMAIL_STATUS,+ named_collection=settings.CLICKHOUSE_KAFKA_WARPSTREAM_CYCLOTRON_NAMED_COLLECTION,+ )+ }+SETTINGS kafka_skip_broken_messages = 100,+ kafka_num_consumers = 1,+ kafka_thread_per_consumer = 1,+ kafka_poll_timeout_ms = 10000,+ kafka_max_block_size = 100000+"""+)+++EMAIL_STATUS_MV_SQL = lambda: (+ f"""+CREATE MATERIALIZED VIEW IF NOT EXISTS {EMAIL_STATUS_MV_TABLE}+TO {EMAIL_STATUS_DATA_TABLE}+AS SELECT+ team_id,+ message_id,+ status,+ version,+ created_at,+ status_at,+ reason,+ _timestamp,+ _offset,+ _partition+FROM {KAFKA_EMAIL_STATUS_TABLE}+"""+)