Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(messaging): add the email_status clickhouse table (#124696)

Harley Alexanderf68c97401ee2
diff --git a/docker/kafka/topics.txt b/docker/kafka/topics.txt
index 40380d9fcb86..87771516e35b 100644
--- a/docker/kafka/topics.txt
+++ b/docker/kafka/topics.txt
@@ -16,6 +16,7 @@ clickhouse_app_metrics
clickhouse_app_metrics2
clickhouse_billing_usage_records
clickhouse_document_embeddings
+clickhouse_email_status
clickhouse_error_tracking_fingerprint_issue_state
clickhouse_error_tracking_issue_fingerprint
clickhouse_events_json
diff --git a/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl b/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl
index e96c12c64e2a..a90ed9b39d84 100644
--- a/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl
+++ b/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl
@@ -112,6 +112,89 @@ database "posthog" {
}
}
+ table "email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_data"
+ }
+ }
+
+ table "email_status_data" {
+ order_by = ["team_id", "message_id"]
+ partition_by = "toYYYYMMDD(created_at)"
+ ttl = "toDate(created_at) + toIntervalDay(31)"
+ settings = {
+ index_granularity = "1024"
+ ttl_only_drop_parts = "1"
+ }
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "replicated_replacing_merge_tree" {
+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_data"
+ replica_name = "{replica}-{shard}"
+ version_column = "version"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -797,6 +880,41 @@ database "posthog" {
}
}
+ table "kafka_email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ engine "kafka" {
+ collection = "warpstream_cyclotron"
+ topic_list = "clickhouse_email_status"
+ group_name = "clickhouse_email_status"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 100000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ thread_per_consumer = true
+ }
+ }
+
table "kafka_hog_invocation_results" {
column "team_id" {
type = "Int64"
@@ -5213,6 +5331,55 @@ database "posthog" {
}
}
+ materialized_view "email_status_mv" {
+ to_table = "posthog.email_status_data"
+ query = <<SQL
+SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status
+SQL
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "Nullable(DateTime)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ }
+
materialized_view "hog_invocation_results_mv" {
to_table = "posthog.hog_invocation_results_data"
query = <<SQL
diff --git a/posthog/clickhouse/hcl/golden/dev/data.hcl b/posthog/clickhouse/hcl/golden/dev/data.hcl
index 15b6bbfe2688..436c085a09a4 100644
--- a/posthog/clickhouse/hcl/golden/dev/data.hcl
+++ b/posthog/clickhouse/hcl/golden/dev/data.hcl
@@ -1214,6 +1214,44 @@ database "posthog" {
}
}
+ table "email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_data"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
diff --git a/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl b/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl
index 4f13b043f4ec..5f44cbb02d30 100644
--- a/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl
+++ b/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl
@@ -112,6 +112,89 @@ database "posthog" {
}
}
+ table "email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_data"
+ }
+ }
+
+ table "email_status_data" {
+ order_by = ["team_id", "message_id"]
+ partition_by = "toYYYYMMDD(created_at)"
+ ttl = "toDate(created_at) + toIntervalDay(31)"
+ settings = {
+ index_granularity = "1024"
+ ttl_only_drop_parts = "1"
+ }
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "replicated_replacing_merge_tree" {
+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_data"
+ replica_name = "{replica}-{shard}"
+ version_column = "version"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -796,6 +879,41 @@ database "posthog" {
}
}
+ table "kafka_email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ engine "kafka" {
+ collection = "warpstream_cyclotron"
+ topic_list = "clickhouse_email_status"
+ group_name = "clickhouse_email_status"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 100000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ thread_per_consumer = true
+ }
+ }
+
table "kafka_error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -5220,6 +5338,55 @@ database "posthog" {
}
}
+ materialized_view "email_status_mv" {
+ to_table = "posthog.email_status_data"
+ query = <<SQL
+SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status
+SQL
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "Nullable(DateTime)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ }
+
materialized_view "error_tracking_fingerprint_issue_state_mv" {
to_table = "posthog.writable_error_tracking_fingerprint_issue_state"
query = <<SQL
diff --git a/posthog/clickhouse/hcl/golden/local-multi/data.hcl b/posthog/clickhouse/hcl/golden/local-multi/data.hcl
index 58c0c7a4a87e..8dc5fb3408e6 100644
--- a/posthog/clickhouse/hcl/golden/local-multi/data.hcl
+++ b/posthog/clickhouse/hcl/golden/local-multi/data.hcl
@@ -1160,6 +1160,44 @@ database "posthog" {
}
}
+ table "email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_data"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hcl
index 885606e02a99..a144006d9c37 100644
--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl
+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl
@@ -1187,6 +1187,89 @@ database "posthog" {
}
}
+ table "email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_data"
+ }
+ }
+
+ table "email_status_data" {
+ order_by = ["team_id", "message_id"]
+ partition_by = "toYYYYMMDD(created_at)"
+ ttl = "toDate(created_at) + toIntervalDay(31)"
+ settings = {
+ index_granularity = "1024"
+ ttl_only_drop_parts = "1"
+ }
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "replicated_replacing_merge_tree" {
+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_data"
+ replica_name = "{replica}-{shard}"
+ version_column = "version"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -2786,6 +2869,41 @@ database "posthog" {
}
}
+ table "kafka_email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ engine "kafka" {
+ collection = "warpstream_cyclotron"
+ topic_list = "clickhouse_email_status"
+ group_name = "clickhouse_email_status"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 100000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ thread_per_consumer = true
+ }
+ }
+
table "kafka_error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -20456,6 +20574,55 @@ SQL
}
}
+ materialized_view "email_status_mv" {
+ to_table = "posthog.email_status_data"
+ query = <<SQL
+SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status
+SQL
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "Nullable(DateTime)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ }
+
materialized_view "error_tracking_fingerprint_issue_state_mv" {
to_table = "posthog.writable_error_tracking_fingerprint_issue_state"
query = <<SQL
diff --git a/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl b/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl
index b51afa6a1e03..5f225b60fe66 100644
--- a/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl
@@ -112,6 +112,89 @@ database "posthog" {
}
}
+ table "email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_data"
+ }
+ }
+
+ table "email_status_data" {
+ order_by = ["team_id", "message_id"]
+ partition_by = "toYYYYMMDD(created_at)"
+ ttl = "toDate(created_at) + toIntervalDay(31)"
+ settings = {
+ index_granularity = "1024"
+ ttl_only_drop_parts = "1"
+ }
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "replicated_replacing_merge_tree" {
+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_data"
+ replica_name = "{replica}-{shard}"
+ version_column = "version"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -797,6 +880,41 @@ database "posthog" {
}
}
+ table "kafka_email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ engine "kafka" {
+ collection = "warpstream_cyclotron"
+ topic_list = "clickhouse_email_status"
+ group_name = "clickhouse_email_status"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 100000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ thread_per_consumer = true
+ }
+ }
+
table "kafka_hog_invocation_results" {
column "team_id" {
type = "Int64"
@@ -5213,6 +5331,55 @@ database "posthog" {
}
}
+ materialized_view "email_status_mv" {
+ to_table = "posthog.email_status_data"
+ query = <<SQL
+SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status
+SQL
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "Nullable(DateTime)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ }
+
materialized_view "hog_invocation_results_mv" {
to_table = "posthog.hog_invocation_results_data"
query = <<SQL
diff --git a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl b/posthog/clickhouse/hcl/golden/prod-eu/data.hcl
index 8d890d4e0d6a..b434e375ebe2 100644
--- a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-eu/data.hcl
@@ -1152,6 +1152,44 @@ database "posthog" {
}
}
+ table "email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_data"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
diff --git a/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl b/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl
index cea262562a94..93d692cd3ba3 100644
--- a/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl
@@ -112,6 +112,89 @@ database "posthog" {
}
}
+ table "email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_data"
+ }
+ }
+
+ table "email_status_data" {
+ order_by = ["team_id", "message_id"]
+ partition_by = "toYYYYMMDD(created_at)"
+ ttl = "toDate(created_at) + toIntervalDay(31)"
+ settings = {
+ index_granularity = "1024"
+ ttl_only_drop_parts = "1"
+ }
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "replicated_replacing_merge_tree" {
+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_data"
+ replica_name = "{replica}-{shard}"
+ version_column = "version"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -831,6 +914,41 @@ database "posthog" {
}
}
+ table "kafka_email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ engine "kafka" {
+ collection = "warpstream_cyclotron"
+ topic_list = "clickhouse_email_status"
+ group_name = "clickhouse_email_status"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 100000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ thread_per_consumer = true
+ }
+ }
+
table "kafka_hog_invocation_results" {
column "team_id" {
type = "Int64"
@@ -5251,6 +5369,55 @@ database "posthog" {
}
}
+ materialized_view "email_status_mv" {
+ to_table = "posthog.email_status_data"
+ query = <<SQL
+SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status
+SQL
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "Nullable(DateTime)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ }
+
materialized_view "hog_invocation_results_mv" {
to_table = "posthog.hog_invocation_results_data"
query = <<SQL
diff --git a/posthog/clickhouse/hcl/golden/prod-us/data.hcl b/posthog/clickhouse/hcl/golden/prod-us/data.hcl
index 0fd5d1e90fde..0a03c13f8977 100644
--- a/posthog/clickhouse/hcl/golden/prod-us/data.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-us/data.hcl
@@ -1246,6 +1246,44 @@ database "posthog" {
}
}
+ table "email_status" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_data"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
diff --git a/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sql b/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sql
new file mode 100644
index 000000000000..8b56f47befa9
--- /dev/null
+++ b/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sql
@@ -0,0 +1,12 @@
+SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status
diff --git a/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl b/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl
index 12981f14e134..0be12c26ae72 100644
--- a/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl
+++ b/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl
@@ -1131,4 +1131,78 @@ database "posthog" {
replica_name = "{replica}-{shard}"
}
}
+
+ table "email_status_data" {
+ order_by = ["team_id", "message_id"]
+ partition_by = "toYYYYMMDD(created_at)"
+ ttl = "toDate(created_at) + toIntervalDay(31)"
+ settings = {
+ index_granularity = "1024"
+ ttl_only_drop_parts = "1"
+ }
+ extend = "_email_status_columns"
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "replicated_replacing_merge_tree" {
+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_data"
+ replica_name = "{replica}-{shard}"
+ version_column = "version"
+ }
+ }
+ table "kafka_email_status" {
+ extend = "_email_status_columns"
+ engine "kafka" {
+ collection = "warpstream_cyclotron"
+ topic_list = "clickhouse_email_status"
+ group_name = "clickhouse_email_status"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 100000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ thread_per_consumer = true
+ }
+ }
+ materialized_view "email_status_mv" {
+ to_table = "posthog.email_status_data"
+ query = file("sql/email_status_mv.sql")
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "_timestamp" {
+ type = "Nullable(DateTime)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ }
}
diff --git a/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl b/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl
index 715c23703fa1..a177a37b0bba 100644
--- a/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl
+++ b/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl
@@ -61,6 +61,30 @@ database "posthog" {
default = "today() + toIntervalDay(7)"
}
}
+ table "_email_status_columns" {
+ abstract = true
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ }
table "_experiment_metric_events_preaggregated_columns" {
abstract = true
column "team_id" {
@@ -1096,6 +1120,23 @@ database "posthog" {
sharding_key = "cityHash64(person_id)"
}
}
+ table "email_status" {
+ extend = "_email_status_columns"
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_data"
+ }
+ }
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
diff --git a/posthog/clickhouse/hcl/sql/dev/auxiliary.sql b/posthog/clickhouse/hcl/sql/dev/auxiliary.sql
index c1f22726c62b..eb95ad514b45 100644
--- a/posthog/clickhouse/hcl/sql/dev/auxiliary.sql
+++ b/posthog/clickhouse/hcl/sql/dev/auxiliary.sql
@@ -15,6 +15,18 @@ CREATE TABLE posthog.cohortpeople (
sign Int8,
version UInt64
) ENGINE = Distributed('posthog_single_shard', 'posthog', 'cohortpeople', rand());
+CREATE TABLE posthog.email_status_data (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
CREATE TABLE posthog.events (
uuid UUID,
event String,
@@ -161,6 +173,15 @@ CREATE TABLE posthog.ingestion_warnings_v2_distributed (
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('aux', 'posthog', 'ingestion_warnings_v2');
+CREATE TABLE posthog.kafka_email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String
+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
CREATE TABLE posthog.kafka_hog_invocation_results (
team_id Int64,
function_kind LowCardinality(String),
@@ -1314,6 +1335,18 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (
lc_estimated_rows SimpleAggregateFunction(max, Int64),
lc_estimated_bytes SimpleAggregateFunction(max, Int64)
) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');
+CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status;
CREATE MATERIALIZED VIEW posthog.hog_invocation_results_mv TO posthog.hog_invocation_results_data (team_id Int64, function_kind LowCardinality(String), function_id String, invocation_id String, parent_run_id String, status LowCardinality(String), attempts UInt8, is_retry UInt8, scheduled_at DateTime64(6, 'UTC'), first_scheduled_at DateTime64(6, 'UTC'), started_at Nullable(DateTime64(6, 'UTC')), finished_at Nullable(DateTime64(6, 'UTC')), duration_ms Nullable(UInt32), error_kind LowCardinality(String), error_message String, event_uuid String, distinct_id String, person_id String, invocation_globals String, version UInt64, is_deleted UInt8, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
function_kind,
@@ -1635,6 +1668,18 @@ CREATE TABLE posthog.conversion_goal_attributed_preaggregated (
computed_at DateTime64(6, 'UTC') DEFAULT now(),
expires_at Date DEFAULT today() + toIntervalDay(7)
) ENGINE = Distributed('aux', 'posthog', 'sharded_conversion_goal_attributed_preaggregated', cityHash64(person_id));
+CREATE TABLE posthog.email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/dev/data.sql b/posthog/clickhouse/hcl/sql/dev/data.sql
index dfa23c56419f..45d354945d44 100644
--- a/posthog/clickhouse/hcl/sql/dev/data.sql
+++ b/posthog/clickhouse/hcl/sql/dev/data.sql
@@ -276,6 +276,18 @@ CREATE TABLE posthog.duplicate_events (
_partition UInt64,
INDEX kafka_timestamp_minmax_duplicate_events _timestamp TYPE minmax GRANULARITY 3
) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.duplicate_events', '{replica}-{shard}') ORDER BY (team_id, distinct_id, event, inserted_at) PARTITION BY toYYYYMMDD(inserted_at) TTL inserted_at + toIntervalDay(7) SETTINGS index_granularity = 512;
+CREATE TABLE posthog.email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql b/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql
index 32aa1b140308..89fa3324b427 100644
--- a/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql
+++ b/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql
@@ -15,6 +15,18 @@ CREATE TABLE posthog.cohortpeople (
sign Int8,
version UInt64
) ENGINE = Distributed('posthog_single_shard', 'posthog', 'cohortpeople', rand());
+CREATE TABLE posthog.email_status_data (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
CREATE TABLE posthog.events (
uuid UUID,
event String,
@@ -161,6 +173,15 @@ CREATE TABLE posthog.ingestion_warnings_v2_distributed (
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('aux', 'posthog', 'ingestion_warnings_v2');
+CREATE TABLE posthog.kafka_email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String
+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
CREATE TABLE posthog.kafka_error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
@@ -1318,6 +1339,18 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (
lc_estimated_rows SimpleAggregateFunction(max, Int64),
lc_estimated_bytes SimpleAggregateFunction(max, Int64)
) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');
+CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status;
CREATE MATERIALIZED VIEW posthog.error_tracking_fingerprint_issue_state_mv TO posthog.writable_error_tracking_fingerprint_issue_state (team_id Int64, fingerprint String, issue_id UUID, issue_name Nullable(String), issue_description Nullable(String), issue_status String, issue_severity Nullable(String), assigned_user_id Nullable(Int64), assigned_role_id Nullable(UUID), first_seen DateTime64(3, 'UTC'), is_deleted Int8, version Int64, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
fingerprint,
@@ -1656,6 +1689,18 @@ CREATE TABLE posthog.conversion_goal_attributed_preaggregated (
computed_at DateTime64(6, 'UTC') DEFAULT now(),
expires_at Date DEFAULT today() + toIntervalDay(7)
) ENGINE = Distributed('aux', 'posthog', 'sharded_conversion_goal_attributed_preaggregated', cityHash64(person_id));
+CREATE TABLE posthog.email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/local-multi/data.sql b/posthog/clickhouse/hcl/sql/local-multi/data.sql
index 3cc9ef918295..5bce7827aabb 100644
--- a/posthog/clickhouse/hcl/sql/local-multi/data.sql
+++ b/posthog/clickhouse/hcl/sql/local-multi/data.sql
@@ -233,6 +233,18 @@ CREATE TABLE posthog.duplicate_events (
_partition UInt64,
INDEX kafka_timestamp_minmax_duplicate_events _timestamp TYPE minmax GRANULARITY 3
) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.duplicate_events', '{replica}-{shard}') ORDER BY (team_id, distinct_id, event, inserted_at) PARTITION BY toYYYYMMDD(inserted_at) TTL inserted_at + toIntervalDay(7) SETTINGS index_granularity = 512;
+CREATE TABLE posthog.email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sql
index 40eac1e062f8..87260bcb07db 100644
--- a/posthog/clickhouse/hcl/sql/local-single/all.sql
+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql
@@ -130,6 +130,18 @@ CREATE TABLE posthog.duplicate_events (
_partition UInt64,
INDEX kafka_timestamp_minmax_duplicate_events _timestamp TYPE minmax GRANULARITY 3
) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.duplicate_events', '{replica}-{shard}') ORDER BY (team_id, distinct_id, event, inserted_at) PARTITION BY toYYYYMMDD(inserted_at) TTL inserted_at + toIntervalDay(7) SETTINGS index_granularity = 512;
+CREATE TABLE posthog.email_status_data (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
CREATE TABLE posthog.error_tracking_issue_fingerprint_overrides (
team_id Int64,
fingerprint String,
@@ -348,6 +360,15 @@ CREATE TABLE posthog.kafka_duplicate_events (
distinct_fields String,
inserted_at DateTime64(3, 'UTC')
) ENGINE = Kafka(msk_cluster) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_duplicate_events', kafka_topic_list = 'clickhouse_ingestion_events_duplicates';
+CREATE TABLE posthog.kafka_email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String
+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
CREATE TABLE posthog.kafka_error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
@@ -4537,6 +4558,18 @@ CREATE MATERIALIZED VIEW posthog.duplicate_events_mv TO posthog.writable_duplica
_offset,
_partition
FROM posthog.kafka_duplicate_events;
+CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status;
CREATE MATERIALIZED VIEW posthog.error_tracking_fingerprint_issue_state_mv TO posthog.writable_error_tracking_fingerprint_issue_state (team_id Int64, fingerprint String, issue_id UUID, issue_name Nullable(String), issue_description Nullable(String), issue_status String, issue_severity Nullable(String), assigned_user_id Nullable(Int64), assigned_role_id Nullable(UUID), first_seen DateTime64(3, 'UTC'), is_deleted Int8, version Int64, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
fingerprint,
@@ -6394,6 +6427,18 @@ CREATE TABLE posthog.distributed_posthog_document_embeddings_text_embedding_3_sm
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('posthog', 'posthog', 'sharded_posthog_document_embeddings_text_embedding_3_small_1536', cityHash64(document_id));
+CREATE TABLE posthog.email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql b/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql
index 950b2f0d12bc..1854007c14e8 100644
--- a/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql
+++ b/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql
@@ -15,6 +15,18 @@ CREATE TABLE posthog.cohortpeople (
sign Int8,
version UInt64
) ENGINE = Distributed('posthog_single_shard', 'posthog', 'cohortpeople', rand());
+CREATE TABLE posthog.email_status_data (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
CREATE TABLE posthog.events (
uuid UUID,
event String,
@@ -161,6 +173,15 @@ CREATE TABLE posthog.ingestion_warnings_v2_distributed (
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('aux', 'posthog', 'ingestion_warnings_v2');
+CREATE TABLE posthog.kafka_email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String
+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
CREATE TABLE posthog.kafka_hog_invocation_results (
team_id Int64,
function_kind LowCardinality(String),
@@ -1314,6 +1335,18 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (
lc_estimated_rows SimpleAggregateFunction(max, Int64),
lc_estimated_bytes SimpleAggregateFunction(max, Int64)
) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');
+CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status;
CREATE MATERIALIZED VIEW posthog.hog_invocation_results_mv TO posthog.hog_invocation_results_data (team_id Int64, function_kind LowCardinality(String), function_id String, invocation_id String, parent_run_id String, status LowCardinality(String), attempts UInt8, is_retry UInt8, scheduled_at DateTime64(6, 'UTC'), first_scheduled_at DateTime64(6, 'UTC'), started_at Nullable(DateTime64(6, 'UTC')), finished_at Nullable(DateTime64(6, 'UTC')), duration_ms Nullable(UInt32), error_kind LowCardinality(String), error_message String, event_uuid String, distinct_id String, person_id String, invocation_globals String, version UInt64, is_deleted UInt8, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
function_kind,
@@ -1635,6 +1668,18 @@ CREATE TABLE posthog.conversion_goal_attributed_preaggregated (
computed_at DateTime64(6, 'UTC') DEFAULT now(),
expires_at Date DEFAULT today() + toIntervalDay(7)
) ENGINE = Distributed('aux', 'posthog', 'sharded_conversion_goal_attributed_preaggregated', cityHash64(person_id));
+CREATE TABLE posthog.email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/prod-eu/data.sql b/posthog/clickhouse/hcl/sql/prod-eu/data.sql
index 9f05bb5663e6..ce33ba61df44 100644
--- a/posthog/clickhouse/hcl/sql/prod-eu/data.sql
+++ b/posthog/clickhouse/hcl/sql/prod-eu/data.sql
@@ -289,6 +289,18 @@ CREATE TABLE posthog.duplicate_events (
_partition UInt64,
INDEX kafka_timestamp_minmax_duplicate_events _timestamp TYPE minmax GRANULARITY 3
) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.duplicate_events', '{replica}-{shard}') ORDER BY (team_id, distinct_id, event, inserted_at) PARTITION BY toYYYYMMDD(inserted_at) TTL inserted_at + toIntervalDay(7) SETTINGS index_granularity = 512;
+CREATE TABLE posthog.email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql b/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql
index 829357461413..a4c004ae6549 100644
--- a/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql
+++ b/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql
@@ -15,6 +15,18 @@ CREATE TABLE posthog.cohortpeople (
sign Int8,
version UInt64
) ENGINE = Distributed('posthog_single_shard', 'posthog', 'cohortpeople', rand());
+CREATE TABLE posthog.email_status_data (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
CREATE TABLE posthog.events (
uuid UUID,
event String,
@@ -161,6 +173,15 @@ CREATE TABLE posthog.ingestion_warnings_v2_distributed (
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('aux', 'posthog', 'ingestion_warnings_v2');
+CREATE TABLE posthog.kafka_email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String
+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
CREATE TABLE posthog.kafka_hog_invocation_results (
team_id Int64,
function_kind LowCardinality(String),
@@ -1314,6 +1335,18 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (
lc_estimated_rows SimpleAggregateFunction(max, Int64),
lc_estimated_bytes SimpleAggregateFunction(max, Int64)
) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');
+CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status;
CREATE MATERIALIZED VIEW posthog.hog_invocation_results_mv TO posthog.hog_invocation_results_data (team_id Int64, function_kind LowCardinality(String), function_id String, invocation_id String, parent_run_id String, status LowCardinality(String), attempts UInt8, is_retry UInt8, scheduled_at DateTime64(6, 'UTC'), first_scheduled_at DateTime64(6, 'UTC'), started_at Nullable(DateTime64(6, 'UTC')), finished_at Nullable(DateTime64(6, 'UTC')), duration_ms Nullable(UInt32), error_kind LowCardinality(String), error_message String, event_uuid String, distinct_id String, person_id String, invocation_globals String, version UInt64, is_deleted UInt8, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
function_kind,
@@ -1635,6 +1668,18 @@ CREATE TABLE posthog.conversion_goal_attributed_preaggregated (
computed_at DateTime64(6, 'UTC') DEFAULT now(),
expires_at Date DEFAULT today() + toIntervalDay(7)
) ENGINE = Distributed('aux', 'posthog', 'sharded_conversion_goal_attributed_preaggregated', cityHash64(person_id));
+CREATE TABLE posthog.email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/prod-us/data.sql b/posthog/clickhouse/hcl/sql/prod-us/data.sql
index a1c7688c4451..1ee516670807 100644
--- a/posthog/clickhouse/hcl/sql/prod-us/data.sql
+++ b/posthog/clickhouse/hcl/sql/prod-us/data.sql
@@ -299,6 +299,18 @@ CREATE TABLE posthog.duplicate_events (
_partition UInt64,
INDEX kafka_timestamp_minmax_duplicate_events _timestamp TYPE minmax GRANULARITY 3
) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.duplicate_events', '{replica}-{shard}') ORDER BY (team_id, distinct_id, event, inserted_at) PARTITION BY toYYYYMMDD(inserted_at) TTL inserted_at + toIntervalDay(7) SETTINGS default_compression_codec = 'lz4', index_granularity = 512;
+CREATE TABLE posthog.email_status (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/kafka_engine.py b/posthog/clickhouse/kafka_engine.py
index 4c15c47cc7b4..71175da3498c 100644
--- a/posthog/clickhouse/kafka_engine.py
+++ b/posthog/clickhouse/kafka_engine.py
@@ -21,6 +21,7 @@ CONSUMER_GROUP_SESSION_REPLAY_FEATURES = "clickhouse_session_replay_features" if
CONSUMER_GROUP_LOG_ENTRIES = "clickhouse_log_entries_v3" if _US else "clickhouse_log_entries"
CONSUMER_GROUP_HOG_INVOCATION_RESULTS = "clickhouse_hog_invocation_results"
CONSUMER_GROUP_MESSAGE_ASSETS = "clickhouse_message_assets"
+CONSUMER_GROUP_EMAIL_STATUS = "clickhouse_email_status"
CONSUMER_GROUP_DOCUMENT_EMBEDDINGS = "clickhouse_document_embeddings2" if _US else "clickhouse_document_embeddings"
CONSUMER_GROUP_HEATMAPS = "clickhouse_heatmaps" if _US else "group1"
CONSUMER_GROUP_PRECALCULATED_EVENTS = "clickhouse_precalculated_events2" if _US else "clickhouse_prefiltered_events"
diff --git a/posthog/clickhouse/migrations/0363_email_status.py b/posthog/clickhouse/migrations/0363_email_status.py
new file mode 100644
index 000000000000..59755a563ab2
--- /dev/null
+++ b/posthog/clickhouse/migrations/0363_email_status.py
@@ -0,0 +1,28 @@
+from posthog.clickhouse.client.connection import NodeRole
+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions
+
+from products.messaging.backend.facade.clickhouse import (
+ DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,
+ EMAIL_STATUS_DATA_TABLE_SQL,
+ EMAIL_STATUS_MV_SQL,
+ KAFKA_EMAIL_STATUS_TABLE_SQL,
+)
+
+operations = [
+ run_sql_with_exceptions(
+ EMAIL_STATUS_DATA_TABLE_SQL(),
+ node_roles=[NodeRole.AUX],
+ ),
+ run_sql_with_exceptions(
+ KAFKA_EMAIL_STATUS_TABLE_SQL(),
+ node_roles=[NodeRole.AUX],
+ ),
+ run_sql_with_exceptions(
+ EMAIL_STATUS_MV_SQL(),
+ node_roles=[NodeRole.AUX],
+ ),
+ run_sql_with_exceptions(
+ DISTRIBUTED_EMAIL_STATUS_TABLE_SQL(),
+ node_roles=[NodeRole.AUX, NodeRole.DATA],
+ ),
+]
diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txt
index e7da1c40d93a..452e42cade07 100644
--- a/posthog/clickhouse/migrations/max_migration.txt
+++ b/posthog/clickhouse/migrations/max_migration.txt
@@ -1 +1 @@
-0362_flag_evaluations_ttl_92_days
+0363_email_status
diff --git a/posthog/clickhouse/schema.py b/posthog/clickhouse/schema.py
index 1433d349dc72..be8c8fbf2eb8 100644
--- a/posthog/clickhouse/schema.py
+++ b/posthog/clickhouse/schema.py
@@ -409,6 +409,12 @@ from products.error_tracking.backend.sql import (
WRITABLE_ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_TABLE_SQL,
)
from products.event_definitions.backend.models.property_definition import PROPERTY_DEFINITIONS_TABLE_SQL
+from products.messaging.backend.facade.clickhouse import (
+ DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,
+ EMAIL_STATUS_DATA_TABLE_SQL,
+ EMAIL_STATUS_MV_SQL,
+ KAFKA_EMAIL_STATUS_TABLE_SQL,
+)
from products.workflows.backend.facade.clickhouse import (
DISTRIBUTED_MESSAGE_ASSETS_TABLE_SQL,
KAFKA_MESSAGE_ASSETS_TABLE_SQL,
@@ -446,6 +452,7 @@ CREATE_MERGETREE_TABLE_QUERIES = (
APP_METRICS2_DATA_TABLE_SQL,
HOG_INVOCATION_RESULTS_DATA_TABLE_SQL,
MESSAGE_ASSETS_DATA_TABLE_SQL,
+ EMAIL_STATUS_DATA_TABLE_SQL,
PERFORMANCE_EVENTS_TABLE_SQL,
SESSION_REPLAY_EVENTS_TABLE_SQL,
WRITABLE_SESSION_REPLAY_EVENTS_TABLE_SQL,
@@ -540,6 +547,7 @@ CREATE_DISTRIBUTED_TABLE_QUERIES = (
DISTRIBUTED_APP_METRICS2_TABLE_SQL,
DISTRIBUTED_HOG_INVOCATION_RESULTS_TABLE_SQL,
DISTRIBUTED_MESSAGE_ASSETS_TABLE_SQL,
+ DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,
WRITABLE_PERFORMANCE_EVENTS_TABLE_SQL,
DISTRIBUTED_PERFORMANCE_EVENTS_TABLE_SQL,
DISTRIBUTED_SESSION_REPLAY_EVENTS_TABLE_SQL,
@@ -646,6 +654,7 @@ CREATE_KAFKA_TABLE_QUERIES = (
KAFKA_APP_METRICS2_TABLE_SQL,
KAFKA_HOG_INVOCATION_RESULTS_TABLE_SQL,
KAFKA_MESSAGE_ASSETS_TABLE_SQL,
+ KAFKA_EMAIL_STATUS_TABLE_SQL,
KAFKA_PERFORMANCE_EVENTS_TABLE_SQL,
KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL,
KAFKA_SESSION_REPLAY_FEATURES_TABLE_SQL,
@@ -684,6 +693,7 @@ CREATE_MV_TABLE_QUERIES = (
APP_METRICS2_MV_TABLE_SQL,
HOG_INVOCATION_RESULTS_MV_SQL,
MESSAGE_ASSETS_MV_SQL,
+ EMAIL_STATUS_MV_SQL,
PERFORMANCE_EVENTS_TABLE_MV_SQL,
SESSION_REPLAY_EVENTS_TABLE_MV_SQL,
SESSION_REPLAY_FEATURES_TABLE_MV_SQL,
diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambr
index aa76937df1f7..ade87a65fc05 100644
--- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr
+++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr
@@ -154,6 +154,28 @@
'''
# ---
+# name: test_create_kafka_table_with_different_kafka_host[kafka_email_status]
+ '''
+
+ CREATE TABLE IF NOT EXISTS kafka_email_status
+ (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String
+ )
+ ENGINE = Kafka(warpstream_cyclotron, kafka_topic_list = 'clickhouse_email_status_test', kafka_group_name = 'clickhouse_email_status', kafka_format = 'JSONEachRow')
+ SETTINGS kafka_skip_broken_messages = 100,
+ kafka_num_consumers = 1,
+ kafka_thread_per_consumer = 1,
+ kafka_poll_timeout_ms = 10000,
+ kafka_max_block_size = 100000
+
+ '''
+# ---
# name: test_create_kafka_table_with_different_kafka_host[kafka_error_tracking_fingerprint_issue_state]
'''
@@ -1543,6 +1565,74 @@
'''
# ---
+# name: test_create_table_query[email_status]
+ '''
+
+ CREATE TABLE IF NOT EXISTS email_status
+ (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String
+
+ , _timestamp DateTime
+ , _offset UInt64
+ , _partition UInt64
+
+ )
+ ENGINE = Distributed('aux', 'posthog_test', 'email_status_data')
+
+ '''
+# ---
+# name: test_create_table_query[email_status_data]
+ '''
+
+ CREATE TABLE IF NOT EXISTS email_status_data
+ (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String
+
+ , _timestamp DateTime
+ , _offset UInt64
+ , _partition UInt64
+
+ )
+ ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.email_status_data', '{replica}-{shard}', version)
+ PARTITION BY toYYYYMMDD(created_at)
+ ORDER BY (team_id, message_id)
+
+ SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1
+
+ '''
+# ---
+# name: test_create_table_query[email_status_mv]
+ '''
+
+ CREATE MATERIALIZED VIEW IF NOT EXISTS email_status_mv
+ TO email_status_data
+ AS SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+ FROM kafka_email_status
+
+ '''
+# ---
# name: test_create_table_query[error_tracking_fingerprint_issue_state]
'''
@@ -2580,6 +2670,28 @@
'''
# ---
+# name: test_create_table_query[kafka_email_status]
+ '''
+
+ CREATE TABLE IF NOT EXISTS kafka_email_status
+ (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String
+ )
+ ENGINE = Kafka(warpstream_cyclotron, kafka_topic_list = 'clickhouse_email_status_test', kafka_group_name = 'clickhouse_email_status', kafka_format = 'JSONEachRow')
+ SETTINGS kafka_skip_broken_messages = 100,
+ kafka_num_consumers = 1,
+ kafka_thread_per_consumer = 1,
+ kafka_poll_timeout_ms = 10000,
+ kafka_max_block_size = 100000
+
+ '''
+# ---
# name: test_create_table_query[kafka_error_tracking_fingerprint_issue_state]
'''
@@ -11805,6 +11917,32 @@
'''
# ---
+# name: test_create_table_query_replicated_and_storage[email_status_data]
+ '''
+
+ CREATE TABLE IF NOT EXISTS email_status_data
+ (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String
+
+ , _timestamp DateTime
+ , _offset UInt64
+ , _partition UInt64
+
+ )
+ ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.email_status_data', '{replica}-{shard}', version)
+ PARTITION BY toYYYYMMDD(created_at)
+ ORDER BY (team_id, message_id)
+
+ SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1
+
+ '''
+# ---
# name: test_create_table_query_replicated_and_storage[error_tracking_issue_fingerprint_overrides]
'''
diff --git a/posthog/kafka_client/routing.py b/posthog/kafka_client/routing.py
index f534707f93dc..49d2c5435dd1 100644
--- a/posthog/kafka_client/routing.py
+++ b/posthog/kafka_client/routing.py
@@ -36,6 +36,7 @@ from posthog.kafka_client.topics import (
KAFKA_DOCUMENT_EMBEDDINGS_INPUT_TOPIC,
KAFKA_DOCUMENT_EMBEDDINGS_TOPIC,
KAFKA_DWH_CDP_RAW_TABLE,
+ KAFKA_EMAIL_STATUS,
KAFKA_ERROR_TRACKING_FINGERPRINT_ISSUE_STATE,
KAFKA_ERROR_TRACKING_ISSUE_FINGERPRINT,
KAFKA_ERROR_TRACKING_ISSUE_FINGERPRINT_EMBEDDINGS,
@@ -104,6 +105,7 @@ _DEFAULT_TOPIC_ROUTING: dict[str, KafkaClusterProfile] = {
# warpstream-cyclotron named collection, so a Django producer has to target
# the same cluster the CDP workers produce to or the rows never arrive.
KAFKA_HOG_INVOCATION_RESULTS: KafkaClusterProfile.CYCLOTRON,
+ KAFKA_EMAIL_STATUS: KafkaClusterProfile.CYCLOTRON,
# --- CALCULATED_EVENTS (Warpstream calculated-events) ---
KAFKA_COHORT_MEMBERSHIP_CHANGED: KafkaClusterProfile.CALCULATED_EVENTS,
# --- REPLAY (Session replay) ---
diff --git a/posthog/kafka_client/topics.py b/posthog/kafka_client/topics.py
index 5a1bcd67d5b7..93908c9d18dc 100644
--- a/posthog/kafka_client/topics.py
+++ b/posthog/kafka_client/topics.py
@@ -35,6 +35,7 @@ KAFKA_LOG_ENTRIES = f"{KAFKA_PREFIX}log_entries{SUFFIX}"
KAFKA_LOG_ENTRIES_V2_TEST = f"{KAFKA_PREFIX}log_entries_v2_test{SUFFIX}"
KAFKA_HOG_INVOCATION_RESULTS = f"{KAFKA_PREFIX}clickhouse_hog_invocation_results{SUFFIX}"
KAFKA_MESSAGE_ASSETS = f"{KAFKA_PREFIX}clickhouse_message_assets{SUFFIX}"
+KAFKA_EMAIL_STATUS = f"{KAFKA_PREFIX}clickhouse_email_status{SUFFIX}"
KAFKA_CLICKHOUSE_HEATMAP_EVENTS = f"{KAFKA_PREFIX}clickhouse_heatmap_events{SUFFIX}"
KAFKA_CLICKHOUSE_AI_EVENTS_JSON = f"{KAFKA_PREFIX}clickhouse_ai_events_json{SUFFIX}"
diff --git a/products/messaging/backend/facade/clickhouse.py b/products/messaging/backend/facade/clickhouse.py
new file mode 100644
index 000000000000..ade6d5e1aad2
--- /dev/null
+++ b/products/messaging/backend/facade/clickhouse.py
@@ -0,0 +1,12 @@
+"""The ClickHouse tables core creates and migrates for this product.
+
+Wiring, not data: `posthog/clickhouse/schema.py` builds these in a local or test ClickHouse, and
+the migration that created them applies the same definitions to a cluster.
+"""
+
+from products.messaging.backend.models.email_status_sql import (
+ DISTRIBUTED_EMAIL_STATUS_TABLE_SQL as DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,
+ EMAIL_STATUS_DATA_TABLE_SQL as EMAIL_STATUS_DATA_TABLE_SQL,
+ EMAIL_STATUS_MV_SQL as EMAIL_STATUS_MV_SQL,
+ KAFKA_EMAIL_STATUS_TABLE_SQL as KAFKA_EMAIL_STATUS_TABLE_SQL,
+)
diff --git a/products/messaging/backend/models/email_status_sql.py b/products/messaging/backend/models/email_status_sql.py
new file mode 100644
index 000000000000..d3abc64a3ba3
--- /dev/null
+++ b/products/messaging/backend/models/email_status_sql.py
@@ -0,0 +1,109 @@
+from django.conf import settings
+
+from posthog.clickhouse.kafka_engine import (
+ CONSUMER_GROUP_EMAIL_STATUS,
+ KAFKA_COLUMNS_WITH_PARTITION,
+ kafka_engine,
+ ttl_period,
+)
+from posthog.clickhouse.table_engines import Distributed, ReplacingMergeTree, ReplicationScheme
+from posthog.kafka_client.topics import KAFKA_EMAIL_STATUS
+
+EMAIL_STATUS_RETENTION_DAYS = 30
+# Parts drop by whole day of `created_at`, so one more day keeps an email created late in a day
+# for its full retention.
+EMAIL_STATUS_TTL_DAYS = EMAIL_STATUS_RETENTION_DAYS + 1
+
+# AUX-resident table family modelled on `hog_invocation_results`. Each status change of a
+# transactional email is one row. A message's rows replace each other by `version`, which puts the
+# status rank above the status time, so a late `sent` never replaces `delivered`.
+EMAIL_STATUS_TABLE = "email_status"
+EMAIL_STATUS_DATA_TABLE = f"{EMAIL_STATUS_TABLE}_data"
+KAFKA_EMAIL_STATUS_TABLE = f"kafka_{EMAIL_STATUS_TABLE}"
+EMAIL_STATUS_MV_TABLE = f"{EMAIL_STATUS_TABLE}_mv"
+
+
+EMAIL_STATUS_KAFKA_COLUMNS = """
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String
+""".strip()
+
+
+# Every row of a message carries the same `created_at`, so the rows share a partition and the
+# ReplacingMergeTree can collapse them.
+EMAIL_STATUS_DATA_TABLE_SQL = lambda: (
+ f"""
+CREATE TABLE IF NOT EXISTS {EMAIL_STATUS_DATA_TABLE}
+(
+ {EMAIL_STATUS_KAFKA_COLUMNS}
+ {KAFKA_COLUMNS_WITH_PARTITION}
+)
+ENGINE = {ReplacingMergeTree(EMAIL_STATUS_DATA_TABLE, ver="version", replication_scheme=ReplicationScheme.REPLICATED)}
+PARTITION BY toYYYYMMDD(created_at)
+ORDER BY (team_id, message_id)
+{ttl_period("created_at", EMAIL_STATUS_TTL_DAYS, unit="DAY")}
+SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1
+"""
+)
+
+
+# Created on both AUX and DATA, so that queries from either cluster reach the data.
+DISTRIBUTED_EMAIL_STATUS_TABLE_SQL = lambda: (
+ f"""
+CREATE TABLE IF NOT EXISTS {EMAIL_STATUS_TABLE}
+(
+ {EMAIL_STATUS_KAFKA_COLUMNS}
+ {KAFKA_COLUMNS_WITH_PARTITION}
+)
+ENGINE = {Distributed(data_table=EMAIL_STATUS_DATA_TABLE, cluster=settings.CLICKHOUSE_AUX_CLUSTER)}
+"""
+)
+
+
+# The CDP producer writes to the cyclotron WarpStream cluster, so the Kafka table reads from its
+# named collection.
+KAFKA_EMAIL_STATUS_TABLE_SQL = lambda: (
+ f"""
+CREATE TABLE IF NOT EXISTS {KAFKA_EMAIL_STATUS_TABLE}
+(
+ {EMAIL_STATUS_KAFKA_COLUMNS}
+)
+ENGINE = {
+ kafka_engine(
+ topic=KAFKA_EMAIL_STATUS,
+ group=CONSUMER_GROUP_EMAIL_STATUS,
+ named_collection=settings.CLICKHOUSE_KAFKA_WARPSTREAM_CYCLOTRON_NAMED_COLLECTION,
+ )
+ }
+SETTINGS kafka_skip_broken_messages = 100,
+ kafka_num_consumers = 1,
+ kafka_thread_per_consumer = 1,
+ kafka_poll_timeout_ms = 10000,
+ kafka_max_block_size = 100000
+"""
+)
+
+
+EMAIL_STATUS_MV_SQL = lambda: (
+ f"""
+CREATE MATERIALIZED VIEW IF NOT EXISTS {EMAIL_STATUS_MV_TABLE}
+TO {EMAIL_STATUS_DATA_TABLE}
+AS SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+FROM {KAFKA_EMAIL_STATUS_TABLE}
+"""
+)