Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(workflows): keep every transactional email status (#127744)

Daniel Marchukd86e11a43a07
diff --git a/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl b/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl
index 0d1a9e4d155e..c6e16571456c 100644
--- a/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl
+++ b/posthog/clickhouse/hcl/golden/dev/auxiliary.hcl
@@ -195,6 +195,106 @@ database "posthog" {
}
}
+ table "email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_events_data"
+ }
+ }
+
+ table "email_status_events_data" {
+ order_by = ["team_id", "created_at", "message_id", "status_at"]
+ partition_by = "toYYYYMMDD(created_at)"
+ ttl = "toDate(created_at) + toIntervalDay(31)"
+ settings = {
+ index_granularity = "1024"
+ ttl_only_drop_parts = "1"
+ }
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "replicated_merge_tree" {
+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_events_data"
+ replica_name = "{replica}-{shard}"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -922,6 +1022,50 @@ database "posthog" {
}
}
+ table "kafka_email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ engine "kafka" {
+ collection = "warpstream_cyclotron"
+ topic_list = "clickhouse_email_status"
+ group_name = "clickhouse_email_status_events"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 100000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ thread_per_consumer = true
+ }
+ }
+
table "kafka_hog_invocation_results" {
column "team_id" {
type = "Int64"
@@ -5345,6 +5489,67 @@ database "posthog" {
}
}
+ materialized_view "email_status_events_mv" {
+ to_table = "posthog.email_status_events_data"
+ query = <<SQL
+SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ recipient,
+ function_id,
+ sending_domain,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status_events
+SQL
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "Nullable(DateTime)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ }
+
materialized_view "email_status_mv" {
to_table = "posthog.email_status_data"
query = <<SQL
@@ -5360,6 +5565,7 @@ SELECT
_offset,
_partition
FROM posthog.kafka_email_status
+WHERE status NOT IN ('opened', 'clicked')
SQL
column "team_id" {
diff --git a/posthog/clickhouse/hcl/golden/dev/data.hcl b/posthog/clickhouse/hcl/golden/dev/data.hcl
index 44836f6f34e1..5a223f1568ac 100644
--- a/posthog/clickhouse/hcl/golden/dev/data.hcl
+++ b/posthog/clickhouse/hcl/golden/dev/data.hcl
@@ -1252,6 +1252,53 @@ database "posthog" {
}
}
+ table "email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_events_data"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
diff --git a/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl b/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl
index f08276d31c6f..2c8b3b741f6d 100644
--- a/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl
+++ b/posthog/clickhouse/hcl/golden/local-multi/auxiliary.hcl
@@ -195,6 +195,106 @@ database "posthog" {
}
}
+ table "email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_events_data"
+ }
+ }
+
+ table "email_status_events_data" {
+ order_by = ["team_id", "created_at", "message_id", "status_at"]
+ partition_by = "toYYYYMMDD(created_at)"
+ ttl = "toDate(created_at) + toIntervalDay(31)"
+ settings = {
+ index_granularity = "1024"
+ ttl_only_drop_parts = "1"
+ }
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "replicated_merge_tree" {
+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_events_data"
+ replica_name = "{replica}-{shard}"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -921,6 +1021,50 @@ database "posthog" {
}
}
+ table "kafka_email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ engine "kafka" {
+ collection = "warpstream_cyclotron"
+ topic_list = "clickhouse_email_status"
+ group_name = "clickhouse_email_status_events"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 100000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ thread_per_consumer = true
+ }
+ }
+
table "kafka_error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -5352,6 +5496,67 @@ database "posthog" {
}
}
+ materialized_view "email_status_events_mv" {
+ to_table = "posthog.email_status_events_data"
+ query = <<SQL
+SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ recipient,
+ function_id,
+ sending_domain,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status_events
+SQL
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "Nullable(DateTime)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ }
+
materialized_view "email_status_mv" {
to_table = "posthog.email_status_data"
query = <<SQL
@@ -5367,6 +5572,7 @@ SELECT
_offset,
_partition
FROM posthog.kafka_email_status
+WHERE status NOT IN ('opened', 'clicked')
SQL
column "team_id" {
diff --git a/posthog/clickhouse/hcl/golden/local-multi/data.hcl b/posthog/clickhouse/hcl/golden/local-multi/data.hcl
index 737e15785045..cc3fb9362333 100644
--- a/posthog/clickhouse/hcl/golden/local-multi/data.hcl
+++ b/posthog/clickhouse/hcl/golden/local-multi/data.hcl
@@ -1198,6 +1198,53 @@ database "posthog" {
}
}
+ table "email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_events_data"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hcl
index 415381254058..a45ad09f78a4 100644
--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl
+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl
@@ -1270,6 +1270,106 @@ database "posthog" {
}
}
+ table "email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_events_data"
+ }
+ }
+
+ table "email_status_events_data" {
+ order_by = ["team_id", "created_at", "message_id", "status_at"]
+ partition_by = "toYYYYMMDD(created_at)"
+ ttl = "toDate(created_at) + toIntervalDay(31)"
+ settings = {
+ index_granularity = "1024"
+ ttl_only_drop_parts = "1"
+ }
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "replicated_merge_tree" {
+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_events_data"
+ replica_name = "{replica}-{shard}"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -2918,6 +3018,50 @@ database "posthog" {
}
}
+ table "kafka_email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ engine "kafka" {
+ collection = "warpstream_cyclotron"
+ topic_list = "clickhouse_email_status"
+ group_name = "clickhouse_email_status_events"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 100000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ thread_per_consumer = true
+ }
+ }
+
table "kafka_error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -20579,6 +20723,67 @@ SQL
}
}
+ materialized_view "email_status_events_mv" {
+ to_table = "posthog.email_status_events_data"
+ query = <<SQL
+SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ recipient,
+ function_id,
+ sending_domain,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status_events
+SQL
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "Nullable(DateTime)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ }
+
materialized_view "email_status_mv" {
to_table = "posthog.email_status_data"
query = <<SQL
@@ -20594,6 +20799,7 @@ SELECT
_offset,
_partition
FROM posthog.kafka_email_status
+WHERE status NOT IN ('opened', 'clicked')
SQL
column "team_id" {
diff --git a/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl b/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl
index ec51438b5762..011c0b0dc0c0 100644
--- a/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-eu/auxiliary.hcl
@@ -195,6 +195,106 @@ database "posthog" {
}
}
+ table "email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_events_data"
+ }
+ }
+
+ table "email_status_events_data" {
+ order_by = ["team_id", "created_at", "message_id", "status_at"]
+ partition_by = "toYYYYMMDD(created_at)"
+ ttl = "toDate(created_at) + toIntervalDay(31)"
+ settings = {
+ index_granularity = "1024"
+ ttl_only_drop_parts = "1"
+ }
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "replicated_merge_tree" {
+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_events_data"
+ replica_name = "{replica}-{shard}"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -922,6 +1022,50 @@ database "posthog" {
}
}
+ table "kafka_email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ engine "kafka" {
+ collection = "warpstream_cyclotron"
+ topic_list = "clickhouse_email_status"
+ group_name = "clickhouse_email_status_events"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 100000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ thread_per_consumer = true
+ }
+ }
+
table "kafka_hog_invocation_results" {
column "team_id" {
type = "Int64"
@@ -5345,6 +5489,67 @@ database "posthog" {
}
}
+ materialized_view "email_status_events_mv" {
+ to_table = "posthog.email_status_events_data"
+ query = <<SQL
+SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ recipient,
+ function_id,
+ sending_domain,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status_events
+SQL
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "Nullable(DateTime)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ }
+
materialized_view "email_status_mv" {
to_table = "posthog.email_status_data"
query = <<SQL
@@ -5360,6 +5565,7 @@ SELECT
_offset,
_partition
FROM posthog.kafka_email_status
+WHERE status NOT IN ('opened', 'clicked')
SQL
column "team_id" {
diff --git a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl b/posthog/clickhouse/hcl/golden/prod-eu/data.hcl
index 41a754bfd235..a4de920c82ff 100644
--- a/posthog/clickhouse/hcl/golden/prod-eu/data.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-eu/data.hcl
@@ -1190,6 +1190,53 @@ database "posthog" {
}
}
+ table "email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_events_data"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
diff --git a/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl b/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl
index 6a10cb79c02c..dfc1bd77758d 100644
--- a/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-us/auxiliary.hcl
@@ -195,6 +195,106 @@ database "posthog" {
}
}
+ table "email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_events_data"
+ }
+ }
+
+ table "email_status_events_data" {
+ order_by = ["team_id", "created_at", "message_id", "status_at"]
+ partition_by = "toYYYYMMDD(created_at)"
+ ttl = "toDate(created_at) + toIntervalDay(31)"
+ settings = {
+ index_granularity = "1024"
+ ttl_only_drop_parts = "1"
+ }
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "replicated_merge_tree" {
+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_events_data"
+ replica_name = "{replica}-{shard}"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
@@ -956,6 +1056,50 @@ database "posthog" {
}
}
+ table "kafka_email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ engine "kafka" {
+ collection = "warpstream_cyclotron"
+ topic_list = "clickhouse_email_status"
+ group_name = "clickhouse_email_status_events"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 100000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ thread_per_consumer = true
+ }
+ }
+
table "kafka_hog_invocation_results" {
column "team_id" {
type = "Int64"
@@ -5383,6 +5527,67 @@ database "posthog" {
}
}
+ materialized_view "email_status_events_mv" {
+ to_table = "posthog.email_status_events_data"
+ query = <<SQL
+SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ recipient,
+ function_id,
+ sending_domain,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status_events
+SQL
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "Nullable(DateTime)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ }
+
materialized_view "email_status_mv" {
to_table = "posthog.email_status_data"
query = <<SQL
@@ -5398,6 +5603,7 @@ SELECT
_offset,
_partition
FROM posthog.kafka_email_status
+WHERE status NOT IN ('opened', 'clicked')
SQL
column "team_id" {
diff --git a/posthog/clickhouse/hcl/golden/prod-us/data.hcl b/posthog/clickhouse/hcl/golden/prod-us/data.hcl
index dabf7d8ac616..4d57267f8c30 100644
--- a/posthog/clickhouse/hcl/golden/prod-us/data.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-us/data.hcl
@@ -1284,6 +1284,53 @@ database "posthog" {
}
}
+ table "email_status_events" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_events_data"
+ }
+ }
+
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
diff --git a/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_events_mv.sql b/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_events_mv.sql
new file mode 100644
index 000000000000..c219f0d96376
--- /dev/null
+++ b/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_events_mv.sql
@@ -0,0 +1,15 @@
+SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ recipient,
+ function_id,
+ sending_domain,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status_events
diff --git a/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sql b/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sql
index 8b56f47befa9..9a0bf3bdea46 100644
--- a/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sql
+++ b/posthog/clickhouse/hcl/roles/auxiliary/shared/sql/email_status_mv.sql
@@ -10,3 +10,4 @@ SELECT
_offset,
_partition
FROM posthog.kafka_email_status
+WHERE status NOT IN ('opened', 'clicked')
diff --git a/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl b/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl
index 0be12c26ae72..4ba342156339 100644
--- a/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl
+++ b/posthog/clickhouse/hcl/roles/auxiliary/shared/tables.hcl
@@ -1156,6 +1156,29 @@ database "posthog" {
version_column = "version"
}
}
+ table "email_status_events_data" {
+ order_by = ["team_id", "created_at", "message_id", "status_at"]
+ partition_by = "toYYYYMMDD(created_at)"
+ ttl = "toDate(created_at) + toIntervalDay(31)"
+ settings = {
+ index_granularity = "1024"
+ ttl_only_drop_parts = "1"
+ }
+ extend = "_email_status_events_columns"
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "replicated_merge_tree" {
+ zoo_path = "/clickhouse/tables/noshard/posthog.email_status_events_data"
+ replica_name = "{replica}-{shard}"
+ }
+ }
table "kafka_email_status" {
extend = "_email_status_columns"
engine "kafka" {
@@ -1170,6 +1193,20 @@ database "posthog" {
thread_per_consumer = true
}
}
+ table "kafka_email_status_events" {
+ extend = "_email_status_events_columns"
+ engine "kafka" {
+ collection = "warpstream_cyclotron"
+ topic_list = "clickhouse_email_status"
+ group_name = "clickhouse_email_status_events"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 100000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ thread_per_consumer = true
+ }
+ }
materialized_view "email_status_mv" {
to_table = "posthog.email_status_data"
query = file("sql/email_status_mv.sql")
@@ -1205,4 +1242,48 @@ database "posthog" {
type = "UInt64"
}
}
+ materialized_view "email_status_events_mv" {
+ to_table = "posthog.email_status_events_data"
+ query = file("sql/email_status_events_mv.sql")
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ column "_timestamp" {
+ type = "Nullable(DateTime)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ }
}
diff --git a/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl b/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl
index fb8396c68902..a62ab670da6a 100644
--- a/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl
+++ b/posthog/clickhouse/hcl/roles/coshared/aux_data/tables.hcl
@@ -85,6 +85,39 @@ database "posthog" {
type = "String"
}
}
+ table "_email_status_events_columns" {
+ abstract = true
+ column "team_id" {
+ type = "Int64"
+ }
+ column "message_id" {
+ type = "UUID"
+ }
+ column "status" {
+ type = "LowCardinality(String)"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ column "created_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "status_at" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "reason" {
+ type = "String"
+ }
+ column "recipient" {
+ type = "String"
+ }
+ column "function_id" {
+ type = "String"
+ }
+ column "sending_domain" {
+ type = "LowCardinality(String)"
+ }
+ }
table "_experiment_metric_events_preaggregated_columns" {
abstract = true
column "team_id" {
@@ -1144,6 +1177,23 @@ database "posthog" {
remote_table = "email_status_data"
}
}
+ table "email_status_events" {
+ extend = "_email_status_events_columns"
+ column "_timestamp" {
+ type = "DateTime"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "email_status_events_data"
+ }
+ }
table "error_tracking_fingerprint_issue_state" {
column "team_id" {
type = "Int64"
diff --git a/posthog/clickhouse/hcl/sql/dev/auxiliary.sql b/posthog/clickhouse/hcl/sql/dev/auxiliary.sql
index 017a00fe11e4..e934b55818a7 100644
--- a/posthog/clickhouse/hcl/sql/dev/auxiliary.sql
+++ b/posthog/clickhouse/hcl/sql/dev/auxiliary.sql
@@ -27,6 +27,21 @@ CREATE TABLE posthog.email_status_data (
_offset UInt64,
_partition UInt64
) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
+CREATE TABLE posthog.email_status_events_data (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.email_status_events_data', '{replica}-{shard}') ORDER BY (team_id, created_at, message_id, status_at) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
CREATE TABLE posthog.events (
uuid UUID,
event String,
@@ -182,6 +197,18 @@ CREATE TABLE posthog.kafka_email_status (
status_at DateTime64(6, 'UTC'),
reason String
) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
+CREATE TABLE posthog.kafka_email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String)
+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status_events', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
CREATE TABLE posthog.kafka_hog_invocation_results (
team_id Int64,
function_kind LowCardinality(String),
@@ -1337,6 +1364,21 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (
lc_estimated_rows SimpleAggregateFunction(max, Int64),
lc_estimated_bytes SimpleAggregateFunction(max, Int64)
) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');
+CREATE MATERIALIZED VIEW posthog.email_status_events_mv TO posthog.email_status_events_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, recipient String, function_id String, sending_domain LowCardinality(String), _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ recipient,
+ function_id,
+ sending_domain,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status_events;
CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
message_id,
@@ -1348,7 +1390,8 @@ CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (t
_timestamp,
_offset,
_partition
-FROM posthog.kafka_email_status;
+FROM posthog.kafka_email_status
+WHERE status NOT IN ('opened', 'clicked');
CREATE MATERIALIZED VIEW posthog.hog_invocation_results_mv TO posthog.hog_invocation_results_data (team_id Int64, function_kind LowCardinality(String), function_id String, invocation_id String, parent_run_id String, status LowCardinality(String), attempts UInt8, is_retry UInt8, scheduled_at DateTime64(6, 'UTC'), first_scheduled_at DateTime64(6, 'UTC'), started_at Nullable(DateTime64(6, 'UTC')), finished_at Nullable(DateTime64(6, 'UTC')), duration_ms Nullable(UInt32), error_kind LowCardinality(String), error_message String, event_uuid String, distinct_id String, person_id String, invocation_globals String, version UInt64, is_deleted UInt8, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
function_kind,
@@ -1682,6 +1725,21 @@ CREATE TABLE posthog.email_status (
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
+CREATE TABLE posthog.email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/dev/data.sql b/posthog/clickhouse/hcl/sql/dev/data.sql
index 0492bb145cf6..be9586c03c5f 100644
--- a/posthog/clickhouse/hcl/sql/dev/data.sql
+++ b/posthog/clickhouse/hcl/sql/dev/data.sql
@@ -288,6 +288,21 @@ CREATE TABLE posthog.email_status (
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
+CREATE TABLE posthog.email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql b/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql
index 0abc88af38d9..2797a1dad6a9 100644
--- a/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql
+++ b/posthog/clickhouse/hcl/sql/local-multi/auxiliary.sql
@@ -27,6 +27,21 @@ CREATE TABLE posthog.email_status_data (
_offset UInt64,
_partition UInt64
) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
+CREATE TABLE posthog.email_status_events_data (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.email_status_events_data', '{replica}-{shard}') ORDER BY (team_id, created_at, message_id, status_at) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
CREATE TABLE posthog.events (
uuid UUID,
event String,
@@ -182,6 +197,18 @@ CREATE TABLE posthog.kafka_email_status (
status_at DateTime64(6, 'UTC'),
reason String
) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
+CREATE TABLE posthog.kafka_email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String)
+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status_events', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
CREATE TABLE posthog.kafka_error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
@@ -1341,6 +1368,21 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (
lc_estimated_rows SimpleAggregateFunction(max, Int64),
lc_estimated_bytes SimpleAggregateFunction(max, Int64)
) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');
+CREATE MATERIALIZED VIEW posthog.email_status_events_mv TO posthog.email_status_events_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, recipient String, function_id String, sending_domain LowCardinality(String), _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ recipient,
+ function_id,
+ sending_domain,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status_events;
CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
message_id,
@@ -1352,7 +1394,8 @@ CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (t
_timestamp,
_offset,
_partition
-FROM posthog.kafka_email_status;
+FROM posthog.kafka_email_status
+WHERE status NOT IN ('opened', 'clicked');
CREATE MATERIALIZED VIEW posthog.error_tracking_fingerprint_issue_state_mv TO posthog.writable_error_tracking_fingerprint_issue_state (team_id Int64, fingerprint String, issue_id UUID, issue_name Nullable(String), issue_description Nullable(String), issue_status String, issue_severity Nullable(String), assigned_user_id Nullable(Int64), assigned_role_id Nullable(UUID), first_seen DateTime64(3, 'UTC'), is_deleted Int8, version Int64, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
fingerprint,
@@ -1703,6 +1746,21 @@ CREATE TABLE posthog.email_status (
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
+CREATE TABLE posthog.email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/local-multi/data.sql b/posthog/clickhouse/hcl/sql/local-multi/data.sql
index 862db991efeb..48c637358af5 100644
--- a/posthog/clickhouse/hcl/sql/local-multi/data.sql
+++ b/posthog/clickhouse/hcl/sql/local-multi/data.sql
@@ -245,6 +245,21 @@ CREATE TABLE posthog.email_status (
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
+CREATE TABLE posthog.email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sql
index 74203b0b5181..1d0014a9c545 100644
--- a/posthog/clickhouse/hcl/sql/local-single/all.sql
+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql
@@ -142,6 +142,21 @@ CREATE TABLE posthog.email_status_data (
_offset UInt64,
_partition UInt64
) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
+CREATE TABLE posthog.email_status_events_data (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.email_status_events_data', '{replica}-{shard}') ORDER BY (team_id, created_at, message_id, status_at) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
CREATE TABLE posthog.error_tracking_issue_fingerprint_overrides (
team_id Int64,
fingerprint String,
@@ -369,6 +384,18 @@ CREATE TABLE posthog.kafka_email_status (
status_at DateTime64(6, 'UTC'),
reason String
) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
+CREATE TABLE posthog.kafka_email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String)
+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status_events', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
CREATE TABLE posthog.kafka_error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
@@ -4557,6 +4584,21 @@ CREATE MATERIALIZED VIEW posthog.duplicate_events_mv TO posthog.writable_duplica
_offset,
_partition
FROM posthog.kafka_duplicate_events;
+CREATE MATERIALIZED VIEW posthog.email_status_events_mv TO posthog.email_status_events_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, recipient String, function_id String, sending_domain LowCardinality(String), _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ recipient,
+ function_id,
+ sending_domain,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status_events;
CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
message_id,
@@ -4568,7 +4610,8 @@ CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (t
_timestamp,
_offset,
_partition
-FROM posthog.kafka_email_status;
+FROM posthog.kafka_email_status
+WHERE status NOT IN ('opened', 'clicked');
CREATE MATERIALIZED VIEW posthog.error_tracking_fingerprint_issue_state_mv TO posthog.writable_error_tracking_fingerprint_issue_state (team_id Int64, fingerprint String, issue_id UUID, issue_name Nullable(String), issue_description Nullable(String), issue_status String, issue_severity Nullable(String), assigned_user_id Nullable(Int64), assigned_role_id Nullable(UUID), first_seen DateTime64(3, 'UTC'), is_deleted Int8, version Int64, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
fingerprint,
@@ -6611,6 +6654,21 @@ CREATE TABLE posthog.email_status (
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
+CREATE TABLE posthog.email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql b/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql
index 6bb9f0de70e1..f08cffdc21df 100644
--- a/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql
+++ b/posthog/clickhouse/hcl/sql/prod-eu/auxiliary.sql
@@ -27,6 +27,21 @@ CREATE TABLE posthog.email_status_data (
_offset UInt64,
_partition UInt64
) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
+CREATE TABLE posthog.email_status_events_data (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.email_status_events_data', '{replica}-{shard}') ORDER BY (team_id, created_at, message_id, status_at) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
CREATE TABLE posthog.events (
uuid UUID,
event String,
@@ -182,6 +197,18 @@ CREATE TABLE posthog.kafka_email_status (
status_at DateTime64(6, 'UTC'),
reason String
) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
+CREATE TABLE posthog.kafka_email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String)
+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status_events', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
CREATE TABLE posthog.kafka_hog_invocation_results (
team_id Int64,
function_kind LowCardinality(String),
@@ -1337,6 +1364,21 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (
lc_estimated_rows SimpleAggregateFunction(max, Int64),
lc_estimated_bytes SimpleAggregateFunction(max, Int64)
) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');
+CREATE MATERIALIZED VIEW posthog.email_status_events_mv TO posthog.email_status_events_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, recipient String, function_id String, sending_domain LowCardinality(String), _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ recipient,
+ function_id,
+ sending_domain,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status_events;
CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
message_id,
@@ -1348,7 +1390,8 @@ CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (t
_timestamp,
_offset,
_partition
-FROM posthog.kafka_email_status;
+FROM posthog.kafka_email_status
+WHERE status NOT IN ('opened', 'clicked');
CREATE MATERIALIZED VIEW posthog.hog_invocation_results_mv TO posthog.hog_invocation_results_data (team_id Int64, function_kind LowCardinality(String), function_id String, invocation_id String, parent_run_id String, status LowCardinality(String), attempts UInt8, is_retry UInt8, scheduled_at DateTime64(6, 'UTC'), first_scheduled_at DateTime64(6, 'UTC'), started_at Nullable(DateTime64(6, 'UTC')), finished_at Nullable(DateTime64(6, 'UTC')), duration_ms Nullable(UInt32), error_kind LowCardinality(String), error_message String, event_uuid String, distinct_id String, person_id String, invocation_globals String, version UInt64, is_deleted UInt8, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
function_kind,
@@ -1682,6 +1725,21 @@ CREATE TABLE posthog.email_status (
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
+CREATE TABLE posthog.email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/prod-eu/data.sql b/posthog/clickhouse/hcl/sql/prod-eu/data.sql
index 6a46b3f226ec..5d22950423f4 100644
--- a/posthog/clickhouse/hcl/sql/prod-eu/data.sql
+++ b/posthog/clickhouse/hcl/sql/prod-eu/data.sql
@@ -301,6 +301,21 @@ CREATE TABLE posthog.email_status (
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
+CREATE TABLE posthog.email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql b/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql
index 515531b2c911..16ac5cdd77da 100644
--- a/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql
+++ b/posthog/clickhouse/hcl/sql/prod-us/auxiliary.sql
@@ -27,6 +27,21 @@ CREATE TABLE posthog.email_status_data (
_offset UInt64,
_partition UInt64
) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/posthog.email_status_data', '{replica}-{shard}', version) ORDER BY (team_id, message_id) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
+CREATE TABLE posthog.email_status_events_data (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/noshard/posthog.email_status_events_data', '{replica}-{shard}') ORDER BY (team_id, created_at, message_id, status_at) PARTITION BY toYYYYMMDD(created_at) TTL toDate(created_at) + toIntervalDay(31) SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1;
CREATE TABLE posthog.events (
uuid UUID,
event String,
@@ -182,6 +197,18 @@ CREATE TABLE posthog.kafka_email_status (
status_at DateTime64(6, 'UTC'),
reason String
) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
+CREATE TABLE posthog.kafka_email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String)
+) ENGINE = Kafka(warpstream_cyclotron) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_email_status_events', kafka_max_block_size = 100000, kafka_num_consumers = 1, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_email_status';
CREATE TABLE posthog.kafka_hog_invocation_results (
team_id Int64,
function_kind LowCardinality(String),
@@ -1337,6 +1364,21 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (
lc_estimated_rows SimpleAggregateFunction(max, Int64),
lc_estimated_bytes SimpleAggregateFunction(max, Int64)
) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');
+CREATE MATERIALIZED VIEW posthog.email_status_events_mv TO posthog.email_status_events_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, recipient String, function_id String, sending_domain LowCardinality(String), _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ recipient,
+ function_id,
+ sending_domain,
+ _timestamp,
+ _offset,
+ _partition
+FROM posthog.kafka_email_status_events;
CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (team_id Int64, message_id UUID, status LowCardinality(String), version UInt64, created_at DateTime64(6, 'UTC'), status_at DateTime64(6, 'UTC'), reason String, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
message_id,
@@ -1348,7 +1390,8 @@ CREATE MATERIALIZED VIEW posthog.email_status_mv TO posthog.email_status_data (t
_timestamp,
_offset,
_partition
-FROM posthog.kafka_email_status;
+FROM posthog.kafka_email_status
+WHERE status NOT IN ('opened', 'clicked');
CREATE MATERIALIZED VIEW posthog.hog_invocation_results_mv TO posthog.hog_invocation_results_data (team_id Int64, function_kind LowCardinality(String), function_id String, invocation_id String, parent_run_id String, status LowCardinality(String), attempts UInt8, is_retry UInt8, scheduled_at DateTime64(6, 'UTC'), first_scheduled_at DateTime64(6, 'UTC'), started_at Nullable(DateTime64(6, 'UTC')), finished_at Nullable(DateTime64(6, 'UTC')), duration_ms Nullable(UInt32), error_kind LowCardinality(String), error_message String, event_uuid String, distinct_id String, person_id String, invocation_globals String, version UInt64, is_deleted UInt8, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
team_id,
function_kind,
@@ -1682,6 +1725,21 @@ CREATE TABLE posthog.email_status (
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
+CREATE TABLE posthog.email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/hcl/sql/prod-us/data.sql b/posthog/clickhouse/hcl/sql/prod-us/data.sql
index 2f9c75229c9c..93603385e76b 100644
--- a/posthog/clickhouse/hcl/sql/prod-us/data.sql
+++ b/posthog/clickhouse/hcl/sql/prod-us/data.sql
@@ -311,6 +311,21 @@ CREATE TABLE posthog.email_status (
_offset UInt64,
_partition UInt64
) ENGINE = Distributed('aux', 'posthog', 'email_status_data');
+CREATE TABLE posthog.email_status_events (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String),
+ _timestamp DateTime,
+ _offset UInt64,
+ _partition UInt64
+) ENGINE = Distributed('aux', 'posthog', 'email_status_events_data');
CREATE TABLE posthog.error_tracking_fingerprint_issue_state (
team_id Int64,
fingerprint String,
diff --git a/posthog/clickhouse/kafka_engine.py b/posthog/clickhouse/kafka_engine.py
index 71175da3498c..1d4c1ed7fd15 100644
--- a/posthog/clickhouse/kafka_engine.py
+++ b/posthog/clickhouse/kafka_engine.py
@@ -22,6 +22,7 @@ CONSUMER_GROUP_LOG_ENTRIES = "clickhouse_log_entries_v3" if _US else "clickhouse
CONSUMER_GROUP_HOG_INVOCATION_RESULTS = "clickhouse_hog_invocation_results"
CONSUMER_GROUP_MESSAGE_ASSETS = "clickhouse_message_assets"
CONSUMER_GROUP_EMAIL_STATUS = "clickhouse_email_status"
+CONSUMER_GROUP_EMAIL_STATUS_EVENTS = "clickhouse_email_status_events"
CONSUMER_GROUP_DOCUMENT_EMBEDDINGS = "clickhouse_document_embeddings2" if _US else "clickhouse_document_embeddings"
CONSUMER_GROUP_HEATMAPS = "clickhouse_heatmaps" if _US else "group1"
CONSUMER_GROUP_PRECALCULATED_EVENTS = "clickhouse_precalculated_events2" if _US else "clickhouse_prefiltered_events"
diff --git a/posthog/clickhouse/migrations/0368_email_status_events.py b/posthog/clickhouse/migrations/0368_email_status_events.py
new file mode 100644
index 000000000000..52e5e0a967ac
--- /dev/null
+++ b/posthog/clickhouse/migrations/0368_email_status_events.py
@@ -0,0 +1,25 @@
+from posthog.clickhouse.client.connection import NodeRole
+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions
+
+from products.messaging.backend.facade.clickhouse import (
+ ALTER_EMAIL_STATUS_MV_QUERY_SQL,
+ DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL,
+ EMAIL_STATUS_EVENTS_DATA_TABLE_SQL,
+ EMAIL_STATUS_EVENTS_MV_SQL,
+ KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL,
+)
+
+# Only adds objects: the new table reads the topic through its own Kafka table and consumer group, and the
+# existing view keeps consuming while its query changes in place, so the email_status feed never pauses.
+operations = [
+ run_sql_with_exceptions(EMAIL_STATUS_EVENTS_DATA_TABLE_SQL(), node_roles=[NodeRole.AUX]),
+ run_sql_with_exceptions(KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL(), node_roles=[NodeRole.AUX]),
+ run_sql_with_exceptions(EMAIL_STATUS_EVENTS_MV_SQL(), node_roles=[NodeRole.AUX]),
+ run_sql_with_exceptions(
+ ALTER_EMAIL_STATUS_MV_QUERY_SQL(),
+ node_roles=[NodeRole.AUX],
+ sharded=False,
+ is_alter_on_replicated_table=False,
+ ),
+ run_sql_with_exceptions(DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL(), node_roles=[NodeRole.AUX, NodeRole.DATA]),
+]
diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txt
index 1f048aa29546..e0822ce38c94 100644
--- a/posthog/clickhouse/migrations/max_migration.txt
+++ b/posthog/clickhouse/migrations/max_migration.txt
@@ -1 +1 @@
-0367_metrics4_view_drop_metrics2
+0368_email_status_events
diff --git a/posthog/clickhouse/schema.py b/posthog/clickhouse/schema.py
index be8c8fbf2eb8..bce58280a253 100644
--- a/posthog/clickhouse/schema.py
+++ b/posthog/clickhouse/schema.py
@@ -410,9 +410,13 @@ from products.error_tracking.backend.sql import (
)
from products.event_definitions.backend.models.property_definition import PROPERTY_DEFINITIONS_TABLE_SQL
from products.messaging.backend.facade.clickhouse import (
+ DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL,
DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,
EMAIL_STATUS_DATA_TABLE_SQL,
+ EMAIL_STATUS_EVENTS_DATA_TABLE_SQL,
+ EMAIL_STATUS_EVENTS_MV_SQL,
EMAIL_STATUS_MV_SQL,
+ KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL,
KAFKA_EMAIL_STATUS_TABLE_SQL,
)
from products.workflows.backend.facade.clickhouse import (
@@ -453,6 +457,7 @@ CREATE_MERGETREE_TABLE_QUERIES = (
HOG_INVOCATION_RESULTS_DATA_TABLE_SQL,
MESSAGE_ASSETS_DATA_TABLE_SQL,
EMAIL_STATUS_DATA_TABLE_SQL,
+ EMAIL_STATUS_EVENTS_DATA_TABLE_SQL,
PERFORMANCE_EVENTS_TABLE_SQL,
SESSION_REPLAY_EVENTS_TABLE_SQL,
WRITABLE_SESSION_REPLAY_EVENTS_TABLE_SQL,
@@ -548,6 +553,7 @@ CREATE_DISTRIBUTED_TABLE_QUERIES = (
DISTRIBUTED_HOG_INVOCATION_RESULTS_TABLE_SQL,
DISTRIBUTED_MESSAGE_ASSETS_TABLE_SQL,
DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,
+ DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL,
WRITABLE_PERFORMANCE_EVENTS_TABLE_SQL,
DISTRIBUTED_PERFORMANCE_EVENTS_TABLE_SQL,
DISTRIBUTED_SESSION_REPLAY_EVENTS_TABLE_SQL,
@@ -655,6 +661,7 @@ CREATE_KAFKA_TABLE_QUERIES = (
KAFKA_HOG_INVOCATION_RESULTS_TABLE_SQL,
KAFKA_MESSAGE_ASSETS_TABLE_SQL,
KAFKA_EMAIL_STATUS_TABLE_SQL,
+ KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL,
KAFKA_PERFORMANCE_EVENTS_TABLE_SQL,
KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL,
KAFKA_SESSION_REPLAY_FEATURES_TABLE_SQL,
@@ -694,6 +701,7 @@ CREATE_MV_TABLE_QUERIES = (
HOG_INVOCATION_RESULTS_MV_SQL,
MESSAGE_ASSETS_MV_SQL,
EMAIL_STATUS_MV_SQL,
+ EMAIL_STATUS_EVENTS_MV_SQL,
PERFORMANCE_EVENTS_TABLE_MV_SQL,
SESSION_REPLAY_EVENTS_TABLE_MV_SQL,
SESSION_REPLAY_FEATURES_TABLE_MV_SQL,
diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambr
index 0276a0d0bd3e..fec26e9b4e92 100644
--- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr
+++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr
@@ -176,6 +176,31 @@
'''
# ---
+# name: test_create_kafka_table_with_different_kafka_host[kafka_email_status_events]
+ '''
+
+ CREATE TABLE IF NOT EXISTS kafka_email_status_events
+ (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String)
+ )
+ ENGINE = Kafka(warpstream_cyclotron, kafka_topic_list = 'clickhouse_email_status_test', kafka_group_name = 'clickhouse_email_status_events', kafka_format = 'JSONEachRow')
+ SETTINGS kafka_skip_broken_messages = 100,
+ kafka_num_consumers = 1,
+ kafka_thread_per_consumer = 1,
+ kafka_poll_timeout_ms = 10000,
+ kafka_max_block_size = 100000
+
+ '''
+# ---
# name: test_create_kafka_table_with_different_kafka_host[kafka_error_tracking_fingerprint_issue_state]
'''
@@ -1613,12 +1638,90 @@
'''
# ---
+# name: test_create_table_query[email_status_events]
+ '''
+
+ CREATE TABLE IF NOT EXISTS email_status_events
+ (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String)
+
+ , _timestamp DateTime
+ , _offset UInt64
+ , _partition UInt64
+
+ )
+ ENGINE = Distributed('aux', 'posthog_test', 'email_status_events_data')
+
+ '''
+# ---
+# name: test_create_table_query[email_status_events_data]
+ '''
+
+ CREATE TABLE IF NOT EXISTS email_status_events_data
+ (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String)
+
+ , _timestamp DateTime
+ , _offset UInt64
+ , _partition UInt64
+
+ )
+ ENGINE = ReplicatedMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.email_status_events_data', '{replica}-{shard}')
+ PARTITION BY toYYYYMMDD(created_at)
+ ORDER BY (team_id, created_at, message_id, status_at)
+
+ SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1
+
+ '''
+# ---
+# name: test_create_table_query[email_status_events_mv]
+ '''
+
+ CREATE MATERIALIZED VIEW IF NOT EXISTS email_status_events_mv
+ TO email_status_events_data
+ AS SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ recipient,
+ function_id,
+ sending_domain,
+ _timestamp,
+ _offset,
+ _partition
+ FROM kafka_email_status_events
+
+ '''
+# ---
# name: test_create_table_query[email_status_mv]
'''
CREATE MATERIALIZED VIEW IF NOT EXISTS email_status_mv
TO email_status_data
- AS SELECT
+ AS
+ SELECT
team_id,
message_id,
status,
@@ -1630,6 +1733,7 @@
_offset,
_partition
FROM kafka_email_status
+ WHERE status NOT IN ('opened', 'clicked')
'''
# ---
@@ -2706,6 +2810,31 @@
'''
# ---
+# name: test_create_table_query[kafka_email_status_events]
+ '''
+
+ CREATE TABLE IF NOT EXISTS kafka_email_status_events
+ (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String)
+ )
+ ENGINE = Kafka(warpstream_cyclotron, kafka_topic_list = 'clickhouse_email_status_test', kafka_group_name = 'clickhouse_email_status_events', kafka_format = 'JSONEachRow')
+ SETTINGS kafka_skip_broken_messages = 100,
+ kafka_num_consumers = 1,
+ kafka_thread_per_consumer = 1,
+ kafka_poll_timeout_ms = 10000,
+ kafka_max_block_size = 100000
+
+ '''
+# ---
# name: test_create_table_query[kafka_error_tracking_fingerprint_issue_state]
'''
@@ -12018,6 +12147,35 @@
'''
# ---
+# name: test_create_table_query_replicated_and_storage[email_status_events_data]
+ '''
+
+ CREATE TABLE IF NOT EXISTS email_status_events_data
+ (
+ team_id Int64,
+ message_id UUID,
+ status LowCardinality(String),
+ version UInt64,
+ created_at DateTime64(6, 'UTC'),
+ status_at DateTime64(6, 'UTC'),
+ reason String,
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String)
+
+ , _timestamp DateTime
+ , _offset UInt64
+ , _partition UInt64
+
+ )
+ ENGINE = ReplicatedMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.email_status_events_data', '{replica}-{shard}')
+ PARTITION BY toYYYYMMDD(created_at)
+ ORDER BY (team_id, created_at, message_id, status_at)
+
+ SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1
+
+ '''
+# ---
# name: test_create_table_query_replicated_and_storage[error_tracking_issue_fingerprint_overrides]
'''
diff --git a/products/messaging/backend/facade/clickhouse.py b/products/messaging/backend/facade/clickhouse.py
index ade6d5e1aad2..7791f486ae1b 100644
--- a/products/messaging/backend/facade/clickhouse.py
+++ b/products/messaging/backend/facade/clickhouse.py
@@ -5,8 +5,13 @@ the migration that created them applies the same definitions to a cluster.
"""
from products.messaging.backend.models.email_status_sql import (
+ ALTER_EMAIL_STATUS_MV_QUERY_SQL as ALTER_EMAIL_STATUS_MV_QUERY_SQL,
+ DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL as DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL,
DISTRIBUTED_EMAIL_STATUS_TABLE_SQL as DISTRIBUTED_EMAIL_STATUS_TABLE_SQL,
EMAIL_STATUS_DATA_TABLE_SQL as EMAIL_STATUS_DATA_TABLE_SQL,
+ EMAIL_STATUS_EVENTS_DATA_TABLE_SQL as EMAIL_STATUS_EVENTS_DATA_TABLE_SQL,
+ EMAIL_STATUS_EVENTS_MV_SQL as EMAIL_STATUS_EVENTS_MV_SQL,
EMAIL_STATUS_MV_SQL as EMAIL_STATUS_MV_SQL,
+ KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL as KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL,
KAFKA_EMAIL_STATUS_TABLE_SQL as KAFKA_EMAIL_STATUS_TABLE_SQL,
)
diff --git a/products/messaging/backend/models/email_status_sql.py b/products/messaging/backend/models/email_status_sql.py
index d3abc64a3ba3..9f5d020ec86e 100644
--- a/products/messaging/backend/models/email_status_sql.py
+++ b/products/messaging/backend/models/email_status_sql.py
@@ -2,11 +2,12 @@ from django.conf import settings
from posthog.clickhouse.kafka_engine import (
CONSUMER_GROUP_EMAIL_STATUS,
+ CONSUMER_GROUP_EMAIL_STATUS_EVENTS,
KAFKA_COLUMNS_WITH_PARTITION,
kafka_engine,
ttl_period,
)
-from posthog.clickhouse.table_engines import Distributed, ReplacingMergeTree, ReplicationScheme
+from posthog.clickhouse.table_engines import Distributed, MergeTreeEngine, ReplacingMergeTree, ReplicationScheme
from posthog.kafka_client.topics import KAFKA_EMAIL_STATUS
EMAIL_STATUS_RETENTION_DAYS = 30
@@ -22,6 +23,17 @@ EMAIL_STATUS_DATA_TABLE = f"{EMAIL_STATUS_TABLE}_data"
KAFKA_EMAIL_STATUS_TABLE = f"kafka_{EMAIL_STATUS_TABLE}"
EMAIL_STATUS_MV_TABLE = f"{EMAIL_STATUS_TABLE}_mv"
+# Keeps every status row of a message, where `email_status` keeps only the latest. The email log reads
+# the timeline, the opens and clicks, and the recipient and sending domain from it.
+EMAIL_STATUS_EVENTS_TABLE = "email_status_events"
+EMAIL_STATUS_EVENTS_DATA_TABLE = f"{EMAIL_STATUS_EVENTS_TABLE}_data"
+EMAIL_STATUS_EVENTS_MV_TABLE = f"{EMAIL_STATUS_EVENTS_TABLE}_mv"
+# Its own Kafka table and consumer group on the same topic, so adding it leaves the `email_status` feed untouched.
+KAFKA_EMAIL_STATUS_EVENTS_TABLE = f"kafka_{EMAIL_STATUS_EVENTS_TABLE}"
+
+# Rows that record what the recipient did with the email. They never change its delivery status.
+EMAIL_ENGAGEMENT_STATUSES = ("opened", "clicked")
+
EMAIL_STATUS_KAFKA_COLUMNS = """
team_id Int64,
@@ -33,6 +45,15 @@ EMAIL_STATUS_KAFKA_COLUMNS = """
reason String
""".strip()
+# The queued row carries the recipient and the workflow, and the sent row the sending domain. Other rows
+# leave them empty.
+EMAIL_STATUS_EVENTS_KAFKA_COLUMNS = f"""
+ {EMAIL_STATUS_KAFKA_COLUMNS},
+ recipient String,
+ function_id String,
+ sending_domain LowCardinality(String)
+""".strip()
+
# Every row of a message carries the same `created_at`, so the rows share a partition and the
# ReplacingMergeTree can collapse them.
@@ -65,18 +86,16 @@ ENGINE = {Distributed(data_table=EMAIL_STATUS_DATA_TABLE, cluster=settings.CLICK
)
-# The CDP producer writes to the cyclotron WarpStream cluster, so the Kafka table reads from its
-# named collection.
-KAFKA_EMAIL_STATUS_TABLE_SQL = lambda: (
- f"""
-CREATE TABLE IF NOT EXISTS {KAFKA_EMAIL_STATUS_TABLE}
+def _kafka_email_status_table_sql(table: str, columns: str, group: str) -> str:
+ return f"""
+CREATE TABLE IF NOT EXISTS {table}
(
- {EMAIL_STATUS_KAFKA_COLUMNS}
+ {columns}
)
ENGINE = {
kafka_engine(
topic=KAFKA_EMAIL_STATUS,
- group=CONSUMER_GROUP_EMAIL_STATUS,
+ group=group,
named_collection=settings.CLICKHOUSE_KAFKA_WARPSTREAM_CYCLOTRON_NAMED_COLLECTION,
)
}
@@ -86,13 +105,83 @@ SETTINGS kafka_skip_broken_messages = 100,
kafka_poll_timeout_ms = 10000,
kafka_max_block_size = 100000
"""
+
+
+# The CDP producer writes to the cyclotron WarpStream cluster, so the Kafka tables read from its
+# named collection.
+KAFKA_EMAIL_STATUS_TABLE_SQL = lambda: _kafka_email_status_table_sql(
+ KAFKA_EMAIL_STATUS_TABLE, EMAIL_STATUS_KAFKA_COLUMNS, CONSUMER_GROUP_EMAIL_STATUS
)
+KAFKA_EMAIL_STATUS_EVENTS_TABLE_SQL = lambda: _kafka_email_status_table_sql(
+ KAFKA_EMAIL_STATUS_EVENTS_TABLE, EMAIL_STATUS_EVENTS_KAFKA_COLUMNS, CONSUMER_GROUP_EMAIL_STATUS_EVENTS
+)
+
+
+# Opens and clicks rank above delivered, so in this ReplacingMergeTree they would replace the delivery status.
+EMAIL_STATUS_MV_QUERY = lambda: (
+ f"""
+SELECT
+ team_id,
+ message_id,
+ status,
+ version,
+ created_at,
+ status_at,
+ reason,
+ _timestamp,
+ _offset,
+ _partition
+FROM {KAFKA_EMAIL_STATUS_TABLE}
+WHERE status NOT IN {EMAIL_ENGAGEMENT_STATUSES}
+"""
+)
EMAIL_STATUS_MV_SQL = lambda: (
f"""
CREATE MATERIALIZED VIEW IF NOT EXISTS {EMAIL_STATUS_MV_TABLE}
TO {EMAIL_STATUS_DATA_TABLE}
+AS {EMAIL_STATUS_MV_QUERY()}"""
+)
+
+# Changes the query of an existing view in place, so it keeps consuming with no gap.
+ALTER_EMAIL_STATUS_MV_QUERY_SQL = lambda: f"ALTER TABLE {EMAIL_STATUS_MV_TABLE} MODIFY QUERY {EMAIL_STATUS_MV_QUERY()}"
+
+
+# A plain MergeTree, so no row replaces another. `created_at` leads the key because ClickHouse doesn't sort a
+# UUID by time. All rows of a message share a `created_at`, so they stay together.
+EMAIL_STATUS_EVENTS_DATA_TABLE_SQL = lambda: (
+ f"""
+CREATE TABLE IF NOT EXISTS {EMAIL_STATUS_EVENTS_DATA_TABLE}
+(
+ {EMAIL_STATUS_EVENTS_KAFKA_COLUMNS}
+ {KAFKA_COLUMNS_WITH_PARTITION}
+)
+ENGINE = {MergeTreeEngine(EMAIL_STATUS_EVENTS_DATA_TABLE, replication_scheme=ReplicationScheme.REPLICATED)}
+PARTITION BY toYYYYMMDD(created_at)
+ORDER BY (team_id, created_at, message_id, status_at)
+{ttl_period("created_at", EMAIL_STATUS_TTL_DAYS, unit="DAY")}
+SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1
+"""
+)
+
+
+DISTRIBUTED_EMAIL_STATUS_EVENTS_TABLE_SQL = lambda: (
+ f"""
+CREATE TABLE IF NOT EXISTS {EMAIL_STATUS_EVENTS_TABLE}
+(
+ {EMAIL_STATUS_EVENTS_KAFKA_COLUMNS}
+ {KAFKA_COLUMNS_WITH_PARTITION}
+)
+ENGINE = {Distributed(data_table=EMAIL_STATUS_EVENTS_DATA_TABLE, cluster=settings.CLICKHOUSE_AUX_CLUSTER)}
+"""
+)
+
+
+EMAIL_STATUS_EVENTS_MV_SQL = lambda: (
+ f"""
+CREATE MATERIALIZED VIEW IF NOT EXISTS {EMAIL_STATUS_EVENTS_MV_TABLE}
+TO {EMAIL_STATUS_EVENTS_DATA_TABLE}
AS SELECT
team_id,
message_id,
@@ -101,9 +190,12 @@ AS SELECT
created_at,
status_at,
reason,
+ recipient,
+ function_id,
+ sending_domain,
_timestamp,
_offset,
_partition
-FROM {KAFKA_EMAIL_STATUS_TABLE}
+FROM {KAFKA_EMAIL_STATUS_EVENTS_TABLE}
"""
)