Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(customer-analytics): ingest account membership from live events (#128660)

Arthur Moreira de Deuscbd1cef9ff2c
diff --git a/posthog/clickhouse/hcl/golden/local-multi/ingestion_events.hcl b/posthog/clickhouse/hcl/golden/local-multi/ingestion_events.hcl
index 33d7a50d4b80..4a56d0bbfaee 100644
--- a/posthog/clickhouse/hcl/golden/local-multi/ingestion_events.hcl
+++ b/posthog/clickhouse/hcl/golden/local-multi/ingestion_events.hcl
@@ -8,6 +8,27 @@ node "events" {
}
database "posthog" {
+ table "distributed_person_group_membership_config" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "group_type_index" {
+ type = "UInt8"
+ }
+ column "enabled" {
+ type = "UInt8"
+ }
+ column "version" {
+ type = "UInt64"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "person_group_membership_config"
+ sharding_key = "sipHash64(team_id)"
+ }
+ }
+
table "kafka_events_json_native_json" {
column "uuid" {
type = "UUID"
@@ -195,6 +216,37 @@ database "posthog" {
}
}
+ table "kafka_person_group_membership" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "distinct_id" {
+ type = "String"
+ }
+ column "properties" {
+ type = "String"
+ }
+ column "timestamp" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "person_mode" {
+ type = "Enum8('full'=0, 'propertyless'=1, 'force_upgrade'=2)"
+ }
+ engine "kafka" {
+ collection = "warpstream_ingestion"
+ topic_list = "clickhouse_events_json"
+ group_name = "clickhouse_person_group_membership"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 10000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ poll_max_batch_size = 10000
+ flush_interval_ms = 7500
+ thread_per_consumer = true
+ }
+ }
+
table "query_log_archive" {
column "initial_query_start_time" {
type = "DateTime64(6, 'UTC')"
@@ -822,6 +874,33 @@ database "posthog" {
}
}
+ table "writable_person_group_membership" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "group_type_index" {
+ type = "UInt8"
+ }
+ column "group_key" {
+ type = "String"
+ }
+ column "distinct_id" {
+ type = "String"
+ }
+ column "first_seen" {
+ type = "SimpleAggregateFunction(min, DateTime64(6, 'UTC'))"
+ }
+ column "last_seen" {
+ type = "SimpleAggregateFunction(max, DateTime64(6, 'UTC'))"
+ }
+ engine "distributed" {
+ cluster_name = "aux"
+ remote_database = "posthog"
+ remote_table = "sharded_person_group_membership"
+ sharding_key = "sipHash64(team_id, group_type_index, group_key)"
+ }
+ }
+
materialized_view "events_json_table_mv" {
to_table = "posthog.writable_events_json"
query = <<SQL
@@ -1097,6 +1176,84 @@ SQL
type = "LowCardinality(String)"
}
}
+
+ materialized_view "person_group_membership_mv" {
+ to_table = "posthog.writable_person_group_membership"
+ query = <<SQL
+SELECT
+ team_id,
+ group_type_index,
+ replaceRegexpAll(
+ JSONExtractRaw(properties, concat('$group_', toString(group_type_index))),
+ '^"|"$',
+ ''
+ ) AS group_key,
+ distinct_id,
+ min(timestamp) AS first_seen,
+ max(timestamp) AS last_seen
+FROM
+ (
+ SELECT
+ team_id,
+ dictGet('posthog.person_group_membership_config_dict', 'group_type_index', tuple(team_id)) AS group_type_index,
+ distinct_id,
+ properties,
+ timestamp
+ FROM posthog.kafka_person_group_membership
+ WHERE
+ dictHas('posthog.person_group_membership_config_dict', tuple(team_id))
+ AND
+ (person_mode != 'propertyless')
+ )
+WHERE notEmpty(group_key)
+GROUP BY
+ team_id, group_type_index, group_key, distinct_id
+SQL
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "group_type_index" {
+ type = "UInt8"
+ }
+ column "group_key" {
+ type = "String"
+ }
+ column "distinct_id" {
+ type = "String"
+ }
+ column "first_seen" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "last_seen" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ }
+
+ dictionary "person_group_membership_config_dict" {
+ primary_key = ["team_id"]
+ lifetime {
+ min = 60
+ max = 120
+ }
+ attribute "team_id" {
+ type = "Int64"
+ }
+ attribute "group_type_index" {
+ type = "UInt8"
+ default = "255"
+ }
+ attribute "enabled" {
+ type = "UInt8"
+ default = "0"
+ }
+ source "clickhouse" {
+ user = "default"
+ query = "SELECT team_id, config.1 AS group_type_index, config.2 AS enabled FROM (SELECT team_id, argMax(tuple(group_type_index, enabled), version) AS config FROM posthog.distributed_person_group_membership_config GROUP BY team_id) WHERE enabled = 1 AND group_type_index <= 4"
+ }
+ layout "complex_key_hashed" {
+ }
+ }
}
named_collection "msk_cluster" {
diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hcl
index a45ad09f78a4..84a59f0ec753 100644
--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl
+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl
@@ -4254,6 +4254,37 @@ database "posthog" {
}
}
+ table "kafka_person_group_membership" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "distinct_id" {
+ type = "String"
+ }
+ column "properties" {
+ type = "String"
+ }
+ column "timestamp" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "person_mode" {
+ type = "Enum8('full'=0, 'propertyless'=1, 'force_upgrade'=2)"
+ }
+ engine "kafka" {
+ collection = "warpstream_ingestion"
+ topic_list = "clickhouse_events_json"
+ group_name = "clickhouse_person_group_membership"
+ format = "JSONEachRow"
+ num_consumers = 1
+ max_block_size = 10000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ poll_max_batch_size = 10000
+ flush_interval_ms = 7500
+ thread_per_consumer = true
+ }
+ }
+
table "kafka_person_overrides" {
column "team_id" {
type = "Int32"
@@ -24098,6 +24129,59 @@ SQL
}
}
+ materialized_view "person_group_membership_mv" {
+ to_table = "posthog.writable_person_group_membership"
+ query = <<SQL
+SELECT
+ team_id,
+ group_type_index,
+ replaceRegexpAll(
+ JSONExtractRaw(properties, concat('$group_', toString(group_type_index))),
+ '^"|"$',
+ ''
+ ) AS group_key,
+ distinct_id,
+ min(timestamp) AS first_seen,
+ max(timestamp) AS last_seen
+FROM
+ (
+ SELECT
+ team_id,
+ dictGet('posthog.person_group_membership_config_dict', 'group_type_index', tuple(team_id)) AS group_type_index,
+ distinct_id,
+ properties,
+ timestamp
+ FROM posthog.kafka_person_group_membership
+ WHERE
+ dictHas('posthog.person_group_membership_config_dict', tuple(team_id))
+ AND
+ (person_mode != 'propertyless')
+ )
+WHERE notEmpty(group_key)
+GROUP BY
+ team_id, group_type_index, group_key, distinct_id
+SQL
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "group_type_index" {
+ type = "UInt8"
+ }
+ column "group_key" {
+ type = "String"
+ }
+ column "distinct_id" {
+ type = "String"
+ }
+ column "first_seen" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "last_seen" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ }
+
materialized_view "person_mv" {
to_table = "posthog.writable_person"
query = <<SQL
diff --git a/posthog/clickhouse/hcl/manifest.hcl b/posthog/clickhouse/hcl/manifest.hcl
index 581c9249a671..da26b32f39c5 100644
--- a/posthog/clickhouse/hcl/manifest.hcl
+++ b/posthog/clickhouse/hcl/manifest.hcl
@@ -130,11 +130,11 @@ role "data" {
# Beyond that they carry only the shared query_log_archive path. Role names are the
# hostClusterRole macros migrations target: events, small, medium.
role "events" {
- env "local-multi" { layers = ["roles/shared/qla.hcl", "roles/coshared/named_collections", "roles/coshared/logs_avro_ingest", "roles/coshared/events_json_write", "roles/ingestion_events/local"] }
+ env "local-multi" { layers = ["roles/shared/qla.hcl", "roles/coshared/named_collections", "roles/coshared/logs_avro_ingest", "roles/coshared/events_json_write", "roles/shared/person_group_membership.hcl", "roles/coshared/person_group_membership_config", "roles/coshared/person_group_membership_write", "roles/ingestion_events/local"] }
}
role "small" {
- env "local-multi" { layers = ["roles/shared/qla.hcl", "roles/shared/person_group_membership.hcl", "roles/coshared/person_group_membership_config", "roles/ingestion_small/shared", "roles/coshared/named_collections", "roles/coshared/aux_small", "roles/coshared/log_entries_write", "roles/coshared/log_entries_aux_write", "roles/coshared/session_replay_write", "roles/ingestion_small/local"] }
+ env "local-multi" { layers = ["roles/shared/qla.hcl", "roles/shared/person_group_membership.hcl", "roles/coshared/person_group_membership_config", "roles/coshared/person_group_membership_write", "roles/coshared/named_collections", "roles/coshared/aux_small", "roles/coshared/log_entries_write", "roles/coshared/log_entries_aux_write", "roles/coshared/session_replay_write", "roles/ingestion_small/local"] }
}
role "medium" {
@@ -146,7 +146,7 @@ role "medium" {
# and not MULTINODE_CLICKHOUSE. Composed as the deduped union of the local-multi stacks it
# hosts, so any name two of those roles declare fails this load instead of drifting.
role "all" {
- env "local-single" { layers = ["roles/shared", "roles/coshared/named_collections", "roles/coshared/apm_ingest", "roles/coshared/logs_avro_ingest", "roles/coshared/custom_metrics", "roles/ops/shared", "roles/ops/local", "roles/logs/base", "roles/logs/traces", "roles/logs/traces_kafka_metrics", "roles/logs/metrics", "roles/logs/local", "roles/apm/shared", "roles/apm/local", "roles/coshared/ai_events_data", "roles/coshared/person_shims", "roles/ai_events/shared", "roles/ai_events/local", "roles/coshared/aux_data", "roles/coshared/log_entries_aux", "roles/coshared/aux_small", "roles/auxiliary/shared", "roles/auxiliary/local", "roles/coshared/sessions_data", "roles/coshared/main_join_readers", "roles/coshared/tophog", "roles/coshared/events_recent", "roles/coshared/events_recent_write", "roles/coshared/batch_exports_data", "roles/coshared/ingestion_warnings_store", "roles/coshared/events_json_write", "roles/coshared/log_entries_write", "roles/coshared/log_entries_aux_write", "roles/coshared/session_replay_write", "roles/coshared/person_group_membership_config", "roles/ingestion_small/shared", "roles/data/shared", "roles/data/common", "roles/coshared/main_proxies", "roles/data/local", "roles/ingestion_events/local", "roles/ingestion_events/local-single", "roles/ingestion_small/local", "roles/ingestion_medium/local"] }
+ env "local-single" { layers = ["roles/shared", "roles/coshared/named_collections", "roles/coshared/apm_ingest", "roles/coshared/logs_avro_ingest", "roles/coshared/custom_metrics", "roles/ops/shared", "roles/ops/local", "roles/logs/base", "roles/logs/traces", "roles/logs/traces_kafka_metrics", "roles/logs/metrics", "roles/logs/local", "roles/apm/shared", "roles/apm/local", "roles/coshared/ai_events_data", "roles/coshared/person_shims", "roles/ai_events/shared", "roles/ai_events/local", "roles/coshared/aux_data", "roles/coshared/log_entries_aux", "roles/coshared/aux_small", "roles/auxiliary/shared", "roles/auxiliary/local", "roles/coshared/sessions_data", "roles/coshared/main_join_readers", "roles/coshared/tophog", "roles/coshared/events_recent", "roles/coshared/events_recent_write", "roles/coshared/batch_exports_data", "roles/coshared/ingestion_warnings_store", "roles/coshared/events_json_write", "roles/coshared/log_entries_write", "roles/coshared/log_entries_aux_write", "roles/coshared/session_replay_write", "roles/coshared/person_group_membership_config", "roles/coshared/person_group_membership_write", "roles/data/shared", "roles/data/common", "roles/coshared/main_proxies", "roles/data/local", "roles/ingestion_events/local", "roles/ingestion_events/local-single", "roles/ingestion_small/local", "roles/ingestion_medium/local"] }
}
# role "endpoints" {
diff --git a/posthog/clickhouse/hcl/roles/ingestion_small/shared/person_group_membership.hcl b/posthog/clickhouse/hcl/roles/coshared/person_group_membership_write/tables.hcl
similarity index 100%
rename from posthog/clickhouse/hcl/roles/ingestion_small/shared/person_group_membership.hcl
rename to posthog/clickhouse/hcl/roles/coshared/person_group_membership_write/tables.hcl
diff --git a/posthog/clickhouse/hcl/roles/ingestion_events/local/person_group_membership.hcl b/posthog/clickhouse/hcl/roles/ingestion_events/local/person_group_membership.hcl
new file mode 100644
index 000000000000..66464637b9ac
--- /dev/null
+++ b/posthog/clickhouse/hcl/roles/ingestion_events/local/person_group_membership.hcl
@@ -0,0 +1,83 @@
+database "posthog" {
+ patch_dictionary "person_group_membership_config_dict" {
+ source "clickhouse" {
+ user = "default"
+ query = "SELECT team_id, config.1 AS group_type_index, config.2 AS enabled FROM (SELECT team_id, argMax(tuple(group_type_index, enabled), version) AS config FROM posthog.distributed_person_group_membership_config GROUP BY team_id) WHERE enabled = 1 AND group_type_index <= 4"
+ }
+ }
+
+ table "kafka_person_group_membership" {
+ column "team_id" {
+ type = "Int64"
+ }
+ column "distinct_id" {
+ type = "String"
+ }
+ column "properties" {
+ type = "String"
+ }
+ column "timestamp" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "person_mode" {
+ type = "Enum8('full'=0, 'propertyless'=1, 'force_upgrade'=2)"
+ }
+ engine "kafka" {
+ collection = "warpstream_ingestion"
+ topic_list = "clickhouse_events_json"
+ group_name = "clickhouse_person_group_membership"
+ format = "JSONEachRow"
+ num_consumers = 1
+ thread_per_consumer = true
+ max_block_size = 10000
+ skip_broken_messages = 100
+ poll_timeout_ms = 10000
+ poll_max_batch_size = 10000
+ flush_interval_ms = 7500
+ }
+ }
+
+ materialized_view "person_group_membership_mv" {
+ to_table = "posthog.writable_person_group_membership"
+ query = <<SQL
+SELECT
+ team_id,
+ group_type_index,
+ replaceRegexpAll(JSONExtractRaw(properties, concat('$group_', toString(group_type_index))), '^"|"$', '') AS group_key,
+ distinct_id,
+ min(timestamp) AS first_seen,
+ max(timestamp) AS last_seen
+FROM (
+ SELECT
+ team_id,
+ dictGet('posthog.person_group_membership_config_dict', 'group_type_index', tuple(team_id)) AS group_type_index,
+ distinct_id,
+ properties,
+ timestamp
+ FROM posthog.kafka_person_group_membership
+ WHERE dictHas('posthog.person_group_membership_config_dict', tuple(team_id)) AND (person_mode != 'propertyless')
+)
+WHERE notEmpty(group_key)
+GROUP BY team_id, group_type_index, group_key, distinct_id
+SQL
+
+ column "team_id" {
+ type = "Int64"
+ }
+ column "group_type_index" {
+ type = "UInt8"
+ }
+ column "group_key" {
+ type = "String"
+ }
+ column "distinct_id" {
+ type = "String"
+ }
+ column "first_seen" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ column "last_seen" {
+ type = "DateTime64(6, 'UTC')"
+ }
+ }
+}
diff --git a/posthog/clickhouse/hcl/sql/local-multi/ingestion_events.sql b/posthog/clickhouse/hcl/sql/local-multi/ingestion_events.sql
index e5f065cae3a9..f9d53d90b997 100644
--- a/posthog/clickhouse/hcl/sql/local-multi/ingestion_events.sql
+++ b/posthog/clickhouse/hcl/sql/local-multi/ingestion_events.sql
@@ -1,6 +1,12 @@
-- AUTO-GENERATED from the declarative HCL by ops/gen-sql.sh — do not edit.
-- Full CREATE schema for the local-multi/events node. Apply to a fresh ClickHouse to build it.
+CREATE TABLE posthog.distributed_person_group_membership_config (
+ team_id Int64,
+ group_type_index UInt8,
+ enabled UInt8,
+ version UInt64
+) ENGINE = Distributed('aux', 'posthog', 'person_group_membership_config', sipHash64(team_id));
CREATE TABLE posthog.kafka_events_json_native_json (
uuid UUID,
event String,
@@ -56,6 +62,13 @@ CREATE TABLE posthog.kafka_logs_avro (
pattern Nullable(String),
pattern_version Nullable(Int32)
) ENGINE = Kafka(warpstream_logs) SETTINGS input_format_avro_allow_missing_fields = 1, kafka_format = 'Avro', kafka_group_name = 'clickhouse-logs-avro-new', kafka_num_consumers = 1, kafka_poll_max_batch_size = 1000, kafka_poll_timeout_ms = 3000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_logs';
+CREATE TABLE posthog.kafka_person_group_membership (
+ team_id Int64,
+ distinct_id String,
+ properties String,
+ timestamp DateTime64(6, 'UTC'),
+ person_mode Enum8('full'=0, 'propertyless'=1, 'force_upgrade'=2)
+) ENGINE = Kafka(warpstream_ingestion) SETTINGS kafka_flush_interval_ms = 7500, kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_person_group_membership', kafka_max_block_size = 10000, kafka_num_consumers = 1, kafka_poll_max_batch_size = 10000, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_events_json';
CREATE TABLE posthog.query_log_archive (
initial_query_start_time DateTime64(6, 'UTC'),
team_id Int64,
@@ -257,6 +270,14 @@ CREATE TABLE posthog.writable_logs34 (
pattern_version UInt8,
internal_source LowCardinality(String) DEFAULT ''
) ENGINE = Distributed('logs', 'posthog', 'logs34') SETTINGS background_insert_batch = 1;
+CREATE TABLE posthog.writable_person_group_membership (
+ team_id Int64,
+ group_type_index UInt8,
+ group_key String,
+ distinct_id String,
+ first_seen SimpleAggregateFunction(min, DateTime64(6, 'UTC')),
+ last_seen SimpleAggregateFunction(max, DateTime64(6, 'UTC'))
+) ENGINE = Distributed('aux', 'posthog', 'sharded_person_group_membership', sipHash64(team_id, group_type_index, group_key));
CREATE MATERIALIZED VIEW posthog.events_json_table_mv TO posthog.writable_events_json (uuid UUID, event String, properties String, temporary_properties String, inserted_at DateTime64(3), timestamp DateTime64(6, 'UTC'), team_id Int64, distinct_id String, elements_chain String, created_at DateTime64(6, 'UTC'), person_id UUID, person_properties String, person_created_at DateTime64(3), group0_properties String, group1_properties String, group2_properties String, group3_properties String, group4_properties String, group0_created_at DateTime64(3), group1_created_at DateTime64(3), group2_created_at DateTime64(3), group3_created_at DateTime64(3), group4_created_at DateTime64(3), person_mode Enum8('full'=0, 'propertyless'=1, 'force_upgrade'=2), historical_migration Bool, captured_at DateTime64(6, 'UTC'), _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64, consumer_breadcrumbs Array(String), total_event_size UInt32) AS SELECT
*,
accurateCast(byteSize(*) + byteSize(toUInt32(0)), 'UInt32') AS total_event_size
@@ -351,3 +372,32 @@ CREATE MATERIALIZED VIEW posthog.kafka_logs34_avro_mv TO posthog.writable_logs34
toUInt8(ifNull(pattern_version, 0)) AS pattern_version,
_headers.value[indexOf(_headers.name, 'internal-source')] AS internal_source
FROM posthog.kafka_logs_avro;
+CREATE MATERIALIZED VIEW posthog.person_group_membership_mv TO posthog.writable_person_group_membership (team_id Int64, group_type_index UInt8, group_key String, distinct_id String, first_seen DateTime64(6, 'UTC'), last_seen DateTime64(6, 'UTC')) AS SELECT
+ team_id,
+ group_type_index,
+ replaceRegexpAll(
+ JSONExtractRaw(properties, concat('$group_', toString(group_type_index))),
+ '^"|"$',
+ ''
+ ) AS group_key,
+ distinct_id,
+ min(timestamp) AS first_seen,
+ max(timestamp) AS last_seen
+FROM
+ (
+ SELECT
+ team_id,
+ dictGet('posthog.person_group_membership_config_dict', 'group_type_index', tuple(team_id)) AS group_type_index,
+ distinct_id,
+ properties,
+ timestamp
+ FROM posthog.kafka_person_group_membership
+ WHERE
+ dictHas('posthog.person_group_membership_config_dict', tuple(team_id))
+ AND
+ (person_mode != 'propertyless')
+ )
+WHERE notEmpty(group_key)
+GROUP BY
+ team_id, group_type_index, group_key, distinct_id;
+CREATE OR REPLACE DICTIONARY posthog.person_group_membership_config_dict (`team_id` Int64, `group_type_index` UInt8 DEFAULT 255, `enabled` UInt8 DEFAULT 0) PRIMARY KEY team_id SOURCE(CLICKHOUSE(USER 'default' QUERY 'SELECT team_id, config.1 AS group_type_index, config.2 AS enabled FROM (SELECT team_id, argMax(tuple(group_type_index, enabled), version) AS config FROM posthog.distributed_person_group_membership_config GROUP BY team_id) WHERE enabled = 1 AND group_type_index <= 4')) LAYOUT(COMPLEX_KEY_HASHED()) LIFETIME(MIN 60 MAX 120);
diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sql
index 1d0014a9c545..b243d5ac1acd 100644
--- a/posthog/clickhouse/hcl/sql/local-single/all.sql
+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql
@@ -754,6 +754,13 @@ CREATE TABLE posthog.kafka_person_distinct_id_overrides (
is_deleted Int8,
version Int64
) ENGINE = Kafka(msk_cluster) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse-person-distinct-id-overrides', kafka_topic_list = 'clickhouse_person_distinct_id';
+CREATE TABLE posthog.kafka_person_group_membership (
+ team_id Int64,
+ distinct_id String,
+ properties String,
+ timestamp DateTime64(6, 'UTC'),
+ person_mode Enum8('full'=0, 'propertyless'=1, 'force_upgrade'=2)
+) ENGINE = Kafka(warpstream_ingestion) SETTINGS kafka_flush_interval_ms = 7500, kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_person_group_membership', kafka_max_block_size = 10000, kafka_num_consumers = 1, kafka_poll_max_batch_size = 10000, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_events_json';
CREATE TABLE posthog.kafka_person_overrides (
team_id Int32,
old_person_id UUID,
@@ -5507,6 +5514,34 @@ CREATE MATERIALIZED VIEW posthog.person_distinct_id_overrides_mv TO posthog.writ
_partition
FROM posthog.kafka_person_distinct_id_overrides
WHERE version > 0;
+CREATE MATERIALIZED VIEW posthog.person_group_membership_mv TO posthog.writable_person_group_membership (team_id Int64, group_type_index UInt8, group_key String, distinct_id String, first_seen DateTime64(6, 'UTC'), last_seen DateTime64(6, 'UTC')) AS SELECT
+ team_id,
+ group_type_index,
+ replaceRegexpAll(
+ JSONExtractRaw(properties, concat('$group_', toString(group_type_index))),
+ '^"|"$',
+ ''
+ ) AS group_key,
+ distinct_id,
+ min(timestamp) AS first_seen,
+ max(timestamp) AS last_seen
+FROM
+ (
+ SELECT
+ team_id,
+ dictGet('posthog.person_group_membership_config_dict', 'group_type_index', tuple(team_id)) AS group_type_index,
+ distinct_id,
+ properties,
+ timestamp
+ FROM posthog.kafka_person_group_membership
+ WHERE
+ dictHas('posthog.person_group_membership_config_dict', tuple(team_id))
+ AND
+ (person_mode != 'propertyless')
+ )
+WHERE notEmpty(group_key)
+GROUP BY
+ team_id, group_type_index, group_key, distinct_id;
CREATE MATERIALIZED VIEW posthog.person_mv TO posthog.writable_person (id UUID, created_at DateTime64(3), team_id Int64, properties String, is_identified Int8, is_deleted Int8, version UInt64, last_seen_at Nullable(DateTime64(3)), _timestamp Nullable(DateTime), _offset UInt64) AS SELECT
id,
created_at,
diff --git a/posthog/clickhouse/kafka_engine.py b/posthog/clickhouse/kafka_engine.py
index 1d4c1ed7fd15..bf71c1f8748a 100644
--- a/posthog/clickhouse/kafka_engine.py
+++ b/posthog/clickhouse/kafka_engine.py
@@ -68,6 +68,7 @@ CONSUMER_GROUP_ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_WS = (
)
CONSUMER_GROUP_ERROR_TRACKING_FINGERPRINT_ISSUE_STATE_WS = "clickhouse_error_tracking_fingerprint_issue_state_ws"
CONSUMER_GROUP_USAGE_REPORT_EVENTS_PREAGG = "clickhouse_usage_report_events_preagg"
+CONSUMER_GROUP_PERSON_GROUP_MEMBERSHIP = "clickhouse_person_group_membership"
STORAGE_POLICY = lambda: "SETTINGS storage_policy = 'hot_to_cold'" if settings.CLICKHOUSE_ENABLE_STORAGE_POLICY else ""
diff --git a/posthog/clickhouse/migrations/0369_person_group_membership_ingestion.py b/posthog/clickhouse/migrations/0369_person_group_membership_ingestion.py
new file mode 100644
index 000000000000..f6f94b479b6c
--- /dev/null
+++ b/posthog/clickhouse/migrations/0369_person_group_membership_ingestion.py
@@ -0,0 +1,37 @@
+from posthog.clickhouse.client.connection import NodeRole
+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions
+from posthog.models.person_group_membership.sql import (
+ DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE_SQL,
+ KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE_SQL,
+ PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY_SQL,
+ PERSON_GROUP_MEMBERSHIP_MV_SQL,
+ WRITABLE_PERSON_GROUP_MEMBERSHIP_TABLE_SQL,
+)
+
+operations = [
+ run_sql_with_exceptions(
+ DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE_SQL(),
+ node_roles=[NodeRole.INGESTION_EVENTS],
+ require_hosts=True,
+ ),
+ run_sql_with_exceptions(
+ PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY_SQL(),
+ node_roles=[NodeRole.INGESTION_EVENTS],
+ require_hosts=True,
+ ),
+ run_sql_with_exceptions(
+ WRITABLE_PERSON_GROUP_MEMBERSHIP_TABLE_SQL(),
+ node_roles=[NodeRole.INGESTION_EVENTS],
+ require_hosts=True,
+ ),
+ run_sql_with_exceptions(
+ KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE_SQL(),
+ node_roles=[NodeRole.INGESTION_EVENTS],
+ require_hosts=True,
+ ),
+ run_sql_with_exceptions(
+ PERSON_GROUP_MEMBERSHIP_MV_SQL(),
+ node_roles=[NodeRole.INGESTION_EVENTS],
+ require_hosts=True,
+ ),
+]
diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txt
index e0822ce38c94..731ac00dcf4d 100644
--- a/posthog/clickhouse/migrations/max_migration.txt
+++ b/posthog/clickhouse/migrations/max_migration.txt
@@ -1 +1 @@
-0368_email_status_events
+0369_person_group_membership_ingestion
diff --git a/posthog/models/person_group_membership/sql.py b/posthog/models/person_group_membership/sql.py
index 0164b71d8dd9..f1b7bb4b906b 100644
--- a/posthog/models/person_group_membership/sql.py
+++ b/posthog/models/person_group_membership/sql.py
@@ -3,11 +3,20 @@ from django.conf import settings
from posthog.hogql.escape_sql import escape_clickhouse_identifier, escape_clickhouse_string
from posthog.clickhouse.client.connection import ClickHouseUser, get_clickhouse_creds
+from posthog.clickhouse.kafka_engine import (
+ CONSUMER_GROUP_PERSON_GROUP_MEMBERSHIP,
+ kafka_engine,
+ kafka_num_consumers,
+ trim_quotes_expr,
+)
from posthog.clickhouse.table_engines import AggregatingMergeTree, Distributed, ReplacingMergeTree, ReplicationScheme
+from posthog.kafka_client.topics import KAFKA_EVENTS_JSON
PERSON_GROUP_MEMBERSHIP_TABLE = "person_group_membership"
SHARDED_PERSON_GROUP_MEMBERSHIP_TABLE = f"sharded_{PERSON_GROUP_MEMBERSHIP_TABLE}"
WRITABLE_PERSON_GROUP_MEMBERSHIP_TABLE = f"writable_{PERSON_GROUP_MEMBERSHIP_TABLE}"
+KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE = f"kafka_{PERSON_GROUP_MEMBERSHIP_TABLE}"
+PERSON_GROUP_MEMBERSHIP_MV = f"{PERSON_GROUP_MEMBERSHIP_TABLE}_mv"
PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE = "person_group_membership_config"
DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE = f"distributed_{PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE}"
@@ -26,6 +35,14 @@ PERSON_GROUP_MEMBERSHIP_COLUMNS = """
last_seen SimpleAggregateFunction(max, DateTime64(6, 'UTC'))
""".strip()
+KAFKA_PERSON_GROUP_MEMBERSHIP_COLUMNS = """
+ team_id Int64,
+ distinct_id String,
+ properties String,
+ timestamp DateTime64(6, 'UTC'),
+ person_mode Enum8('full' = 0, 'propertyless' = 1, 'force_upgrade' = 2)
+""".strip()
+
PERSON_GROUP_MEMBERSHIP_CONFIG_COLUMNS = """
team_id Int64,
group_type_index UInt8,
@@ -134,3 +151,68 @@ SOURCE(CLICKHOUSE({source}))
LIFETIME(MIN 60 MAX {PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY_LIFETIME_MAX_SECONDS})
LAYOUT(COMPLEX_KEY_HASHED())
"""
+
+
+def KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE_SQL() -> str:
+ # A dedicated consumer group keeps membership lag and failures off the main events consumer.
+ # Each row carries the full properties JSON, so blocks stay at the flag_evaluations sizes.
+ return f"""
+CREATE TABLE IF NOT EXISTS {KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE}
+(
+ {KAFKA_PERSON_GROUP_MEMBERSHIP_COLUMNS}
+)
+ENGINE = {
+ kafka_engine(
+ topic=KAFKA_EVENTS_JSON,
+ group=CONSUMER_GROUP_PERSON_GROUP_MEMBERSHIP,
+ named_collection=settings.CLICKHOUSE_KAFKA_WARPSTREAM_INGESTION_NAMED_COLLECTION,
+ )
+ }
+SETTINGS
+ kafka_max_block_size = 10000,
+ kafka_poll_max_batch_size = 10000,
+ kafka_poll_timeout_ms = 10000,
+ kafka_flush_interval_ms = 7500,
+ kafka_num_consumers = {kafka_num_consumers(1)},
+ kafka_thread_per_consumer = 1,
+ kafka_skip_broken_messages = 100
+"""
+
+
+# Matches the events $group_N columns the backfill reads, so numeric keys become strings in both paths.
+GROUP_KEY_EXPR = trim_quotes_expr("JSONExtractRaw(properties, concat('$group_', toString(group_type_index)))")
+
+
+def PERSON_GROUP_MEMBERSHIP_MV_SELECT_SQL(source_table: str) -> str:
+ # Filter on the dictionary before extracting the group key, so ineligible events skip JSON parsing.
+ # The dictionary holds only enabled, valid configs, so dictHas fails closed for every other team.
+ # dictHas also avoids the inverse dictionary lookup rewrite, which rejects dictGet comparisons on complex keys.
+ # Personless events cannot be backed by a person profile. The read path keeps identified persons only.
+ dictionary = f"{settings.CLICKHOUSE_DATABASE}.{PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY}"
+ return f"""SELECT
+ team_id,
+ group_type_index,
+ {GROUP_KEY_EXPR} AS group_key,
+ distinct_id,
+ min(timestamp) AS first_seen,
+ max(timestamp) AS last_seen
+FROM (
+ SELECT
+ team_id,
+ dictGet('{dictionary}', 'group_type_index', tuple(team_id)) AS group_type_index,
+ distinct_id,
+ properties,
+ timestamp
+ FROM {source_table}
+ WHERE dictHas('{dictionary}', tuple(team_id)) AND (person_mode != 'propertyless')
+)
+WHERE notEmpty(group_key)
+GROUP BY team_id, group_type_index, group_key, distinct_id"""
+
+
+def PERSON_GROUP_MEMBERSHIP_MV_SQL() -> str:
+ return f"""
+CREATE MATERIALIZED VIEW IF NOT EXISTS {PERSON_GROUP_MEMBERSHIP_MV}
+TO {WRITABLE_PERSON_GROUP_MEMBERSHIP_TABLE}
+AS {PERSON_GROUP_MEMBERSHIP_MV_SELECT_SQL(KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE)}
+"""
diff --git a/posthog/models/person_group_membership/test/__snapshots__/test_sql.ambr b/posthog/models/person_group_membership/test/__snapshots__/test_sql.ambr
index 26ae415fc6a1..83519754257b 100644
--- a/posthog/models/person_group_membership/test/__snapshots__/test_sql.ambr
+++ b/posthog/models/person_group_membership/test/__snapshots__/test_sql.ambr
@@ -15,3 +15,53 @@
'''
# ---
+# name: test_ingestion_sql[kafka_table]
+ '''
+
+ CREATE TABLE IF NOT EXISTS kafka_person_group_membership
+ (
+ team_id Int64,
+ distinct_id String,
+ properties String,
+ timestamp DateTime64(6, 'UTC'),
+ person_mode Enum8('full' = 0, 'propertyless' = 1, 'force_upgrade' = 2)
+ )
+ ENGINE = Kafka(warpstream_ingestion, kafka_topic_list = 'clickhouse_events_json_test', kafka_group_name = 'clickhouse_person_group_membership', kafka_format = 'JSONEachRow')
+ SETTINGS
+ kafka_max_block_size = 10000,
+ kafka_poll_max_batch_size = 10000,
+ kafka_poll_timeout_ms = 10000,
+ kafka_flush_interval_ms = 7500,
+ kafka_num_consumers = 1,
+ kafka_thread_per_consumer = 1,
+ kafka_skip_broken_messages = 100
+
+ '''
+# ---
+# name: test_ingestion_sql[view]
+ '''
+
+ CREATE MATERIALIZED VIEW IF NOT EXISTS person_group_membership_mv
+ TO writable_person_group_membership
+ AS SELECT
+ team_id,
+ group_type_index,
+ replaceRegexpAll(JSONExtractRaw(properties, concat('$group_', toString(group_type_index))), '^"|"$', '') AS group_key,
+ distinct_id,
+ min(timestamp) AS first_seen,
+ max(timestamp) AS last_seen
+ FROM (
+ SELECT
+ team_id,
+ dictGet('posthog.person_group_membership_config_dict', 'group_type_index', tuple(team_id)) AS group_type_index,
+ distinct_id,
+ properties,
+ timestamp
+ FROM kafka_person_group_membership
+ WHERE dictHas('posthog.person_group_membership_config_dict', tuple(team_id)) AND (person_mode != 'propertyless')
+ )
+ WHERE notEmpty(group_key)
+ GROUP BY team_id, group_type_index, group_key, distinct_id
+
+ '''
+# ---
diff --git a/posthog/models/person_group_membership/test/test_sql.py b/posthog/models/person_group_membership/test/test_sql.py
index 6a79b226a0e4..cb0a5bee0c8e 100644
--- a/posthog/models/person_group_membership/test/test_sql.py
+++ b/posthog/models/person_group_membership/test/test_sql.py
@@ -1,3 +1,4 @@
+import json
import importlib
from collections.abc import Callable, Iterator
from datetime import UTC, datetime
@@ -17,15 +18,21 @@ from posthog.clickhouse.schema import (
)
from posthog.models.person_group_membership.sql import (
DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE,
+ KAFKA_PERSON_GROUP_MEMBERSHIP_COLUMNS,
+ KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE_SQL,
PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY,
PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY_SQL,
PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE,
+ PERSON_GROUP_MEMBERSHIP_MV_SELECT_SQL,
+ PERSON_GROUP_MEMBERSHIP_MV_SQL,
PERSON_GROUP_MEMBERSHIP_TABLE,
SHARDED_PERSON_GROUP_MEMBERSHIP_TABLE,
WRITABLE_PERSON_GROUP_MEMBERSHIP_TABLE,
)
MIGRATION = "posthog.clickhouse.migrations.0343_person_group_membership"
+INGESTION_MIGRATION = "posthog.clickhouse.migrations.0369_person_group_membership_ingestion"
+SOURCE_TABLE = "test_source_person_group_membership"
def test_dictionary_sql(snapshot, mocker, settings) -> None:
@@ -38,30 +45,52 @@ def test_dictionary_sql(snapshot, mocker, settings) -> None:
creds.assert_called_once_with(ClickHouseUser.DICT_READER)
[email protected]("deployment", ["", "DEV", "US", "EU"])
[email protected]("multinode", [False, True])
-def test_migration_placement(deployment: str, multinode: bool) -> None:
- module = importlib.import_module(MIGRATION)
- try:
- with patch.multiple("posthog.settings", CLOUD_DEPLOYMENT=deployment, MULTINODE_CLICKHOUSE=multinode):
- operations = importlib.reload(module).operations
- expected_roles = [
+ "build_sql", [KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE_SQL, PERSON_GROUP_MEMBERSHIP_MV_SQL], ids=["kafka_table", "view"]
+)
+def test_ingestion_sql(build_sql: Callable[[], str], snapshot, settings) -> None:
+ settings.CLICKHOUSE_DATABASE = "posthog"
+ settings.CLICKHOUSE_KAFKA_WARPSTREAM_INGESTION_NAMED_COLLECTION = "warpstream_ingestion"
+ assert build_sql() == snapshot
+
+
+ "migration,expected_roles,consumes_kafka",
+ [
+ (
+ MIGRATION,
+ [
[NodeRole.AUX],
[NodeRole.AUX],
[NodeRole.DATA, NodeRole.INGESTION_SMALL],
[NodeRole.DATA],
[NodeRole.INGESTION_SMALL],
[NodeRole.INGESTION_SMALL],
- ]
+ ],
+ False,
+ ),
+ (INGESTION_MIGRATION, [[NodeRole.INGESTION_EVENTS]] * 5, True),
+ ],
+ ids=["storage", "ingestion"],
+)
[email protected]("deployment", ["", "DEV", "US", "EU"])
[email protected]("multinode", [False, True])
+def test_migration_placement(
+ migration: str, expected_roles: list[list[NodeRole]], consumes_kafka: bool, deployment: str, multinode: bool
+) -> None:
+ module = importlib.import_module(migration)
+ try:
+ with patch.multiple("posthog.settings", CLOUD_DEPLOYMENT=deployment, MULTINODE_CLICKHOUSE=multinode):
+ operations = importlib.reload(module).operations
assert [op._node_roles for op in operations] == expected_roles
assert [op._effective_node_roles for op in operations] == (
- expected_roles if deployment or multinode else [[NodeRole.ALL]] * 6
+ expected_roles if deployment or multinode else [[NodeRole.ALL]] * len(expected_roles)
)
for op in operations:
assert "ON CLUSTER" not in op._sql
assert "IF NOT EXISTS" in op._sql
- assert "Kafka" not in op._sql
- assert "MATERIALIZED VIEW" not in op._sql
+ assert any("Kafka" in op._sql for op in operations) == consumes_kafka
+ assert any("kafka_events_json_ws" in op._sql for op in operations) is False
finally:
importlib.reload(module)
@@ -214,7 +243,8 @@ def test_distributed_writes_and_reads_aggregate_overlap() -> None:
)
assert sync_execute(
f"""SELECT team_id, group_type_index, group_key, distinct_id, min(first_seen), max(last_seen)
- FROM {PERSON_GROUP_MEMBERSHIP_TABLE} GROUP BY team_id, group_type_index, group_key, distinct_id"""
+ FROM {PERSON_GROUP_MEMBERSHIP_TABLE} GROUP BY team_id, group_type_index, group_key, distinct_id
+ ORDER BY distinct_id"""
) == [
(
1001,
@@ -254,3 +284,60 @@ def test_storage_supports_distinct_id_and_team_deletion(lightweight: bool) -> No
)
== expected
)
+
+
[email protected]("group_type_index", range(5))
[email protected]("membership_schema")
+def test_ingestion_admits_only_eligible_events(group_type_index: int) -> None:
+ sync_execute(
+ f"INSERT INTO {DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE} VALUES",
+ [(2001, group_type_index, 1, 1), (2002, group_type_index, 1, 1), (2002, group_type_index, 0, 2)],
+ settings={"distributed_foreground_insert": 1},
+ )
+ sync_execute(f"SYSTEM RELOAD DICTIONARY {PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY}")
+ every_group = json.dumps({f"$group_{index}": f"account-{index}" for index in range(5)})
+ configured_empty = json.dumps({f"$group_{group_type_index}": ""})
+ events = [
+ (2001, "identified", every_group, datetime(2024, 1, 2, tzinfo=UTC), "full"),
+ (2001, "identified", every_group, datetime(2024, 1, 1, tzinfo=UTC), "force_upgrade"),
+ (2001, "identified", every_group, datetime(2024, 1, 3, tzinfo=UTC), "full"),
+ (2001, "numeric-key", json.dumps({f"$group_{group_type_index}": 42}), datetime(2024, 1, 1, tzinfo=UTC), "full"),
+ (2001, "personless", every_group, datetime(2024, 1, 1, tzinfo=UTC), "propertyless"),
+ (2001, "empty-key", configured_empty, datetime(2024, 1, 1, tzinfo=UTC), "full"),
+ (2001, "no-properties", "", datetime(2024, 1, 1, tzinfo=UTC), "full"),
+ (2002, "disabled-team", every_group, datetime(2024, 1, 1, tzinfo=UTC), "full"),
+ (2003, "missing-team", every_group, datetime(2024, 1, 1, tzinfo=UTC), "full"),
+ ]
+ sync_execute(f"DROP TABLE IF EXISTS {SOURCE_TABLE}")
+ sync_execute(f"CREATE TABLE {SOURCE_TABLE} ({KAFKA_PERSON_GROUP_MEMBERSHIP_COLUMNS}) ENGINE = Memory")
+ try:
+ sync_execute(f"INSERT INTO {SOURCE_TABLE} VALUES", events)
+ for _ in range(2):
+ sync_execute(
+ f"INSERT INTO {WRITABLE_PERSON_GROUP_MEMBERSHIP_TABLE} {PERSON_GROUP_MEMBERSHIP_MV_SELECT_SQL(SOURCE_TABLE)}",
+ settings={"distributed_foreground_insert": 1},
+ )
+ finally:
+ sync_execute(f"DROP TABLE IF EXISTS {SOURCE_TABLE}")
+ assert sync_execute(
+ f"""SELECT team_id, group_type_index, group_key, distinct_id, min(first_seen), max(last_seen)
+ FROM {PERSON_GROUP_MEMBERSHIP_TABLE} GROUP BY team_id, group_type_index, group_key, distinct_id
+ ORDER BY distinct_id"""
+ ) == [
+ (
+ 2001,
+ group_type_index,
+ f"account-{group_type_index}",
+ "identified",
+ datetime(2024, 1, 1, tzinfo=UTC),
+ datetime(2024, 1, 3, tzinfo=UTC),
+ ),
+ (
+ 2001,
+ group_type_index,
+ "42",
+ "numeric-key",
+ datetime(2024, 1, 1, tzinfo=UTC),
+ datetime(2024, 1, 1, tzinfo=UTC),
+ ),
+ ]
diff --git a/products/customer_analytics/backend/logic/membership_backfill.py b/products/customer_analytics/backend/logic/membership_backfill.py
index aeee6acdeadf..cbf4f03caccf 100644
--- a/products/customer_analytics/backend/logic/membership_backfill.py
+++ b/products/customer_analytics/backend/logic/membership_backfill.py
@@ -788,7 +788,7 @@ class MembershipBackfillService:
results = (
get_cluster(cluster=settings.CLICKHOUSE_MIGRATIONS_CLUSTER, data_cluster=settings.CLICKHOUSE_CLUSTER)
- .map_hosts_by_roles(check, [NodeRole.INGESTION_SMALL], concurrency=2, require_hosts=True)
+ .map_hosts_by_roles(check, [NodeRole.INGESTION_EVENTS], concurrency=2, require_hosts=True)
.result()
)
self.check_attempt()