:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(customer-analytics): ingest account membership from live events (#128660)
Arthur Moreira de Deuscbd1cef9ff2c
diff --git a/posthog/clickhouse/hcl/golden/local-multi/ingestion_events.hcl b/posthog/clickhouse/hcl/golden/local-multi/ingestion_events.hclindex 33d7a50d4b80..4a56d0bbfaee 100644--- a/posthog/clickhouse/hcl/golden/local-multi/ingestion_events.hcl+++ b/posthog/clickhouse/hcl/golden/local-multi/ingestion_events.hcl@@ -8,6 +8,27 @@ node "events" {}database "posthog" {+ table "distributed_person_group_membership_config" {+ column "team_id" {+ type = "Int64"+ }+ column "group_type_index" {+ type = "UInt8"+ }+ column "enabled" {+ type = "UInt8"+ }+ column "version" {+ type = "UInt64"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "person_group_membership_config"+ sharding_key = "sipHash64(team_id)"+ }+ }+table "kafka_events_json_native_json" {column "uuid" {type = "UUID"@@ -195,6 +216,37 @@ database "posthog" {}}+ table "kafka_person_group_membership" {+ column "team_id" {+ type = "Int64"+ }+ column "distinct_id" {+ type = "String"+ }+ column "properties" {+ type = "String"+ }+ column "timestamp" {+ type = "DateTime64(6, 'UTC')"+ }+ column "person_mode" {+ type = "Enum8('full'=0, 'propertyless'=1, 'force_upgrade'=2)"+ }+ engine "kafka" {+ collection = "warpstream_ingestion"+ topic_list = "clickhouse_events_json"+ group_name = "clickhouse_person_group_membership"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 10000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ poll_max_batch_size = 10000+ flush_interval_ms = 7500+ thread_per_consumer = true+ }+ }+table "query_log_archive" {column "initial_query_start_time" {type = "DateTime64(6, 'UTC')"@@ -822,6 +874,33 @@ database "posthog" {}}+ table "writable_person_group_membership" {+ column "team_id" {+ type = "Int64"+ }+ column "group_type_index" {+ type = "UInt8"+ }+ column "group_key" {+ type = "String"+ }+ column "distinct_id" {+ type = "String"+ }+ column "first_seen" {+ type = "SimpleAggregateFunction(min, DateTime64(6, 'UTC'))"+ }+ column "last_seen" {+ type = "SimpleAggregateFunction(max, DateTime64(6, 'UTC'))"+ }+ engine "distributed" {+ cluster_name = "aux"+ remote_database = "posthog"+ remote_table = "sharded_person_group_membership"+ sharding_key = "sipHash64(team_id, group_type_index, group_key)"+ }+ }+materialized_view "events_json_table_mv" {to_table = "posthog.writable_events_json"query = <<SQL@@ -1097,6 +1176,84 @@ SQLtype = "LowCardinality(String)"}}++ materialized_view "person_group_membership_mv" {+ to_table = "posthog.writable_person_group_membership"+ query = <<SQL+SELECT+ team_id,+ group_type_index,+ replaceRegexpAll(+ JSONExtractRaw(properties, concat('$group_', toString(group_type_index))),+ '^"|"$',+ ''+ ) AS group_key,+ distinct_id,+ min(timestamp) AS first_seen,+ max(timestamp) AS last_seen+FROM+ (+ SELECT+ team_id,+ dictGet('posthog.person_group_membership_config_dict', 'group_type_index', tuple(team_id)) AS group_type_index,+ distinct_id,+ properties,+ timestamp+ FROM posthog.kafka_person_group_membership+ WHERE+ dictHas('posthog.person_group_membership_config_dict', tuple(team_id))+ AND+ (person_mode != 'propertyless')+ )+WHERE notEmpty(group_key)+GROUP BY+ team_id, group_type_index, group_key, distinct_id+SQL++ column "team_id" {+ type = "Int64"+ }+ column "group_type_index" {+ type = "UInt8"+ }+ column "group_key" {+ type = "String"+ }+ column "distinct_id" {+ type = "String"+ }+ column "first_seen" {+ type = "DateTime64(6, 'UTC')"+ }+ column "last_seen" {+ type = "DateTime64(6, 'UTC')"+ }+ }++ dictionary "person_group_membership_config_dict" {+ primary_key = ["team_id"]+ lifetime {+ min = 60+ max = 120+ }+ attribute "team_id" {+ type = "Int64"+ }+ attribute "group_type_index" {+ type = "UInt8"+ default = "255"+ }+ attribute "enabled" {+ type = "UInt8"+ default = "0"+ }+ source "clickhouse" {+ user = "default"+ query = "SELECT team_id, config.1 AS group_type_index, config.2 AS enabled FROM (SELECT team_id, argMax(tuple(group_type_index, enabled), version) AS config FROM posthog.distributed_person_group_membership_config GROUP BY team_id) WHERE enabled = 1 AND group_type_index <= 4"+ }+ layout "complex_key_hashed" {+ }+ }}named_collection "msk_cluster" {diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hclindex a45ad09f78a4..84a59f0ec753 100644--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl@@ -4254,6 +4254,37 @@ database "posthog" {}}+ table "kafka_person_group_membership" {+ column "team_id" {+ type = "Int64"+ }+ column "distinct_id" {+ type = "String"+ }+ column "properties" {+ type = "String"+ }+ column "timestamp" {+ type = "DateTime64(6, 'UTC')"+ }+ column "person_mode" {+ type = "Enum8('full'=0, 'propertyless'=1, 'force_upgrade'=2)"+ }+ engine "kafka" {+ collection = "warpstream_ingestion"+ topic_list = "clickhouse_events_json"+ group_name = "clickhouse_person_group_membership"+ format = "JSONEachRow"+ num_consumers = 1+ max_block_size = 10000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ poll_max_batch_size = 10000+ flush_interval_ms = 7500+ thread_per_consumer = true+ }+ }+table "kafka_person_overrides" {column "team_id" {type = "Int32"@@ -24098,6 +24129,59 @@ SQL}}+ materialized_view "person_group_membership_mv" {+ to_table = "posthog.writable_person_group_membership"+ query = <<SQL+SELECT+ team_id,+ group_type_index,+ replaceRegexpAll(+ JSONExtractRaw(properties, concat('$group_', toString(group_type_index))),+ '^"|"$',+ ''+ ) AS group_key,+ distinct_id,+ min(timestamp) AS first_seen,+ max(timestamp) AS last_seen+FROM+ (+ SELECT+ team_id,+ dictGet('posthog.person_group_membership_config_dict', 'group_type_index', tuple(team_id)) AS group_type_index,+ distinct_id,+ properties,+ timestamp+ FROM posthog.kafka_person_group_membership+ WHERE+ dictHas('posthog.person_group_membership_config_dict', tuple(team_id))+ AND+ (person_mode != 'propertyless')+ )+WHERE notEmpty(group_key)+GROUP BY+ team_id, group_type_index, group_key, distinct_id+SQL++ column "team_id" {+ type = "Int64"+ }+ column "group_type_index" {+ type = "UInt8"+ }+ column "group_key" {+ type = "String"+ }+ column "distinct_id" {+ type = "String"+ }+ column "first_seen" {+ type = "DateTime64(6, 'UTC')"+ }+ column "last_seen" {+ type = "DateTime64(6, 'UTC')"+ }+ }+materialized_view "person_mv" {to_table = "posthog.writable_person"query = <<SQLdiff --git a/posthog/clickhouse/hcl/manifest.hcl b/posthog/clickhouse/hcl/manifest.hclindex 581c9249a671..da26b32f39c5 100644--- a/posthog/clickhouse/hcl/manifest.hcl+++ b/posthog/clickhouse/hcl/manifest.hcl@@ -130,11 +130,11 @@ role "data" {# Beyond that they carry only the shared query_log_archive path. Role names are the# hostClusterRole macros migrations target: events, small, medium.role "events" {- env "local-multi" { layers = ["roles/shared/qla.hcl", "roles/coshared/named_collections", "roles/coshared/logs_avro_ingest", "roles/coshared/events_json_write", "roles/ingestion_events/local"] }+ env "local-multi" { layers = ["roles/shared/qla.hcl", "roles/coshared/named_collections", "roles/coshared/logs_avro_ingest", "roles/coshared/events_json_write", "roles/shared/person_group_membership.hcl", "roles/coshared/person_group_membership_config", "roles/coshared/person_group_membership_write", "roles/ingestion_events/local"] }}role "small" {- env "local-multi" { layers = ["roles/shared/qla.hcl", "roles/shared/person_group_membership.hcl", "roles/coshared/person_group_membership_config", "roles/ingestion_small/shared", "roles/coshared/named_collections", "roles/coshared/aux_small", "roles/coshared/log_entries_write", "roles/coshared/log_entries_aux_write", "roles/coshared/session_replay_write", "roles/ingestion_small/local"] }+ env "local-multi" { layers = ["roles/shared/qla.hcl", "roles/shared/person_group_membership.hcl", "roles/coshared/person_group_membership_config", "roles/coshared/person_group_membership_write", "roles/coshared/named_collections", "roles/coshared/aux_small", "roles/coshared/log_entries_write", "roles/coshared/log_entries_aux_write", "roles/coshared/session_replay_write", "roles/ingestion_small/local"] }}role "medium" {@@ -146,7 +146,7 @@ role "medium" {# and not MULTINODE_CLICKHOUSE. Composed as the deduped union of the local-multi stacks it# hosts, so any name two of those roles declare fails this load instead of drifting.role "all" {- env "local-single" { layers = ["roles/shared", "roles/coshared/named_collections", "roles/coshared/apm_ingest", "roles/coshared/logs_avro_ingest", "roles/coshared/custom_metrics", "roles/ops/shared", "roles/ops/local", "roles/logs/base", "roles/logs/traces", "roles/logs/traces_kafka_metrics", "roles/logs/metrics", "roles/logs/local", "roles/apm/shared", "roles/apm/local", "roles/coshared/ai_events_data", "roles/coshared/person_shims", "roles/ai_events/shared", "roles/ai_events/local", "roles/coshared/aux_data", "roles/coshared/log_entries_aux", "roles/coshared/aux_small", "roles/auxiliary/shared", "roles/auxiliary/local", "roles/coshared/sessions_data", "roles/coshared/main_join_readers", "roles/coshared/tophog", "roles/coshared/events_recent", "roles/coshared/events_recent_write", "roles/coshared/batch_exports_data", "roles/coshared/ingestion_warnings_store", "roles/coshared/events_json_write", "roles/coshared/log_entries_write", "roles/coshared/log_entries_aux_write", "roles/coshared/session_replay_write", "roles/coshared/person_group_membership_config", "roles/ingestion_small/shared", "roles/data/shared", "roles/data/common", "roles/coshared/main_proxies", "roles/data/local", "roles/ingestion_events/local", "roles/ingestion_events/local-single", "roles/ingestion_small/local", "roles/ingestion_medium/local"] }+ env "local-single" { layers = ["roles/shared", "roles/coshared/named_collections", "roles/coshared/apm_ingest", "roles/coshared/logs_avro_ingest", "roles/coshared/custom_metrics", "roles/ops/shared", "roles/ops/local", "roles/logs/base", "roles/logs/traces", "roles/logs/traces_kafka_metrics", "roles/logs/metrics", "roles/logs/local", "roles/apm/shared", "roles/apm/local", "roles/coshared/ai_events_data", "roles/coshared/person_shims", "roles/ai_events/shared", "roles/ai_events/local", "roles/coshared/aux_data", "roles/coshared/log_entries_aux", "roles/coshared/aux_small", "roles/auxiliary/shared", "roles/auxiliary/local", "roles/coshared/sessions_data", "roles/coshared/main_join_readers", "roles/coshared/tophog", "roles/coshared/events_recent", "roles/coshared/events_recent_write", "roles/coshared/batch_exports_data", "roles/coshared/ingestion_warnings_store", "roles/coshared/events_json_write", "roles/coshared/log_entries_write", "roles/coshared/log_entries_aux_write", "roles/coshared/session_replay_write", "roles/coshared/person_group_membership_config", "roles/coshared/person_group_membership_write", "roles/data/shared", "roles/data/common", "roles/coshared/main_proxies", "roles/data/local", "roles/ingestion_events/local", "roles/ingestion_events/local-single", "roles/ingestion_small/local", "roles/ingestion_medium/local"] }}# role "endpoints" {diff --git a/posthog/clickhouse/hcl/roles/ingestion_small/shared/person_group_membership.hcl b/posthog/clickhouse/hcl/roles/coshared/person_group_membership_write/tables.hclsimilarity index 100%rename from posthog/clickhouse/hcl/roles/ingestion_small/shared/person_group_membership.hclrename to posthog/clickhouse/hcl/roles/coshared/person_group_membership_write/tables.hcldiff --git a/posthog/clickhouse/hcl/roles/ingestion_events/local/person_group_membership.hcl b/posthog/clickhouse/hcl/roles/ingestion_events/local/person_group_membership.hclnew file mode 100644index 000000000000..66464637b9ac--- /dev/null+++ b/posthog/clickhouse/hcl/roles/ingestion_events/local/person_group_membership.hcl@@ -0,0 +1,83 @@+database "posthog" {+ patch_dictionary "person_group_membership_config_dict" {+ source "clickhouse" {+ user = "default"+ query = "SELECT team_id, config.1 AS group_type_index, config.2 AS enabled FROM (SELECT team_id, argMax(tuple(group_type_index, enabled), version) AS config FROM posthog.distributed_person_group_membership_config GROUP BY team_id) WHERE enabled = 1 AND group_type_index <= 4"+ }+ }++ table "kafka_person_group_membership" {+ column "team_id" {+ type = "Int64"+ }+ column "distinct_id" {+ type = "String"+ }+ column "properties" {+ type = "String"+ }+ column "timestamp" {+ type = "DateTime64(6, 'UTC')"+ }+ column "person_mode" {+ type = "Enum8('full'=0, 'propertyless'=1, 'force_upgrade'=2)"+ }+ engine "kafka" {+ collection = "warpstream_ingestion"+ topic_list = "clickhouse_events_json"+ group_name = "clickhouse_person_group_membership"+ format = "JSONEachRow"+ num_consumers = 1+ thread_per_consumer = true+ max_block_size = 10000+ skip_broken_messages = 100+ poll_timeout_ms = 10000+ poll_max_batch_size = 10000+ flush_interval_ms = 7500+ }+ }++ materialized_view "person_group_membership_mv" {+ to_table = "posthog.writable_person_group_membership"+ query = <<SQL+SELECT+ team_id,+ group_type_index,+ replaceRegexpAll(JSONExtractRaw(properties, concat('$group_', toString(group_type_index))), '^"|"$', '') AS group_key,+ distinct_id,+ min(timestamp) AS first_seen,+ max(timestamp) AS last_seen+FROM (+ SELECT+ team_id,+ dictGet('posthog.person_group_membership_config_dict', 'group_type_index', tuple(team_id)) AS group_type_index,+ distinct_id,+ properties,+ timestamp+ FROM posthog.kafka_person_group_membership+ WHERE dictHas('posthog.person_group_membership_config_dict', tuple(team_id)) AND (person_mode != 'propertyless')+)+WHERE notEmpty(group_key)+GROUP BY team_id, group_type_index, group_key, distinct_id+SQL++ column "team_id" {+ type = "Int64"+ }+ column "group_type_index" {+ type = "UInt8"+ }+ column "group_key" {+ type = "String"+ }+ column "distinct_id" {+ type = "String"+ }+ column "first_seen" {+ type = "DateTime64(6, 'UTC')"+ }+ column "last_seen" {+ type = "DateTime64(6, 'UTC')"+ }+ }+}diff --git a/posthog/clickhouse/hcl/sql/local-multi/ingestion_events.sql b/posthog/clickhouse/hcl/sql/local-multi/ingestion_events.sqlindex e5f065cae3a9..f9d53d90b997 100644--- a/posthog/clickhouse/hcl/sql/local-multi/ingestion_events.sql+++ b/posthog/clickhouse/hcl/sql/local-multi/ingestion_events.sql@@ -1,6 +1,12 @@-- AUTO-GENERATED from the declarative HCL by ops/gen-sql.sh — do not edit.-- Full CREATE schema for the local-multi/events node. Apply to a fresh ClickHouse to build it.+CREATE TABLE posthog.distributed_person_group_membership_config (+ team_id Int64,+ group_type_index UInt8,+ enabled UInt8,+ version UInt64+) ENGINE = Distributed('aux', 'posthog', 'person_group_membership_config', sipHash64(team_id));CREATE TABLE posthog.kafka_events_json_native_json (uuid UUID,event String,@@ -56,6 +62,13 @@ CREATE TABLE posthog.kafka_logs_avro (pattern Nullable(String),pattern_version Nullable(Int32)) ENGINE = Kafka(warpstream_logs) SETTINGS input_format_avro_allow_missing_fields = 1, kafka_format = 'Avro', kafka_group_name = 'clickhouse-logs-avro-new', kafka_num_consumers = 1, kafka_poll_max_batch_size = 1000, kafka_poll_timeout_ms = 3000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_logs';+CREATE TABLE posthog.kafka_person_group_membership (+ team_id Int64,+ distinct_id String,+ properties String,+ timestamp DateTime64(6, 'UTC'),+ person_mode Enum8('full'=0, 'propertyless'=1, 'force_upgrade'=2)+) ENGINE = Kafka(warpstream_ingestion) SETTINGS kafka_flush_interval_ms = 7500, kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_person_group_membership', kafka_max_block_size = 10000, kafka_num_consumers = 1, kafka_poll_max_batch_size = 10000, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_events_json';CREATE TABLE posthog.query_log_archive (initial_query_start_time DateTime64(6, 'UTC'),team_id Int64,@@ -257,6 +270,14 @@ CREATE TABLE posthog.writable_logs34 (pattern_version UInt8,internal_source LowCardinality(String) DEFAULT '') ENGINE = Distributed('logs', 'posthog', 'logs34') SETTINGS background_insert_batch = 1;+CREATE TABLE posthog.writable_person_group_membership (+ team_id Int64,+ group_type_index UInt8,+ group_key String,+ distinct_id String,+ first_seen SimpleAggregateFunction(min, DateTime64(6, 'UTC')),+ last_seen SimpleAggregateFunction(max, DateTime64(6, 'UTC'))+) ENGINE = Distributed('aux', 'posthog', 'sharded_person_group_membership', sipHash64(team_id, group_type_index, group_key));CREATE MATERIALIZED VIEW posthog.events_json_table_mv TO posthog.writable_events_json (uuid UUID, event String, properties String, temporary_properties String, inserted_at DateTime64(3), timestamp DateTime64(6, 'UTC'), team_id Int64, distinct_id String, elements_chain String, created_at DateTime64(6, 'UTC'), person_id UUID, person_properties String, person_created_at DateTime64(3), group0_properties String, group1_properties String, group2_properties String, group3_properties String, group4_properties String, group0_created_at DateTime64(3), group1_created_at DateTime64(3), group2_created_at DateTime64(3), group3_created_at DateTime64(3), group4_created_at DateTime64(3), person_mode Enum8('full'=0, 'propertyless'=1, 'force_upgrade'=2), historical_migration Bool, captured_at DateTime64(6, 'UTC'), _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64, consumer_breadcrumbs Array(String), total_event_size UInt32) AS SELECT*,accurateCast(byteSize(*) + byteSize(toUInt32(0)), 'UInt32') AS total_event_size@@ -351,3 +372,32 @@ CREATE MATERIALIZED VIEW posthog.kafka_logs34_avro_mv TO posthog.writable_logs34toUInt8(ifNull(pattern_version, 0)) AS pattern_version,_headers.value[indexOf(_headers.name, 'internal-source')] AS internal_sourceFROM posthog.kafka_logs_avro;+CREATE MATERIALIZED VIEW posthog.person_group_membership_mv TO posthog.writable_person_group_membership (team_id Int64, group_type_index UInt8, group_key String, distinct_id String, first_seen DateTime64(6, 'UTC'), last_seen DateTime64(6, 'UTC')) AS SELECT+ team_id,+ group_type_index,+ replaceRegexpAll(+ JSONExtractRaw(properties, concat('$group_', toString(group_type_index))),+ '^"|"$',+ ''+ ) AS group_key,+ distinct_id,+ min(timestamp) AS first_seen,+ max(timestamp) AS last_seen+FROM+ (+ SELECT+ team_id,+ dictGet('posthog.person_group_membership_config_dict', 'group_type_index', tuple(team_id)) AS group_type_index,+ distinct_id,+ properties,+ timestamp+ FROM posthog.kafka_person_group_membership+ WHERE+ dictHas('posthog.person_group_membership_config_dict', tuple(team_id))+ AND+ (person_mode != 'propertyless')+ )+WHERE notEmpty(group_key)+GROUP BY+ team_id, group_type_index, group_key, distinct_id;+CREATE OR REPLACE DICTIONARY posthog.person_group_membership_config_dict (`team_id` Int64, `group_type_index` UInt8 DEFAULT 255, `enabled` UInt8 DEFAULT 0) PRIMARY KEY team_id SOURCE(CLICKHOUSE(USER 'default' QUERY 'SELECT team_id, config.1 AS group_type_index, config.2 AS enabled FROM (SELECT team_id, argMax(tuple(group_type_index, enabled), version) AS config FROM posthog.distributed_person_group_membership_config GROUP BY team_id) WHERE enabled = 1 AND group_type_index <= 4')) LAYOUT(COMPLEX_KEY_HASHED()) LIFETIME(MIN 60 MAX 120);diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sqlindex 1d0014a9c545..b243d5ac1acd 100644--- a/posthog/clickhouse/hcl/sql/local-single/all.sql+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql@@ -754,6 +754,13 @@ CREATE TABLE posthog.kafka_person_distinct_id_overrides (is_deleted Int8,version Int64) ENGINE = Kafka(msk_cluster) SETTINGS kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse-person-distinct-id-overrides', kafka_topic_list = 'clickhouse_person_distinct_id';+CREATE TABLE posthog.kafka_person_group_membership (+ team_id Int64,+ distinct_id String,+ properties String,+ timestamp DateTime64(6, 'UTC'),+ person_mode Enum8('full'=0, 'propertyless'=1, 'force_upgrade'=2)+) ENGINE = Kafka(warpstream_ingestion) SETTINGS kafka_flush_interval_ms = 7500, kafka_format = 'JSONEachRow', kafka_group_name = 'clickhouse_person_group_membership', kafka_max_block_size = 10000, kafka_num_consumers = 1, kafka_poll_max_batch_size = 10000, kafka_poll_timeout_ms = 10000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_events_json';CREATE TABLE posthog.kafka_person_overrides (team_id Int32,old_person_id UUID,@@ -5507,6 +5514,34 @@ CREATE MATERIALIZED VIEW posthog.person_distinct_id_overrides_mv TO posthog.writ_partitionFROM posthog.kafka_person_distinct_id_overridesWHERE version > 0;+CREATE MATERIALIZED VIEW posthog.person_group_membership_mv TO posthog.writable_person_group_membership (team_id Int64, group_type_index UInt8, group_key String, distinct_id String, first_seen DateTime64(6, 'UTC'), last_seen DateTime64(6, 'UTC')) AS SELECT+ team_id,+ group_type_index,+ replaceRegexpAll(+ JSONExtractRaw(properties, concat('$group_', toString(group_type_index))),+ '^"|"$',+ ''+ ) AS group_key,+ distinct_id,+ min(timestamp) AS first_seen,+ max(timestamp) AS last_seen+FROM+ (+ SELECT+ team_id,+ dictGet('posthog.person_group_membership_config_dict', 'group_type_index', tuple(team_id)) AS group_type_index,+ distinct_id,+ properties,+ timestamp+ FROM posthog.kafka_person_group_membership+ WHERE+ dictHas('posthog.person_group_membership_config_dict', tuple(team_id))+ AND+ (person_mode != 'propertyless')+ )+WHERE notEmpty(group_key)+GROUP BY+ team_id, group_type_index, group_key, distinct_id;CREATE MATERIALIZED VIEW posthog.person_mv TO posthog.writable_person (id UUID, created_at DateTime64(3), team_id Int64, properties String, is_identified Int8, is_deleted Int8, version UInt64, last_seen_at Nullable(DateTime64(3)), _timestamp Nullable(DateTime), _offset UInt64) AS SELECTid,created_at,diff --git a/posthog/clickhouse/kafka_engine.py b/posthog/clickhouse/kafka_engine.pyindex 1d4c1ed7fd15..bf71c1f8748a 100644--- a/posthog/clickhouse/kafka_engine.py+++ b/posthog/clickhouse/kafka_engine.py@@ -68,6 +68,7 @@ CONSUMER_GROUP_ERROR_TRACKING_ISSUE_FINGERPRINT_OVERRIDES_WS = ()CONSUMER_GROUP_ERROR_TRACKING_FINGERPRINT_ISSUE_STATE_WS = "clickhouse_error_tracking_fingerprint_issue_state_ws"CONSUMER_GROUP_USAGE_REPORT_EVENTS_PREAGG = "clickhouse_usage_report_events_preagg"+CONSUMER_GROUP_PERSON_GROUP_MEMBERSHIP = "clickhouse_person_group_membership"STORAGE_POLICY = lambda: "SETTINGS storage_policy = 'hot_to_cold'" if settings.CLICKHOUSE_ENABLE_STORAGE_POLICY else ""diff --git a/posthog/clickhouse/migrations/0369_person_group_membership_ingestion.py b/posthog/clickhouse/migrations/0369_person_group_membership_ingestion.pynew file mode 100644index 000000000000..f6f94b479b6c--- /dev/null+++ b/posthog/clickhouse/migrations/0369_person_group_membership_ingestion.py@@ -0,0 +1,37 @@+from posthog.clickhouse.client.connection import NodeRole+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions+from posthog.models.person_group_membership.sql import (+ DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE_SQL,+ KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE_SQL,+ PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY_SQL,+ PERSON_GROUP_MEMBERSHIP_MV_SQL,+ WRITABLE_PERSON_GROUP_MEMBERSHIP_TABLE_SQL,+)++operations = [+ run_sql_with_exceptions(+ DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE_SQL(),+ node_roles=[NodeRole.INGESTION_EVENTS],+ require_hosts=True,+ ),+ run_sql_with_exceptions(+ PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY_SQL(),+ node_roles=[NodeRole.INGESTION_EVENTS],+ require_hosts=True,+ ),+ run_sql_with_exceptions(+ WRITABLE_PERSON_GROUP_MEMBERSHIP_TABLE_SQL(),+ node_roles=[NodeRole.INGESTION_EVENTS],+ require_hosts=True,+ ),+ run_sql_with_exceptions(+ KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE_SQL(),+ node_roles=[NodeRole.INGESTION_EVENTS],+ require_hosts=True,+ ),+ run_sql_with_exceptions(+ PERSON_GROUP_MEMBERSHIP_MV_SQL(),+ node_roles=[NodeRole.INGESTION_EVENTS],+ require_hosts=True,+ ),+]diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txtindex e0822ce38c94..731ac00dcf4d 100644--- a/posthog/clickhouse/migrations/max_migration.txt+++ b/posthog/clickhouse/migrations/max_migration.txt@@ -1 +1 @@-0368_email_status_events+0369_person_group_membership_ingestiondiff --git a/posthog/models/person_group_membership/sql.py b/posthog/models/person_group_membership/sql.pyindex 0164b71d8dd9..f1b7bb4b906b 100644--- a/posthog/models/person_group_membership/sql.py+++ b/posthog/models/person_group_membership/sql.py@@ -3,11 +3,20 @@ from django.conf import settingsfrom posthog.hogql.escape_sql import escape_clickhouse_identifier, escape_clickhouse_stringfrom posthog.clickhouse.client.connection import ClickHouseUser, get_clickhouse_creds+from posthog.clickhouse.kafka_engine import (+ CONSUMER_GROUP_PERSON_GROUP_MEMBERSHIP,+ kafka_engine,+ kafka_num_consumers,+ trim_quotes_expr,+)from posthog.clickhouse.table_engines import AggregatingMergeTree, Distributed, ReplacingMergeTree, ReplicationScheme+from posthog.kafka_client.topics import KAFKA_EVENTS_JSONPERSON_GROUP_MEMBERSHIP_TABLE = "person_group_membership"SHARDED_PERSON_GROUP_MEMBERSHIP_TABLE = f"sharded_{PERSON_GROUP_MEMBERSHIP_TABLE}"WRITABLE_PERSON_GROUP_MEMBERSHIP_TABLE = f"writable_{PERSON_GROUP_MEMBERSHIP_TABLE}"+KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE = f"kafka_{PERSON_GROUP_MEMBERSHIP_TABLE}"+PERSON_GROUP_MEMBERSHIP_MV = f"{PERSON_GROUP_MEMBERSHIP_TABLE}_mv"PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE = "person_group_membership_config"DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE = f"distributed_{PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE}"@@ -26,6 +35,14 @@ PERSON_GROUP_MEMBERSHIP_COLUMNS = """last_seen SimpleAggregateFunction(max, DateTime64(6, 'UTC'))""".strip()+KAFKA_PERSON_GROUP_MEMBERSHIP_COLUMNS = """+ team_id Int64,+ distinct_id String,+ properties String,+ timestamp DateTime64(6, 'UTC'),+ person_mode Enum8('full' = 0, 'propertyless' = 1, 'force_upgrade' = 2)+""".strip()+PERSON_GROUP_MEMBERSHIP_CONFIG_COLUMNS = """team_id Int64,group_type_index UInt8,@@ -134,3 +151,68 @@ SOURCE(CLICKHOUSE({source}))LIFETIME(MIN 60 MAX {PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY_LIFETIME_MAX_SECONDS})LAYOUT(COMPLEX_KEY_HASHED())"""+++def KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE_SQL() -> str:+ # A dedicated consumer group keeps membership lag and failures off the main events consumer.+ # Each row carries the full properties JSON, so blocks stay at the flag_evaluations sizes.+ return f"""+CREATE TABLE IF NOT EXISTS {KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE}+(+ {KAFKA_PERSON_GROUP_MEMBERSHIP_COLUMNS}+)+ENGINE = {+ kafka_engine(+ topic=KAFKA_EVENTS_JSON,+ group=CONSUMER_GROUP_PERSON_GROUP_MEMBERSHIP,+ named_collection=settings.CLICKHOUSE_KAFKA_WARPSTREAM_INGESTION_NAMED_COLLECTION,+ )+ }+SETTINGS+ kafka_max_block_size = 10000,+ kafka_poll_max_batch_size = 10000,+ kafka_poll_timeout_ms = 10000,+ kafka_flush_interval_ms = 7500,+ kafka_num_consumers = {kafka_num_consumers(1)},+ kafka_thread_per_consumer = 1,+ kafka_skip_broken_messages = 100+"""+++# Matches the events $group_N columns the backfill reads, so numeric keys become strings in both paths.+GROUP_KEY_EXPR = trim_quotes_expr("JSONExtractRaw(properties, concat('$group_', toString(group_type_index)))")+++def PERSON_GROUP_MEMBERSHIP_MV_SELECT_SQL(source_table: str) -> str:+ # Filter on the dictionary before extracting the group key, so ineligible events skip JSON parsing.+ # The dictionary holds only enabled, valid configs, so dictHas fails closed for every other team.+ # dictHas also avoids the inverse dictionary lookup rewrite, which rejects dictGet comparisons on complex keys.+ # Personless events cannot be backed by a person profile. The read path keeps identified persons only.+ dictionary = f"{settings.CLICKHOUSE_DATABASE}.{PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY}"+ return f"""SELECT+ team_id,+ group_type_index,+ {GROUP_KEY_EXPR} AS group_key,+ distinct_id,+ min(timestamp) AS first_seen,+ max(timestamp) AS last_seen+FROM (+ SELECT+ team_id,+ dictGet('{dictionary}', 'group_type_index', tuple(team_id)) AS group_type_index,+ distinct_id,+ properties,+ timestamp+ FROM {source_table}+ WHERE dictHas('{dictionary}', tuple(team_id)) AND (person_mode != 'propertyless')+)+WHERE notEmpty(group_key)+GROUP BY team_id, group_type_index, group_key, distinct_id"""+++def PERSON_GROUP_MEMBERSHIP_MV_SQL() -> str:+ return f"""+CREATE MATERIALIZED VIEW IF NOT EXISTS {PERSON_GROUP_MEMBERSHIP_MV}+TO {WRITABLE_PERSON_GROUP_MEMBERSHIP_TABLE}+AS {PERSON_GROUP_MEMBERSHIP_MV_SELECT_SQL(KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE)}+"""diff --git a/posthog/models/person_group_membership/test/__snapshots__/test_sql.ambr b/posthog/models/person_group_membership/test/__snapshots__/test_sql.ambrindex 26ae415fc6a1..83519754257b 100644--- a/posthog/models/person_group_membership/test/__snapshots__/test_sql.ambr+++ b/posthog/models/person_group_membership/test/__snapshots__/test_sql.ambr@@ -15,3 +15,53 @@'''# ---+# name: test_ingestion_sql[kafka_table]+ '''++ CREATE TABLE IF NOT EXISTS kafka_person_group_membership+ (+ team_id Int64,+ distinct_id String,+ properties String,+ timestamp DateTime64(6, 'UTC'),+ person_mode Enum8('full' = 0, 'propertyless' = 1, 'force_upgrade' = 2)+ )+ ENGINE = Kafka(warpstream_ingestion, kafka_topic_list = 'clickhouse_events_json_test', kafka_group_name = 'clickhouse_person_group_membership', kafka_format = 'JSONEachRow')+ SETTINGS+ kafka_max_block_size = 10000,+ kafka_poll_max_batch_size = 10000,+ kafka_poll_timeout_ms = 10000,+ kafka_flush_interval_ms = 7500,+ kafka_num_consumers = 1,+ kafka_thread_per_consumer = 1,+ kafka_skip_broken_messages = 100++ '''+# ---+# name: test_ingestion_sql[view]+ '''++ CREATE MATERIALIZED VIEW IF NOT EXISTS person_group_membership_mv+ TO writable_person_group_membership+ AS SELECT+ team_id,+ group_type_index,+ replaceRegexpAll(JSONExtractRaw(properties, concat('$group_', toString(group_type_index))), '^"|"$', '') AS group_key,+ distinct_id,+ min(timestamp) AS first_seen,+ max(timestamp) AS last_seen+ FROM (+ SELECT+ team_id,+ dictGet('posthog.person_group_membership_config_dict', 'group_type_index', tuple(team_id)) AS group_type_index,+ distinct_id,+ properties,+ timestamp+ FROM kafka_person_group_membership+ WHERE dictHas('posthog.person_group_membership_config_dict', tuple(team_id)) AND (person_mode != 'propertyless')+ )+ WHERE notEmpty(group_key)+ GROUP BY team_id, group_type_index, group_key, distinct_id++ '''+# ---diff --git a/posthog/models/person_group_membership/test/test_sql.py b/posthog/models/person_group_membership/test/test_sql.pyindex 6a79b226a0e4..cb0a5bee0c8e 100644--- a/posthog/models/person_group_membership/test/test_sql.py+++ b/posthog/models/person_group_membership/test/test_sql.py@@ -1,3 +1,4 @@+import jsonimport importlibfrom collections.abc import Callable, Iteratorfrom datetime import UTC, datetime@@ -17,15 +18,21 @@ from posthog.clickhouse.schema import ()from posthog.models.person_group_membership.sql import (DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE,+ KAFKA_PERSON_GROUP_MEMBERSHIP_COLUMNS,+ KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE_SQL,PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY,PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY_SQL,PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE,+ PERSON_GROUP_MEMBERSHIP_MV_SELECT_SQL,+ PERSON_GROUP_MEMBERSHIP_MV_SQL,PERSON_GROUP_MEMBERSHIP_TABLE,SHARDED_PERSON_GROUP_MEMBERSHIP_TABLE,WRITABLE_PERSON_GROUP_MEMBERSHIP_TABLE,)MIGRATION = "posthog.clickhouse.migrations.0343_person_group_membership"+INGESTION_MIGRATION = "posthog.clickhouse.migrations.0369_person_group_membership_ingestion"+SOURCE_TABLE = "test_source_person_group_membership"def test_dictionary_sql(snapshot, mocker, settings) -> None:@@ -38,30 +45,52 @@ def test_dictionary_sql(snapshot, mocker, settings) -> None:creds.assert_called_once_with(ClickHouseUser.DICT_READER)[email protected]("deployment", ["", "DEV", "US", "EU"])[email protected]("multinode", [False, True])-def test_migration_placement(deployment: str, multinode: bool) -> None:- module = importlib.import_module(MIGRATION)- try:- with patch.multiple("posthog.settings", CLOUD_DEPLOYMENT=deployment, MULTINODE_CLICKHOUSE=multinode):- operations = importlib.reload(module).operations- expected_roles = [+ "build_sql", [KAFKA_PERSON_GROUP_MEMBERSHIP_TABLE_SQL, PERSON_GROUP_MEMBERSHIP_MV_SQL], ids=["kafka_table", "view"]+)+def test_ingestion_sql(build_sql: Callable[[], str], snapshot, settings) -> None:+ settings.CLICKHOUSE_DATABASE = "posthog"+ settings.CLICKHOUSE_KAFKA_WARPSTREAM_INGESTION_NAMED_COLLECTION = "warpstream_ingestion"+ assert build_sql() == snapshot+++ "migration,expected_roles,consumes_kafka",+ [+ (+ MIGRATION,+ [[NodeRole.AUX],[NodeRole.AUX],[NodeRole.DATA, NodeRole.INGESTION_SMALL],[NodeRole.DATA],[NodeRole.INGESTION_SMALL],[NodeRole.INGESTION_SMALL],- ]+ ],+ False,+ ),+ (INGESTION_MIGRATION, [[NodeRole.INGESTION_EVENTS]] * 5, True),+ ],+ ids=["storage", "ingestion"],+)[email protected]("deployment", ["", "DEV", "US", "EU"])[email protected]("multinode", [False, True])+def test_migration_placement(+ migration: str, expected_roles: list[list[NodeRole]], consumes_kafka: bool, deployment: str, multinode: bool+) -> None:+ module = importlib.import_module(migration)+ try:+ with patch.multiple("posthog.settings", CLOUD_DEPLOYMENT=deployment, MULTINODE_CLICKHOUSE=multinode):+ operations = importlib.reload(module).operationsassert [op._node_roles for op in operations] == expected_rolesassert [op._effective_node_roles for op in operations] == (- expected_roles if deployment or multinode else [[NodeRole.ALL]] * 6+ expected_roles if deployment or multinode else [[NodeRole.ALL]] * len(expected_roles))for op in operations:assert "ON CLUSTER" not in op._sqlassert "IF NOT EXISTS" in op._sql- assert "Kafka" not in op._sql- assert "MATERIALIZED VIEW" not in op._sql+ assert any("Kafka" in op._sql for op in operations) == consumes_kafka+ assert any("kafka_events_json_ws" in op._sql for op in operations) is Falsefinally:importlib.reload(module)@@ -214,7 +243,8 @@ def test_distributed_writes_and_reads_aggregate_overlap() -> None:)assert sync_execute(f"""SELECT team_id, group_type_index, group_key, distinct_id, min(first_seen), max(last_seen)- FROM {PERSON_GROUP_MEMBERSHIP_TABLE} GROUP BY team_id, group_type_index, group_key, distinct_id"""+ FROM {PERSON_GROUP_MEMBERSHIP_TABLE} GROUP BY team_id, group_type_index, group_key, distinct_id+ ORDER BY distinct_id""") == [(1001,@@ -254,3 +284,60 @@ def test_storage_supports_distinct_id_and_team_deletion(lightweight: bool) -> No)== expected)++[email protected]("group_type_index", range(5))[email protected]("membership_schema")+def test_ingestion_admits_only_eligible_events(group_type_index: int) -> None:+ sync_execute(+ f"INSERT INTO {DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE} VALUES",+ [(2001, group_type_index, 1, 1), (2002, group_type_index, 1, 1), (2002, group_type_index, 0, 2)],+ settings={"distributed_foreground_insert": 1},+ )+ sync_execute(f"SYSTEM RELOAD DICTIONARY {PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY}")+ every_group = json.dumps({f"$group_{index}": f"account-{index}" for index in range(5)})+ configured_empty = json.dumps({f"$group_{group_type_index}": ""})+ events = [+ (2001, "identified", every_group, datetime(2024, 1, 2, tzinfo=UTC), "full"),+ (2001, "identified", every_group, datetime(2024, 1, 1, tzinfo=UTC), "force_upgrade"),+ (2001, "identified", every_group, datetime(2024, 1, 3, tzinfo=UTC), "full"),+ (2001, "numeric-key", json.dumps({f"$group_{group_type_index}": 42}), datetime(2024, 1, 1, tzinfo=UTC), "full"),+ (2001, "personless", every_group, datetime(2024, 1, 1, tzinfo=UTC), "propertyless"),+ (2001, "empty-key", configured_empty, datetime(2024, 1, 1, tzinfo=UTC), "full"),+ (2001, "no-properties", "", datetime(2024, 1, 1, tzinfo=UTC), "full"),+ (2002, "disabled-team", every_group, datetime(2024, 1, 1, tzinfo=UTC), "full"),+ (2003, "missing-team", every_group, datetime(2024, 1, 1, tzinfo=UTC), "full"),+ ]+ sync_execute(f"DROP TABLE IF EXISTS {SOURCE_TABLE}")+ sync_execute(f"CREATE TABLE {SOURCE_TABLE} ({KAFKA_PERSON_GROUP_MEMBERSHIP_COLUMNS}) ENGINE = Memory")+ try:+ sync_execute(f"INSERT INTO {SOURCE_TABLE} VALUES", events)+ for _ in range(2):+ sync_execute(+ f"INSERT INTO {WRITABLE_PERSON_GROUP_MEMBERSHIP_TABLE} {PERSON_GROUP_MEMBERSHIP_MV_SELECT_SQL(SOURCE_TABLE)}",+ settings={"distributed_foreground_insert": 1},+ )+ finally:+ sync_execute(f"DROP TABLE IF EXISTS {SOURCE_TABLE}")+ assert sync_execute(+ f"""SELECT team_id, group_type_index, group_key, distinct_id, min(first_seen), max(last_seen)+ FROM {PERSON_GROUP_MEMBERSHIP_TABLE} GROUP BY team_id, group_type_index, group_key, distinct_id+ ORDER BY distinct_id"""+ ) == [+ (+ 2001,+ group_type_index,+ f"account-{group_type_index}",+ "identified",+ datetime(2024, 1, 1, tzinfo=UTC),+ datetime(2024, 1, 3, tzinfo=UTC),+ ),+ (+ 2001,+ group_type_index,+ "42",+ "numeric-key",+ datetime(2024, 1, 1, tzinfo=UTC),+ datetime(2024, 1, 1, tzinfo=UTC),+ ),+ ]diff --git a/products/customer_analytics/backend/logic/membership_backfill.py b/products/customer_analytics/backend/logic/membership_backfill.pyindex aeee6acdeadf..cbf4f03caccf 100644--- a/products/customer_analytics/backend/logic/membership_backfill.py+++ b/products/customer_analytics/backend/logic/membership_backfill.py@@ -788,7 +788,7 @@ class MembershipBackfillService:results = (get_cluster(cluster=settings.CLICKHOUSE_MIGRATIONS_CLUSTER, data_cluster=settings.CLICKHOUSE_CLUSTER)- .map_hosts_by_roles(check, [NodeRole.INGESTION_SMALL], concurrency=2, require_hosts=True)+ .map_hosts_by_roles(check, [NodeRole.INGESTION_EVENTS], concurrency=2, require_hosts=True).result())self.check_attempt()