:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(clickhouse): run logs kafka ingest on ingestion-apm in prod (#122283)
Bryan Ciaraldi3c382acdd32d
diff --git a/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl b/posthog/clickhouse/hcl/golden/prod-eu/apm.hclindex 197ea16b9716..4341625ce848 100644--- a/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl+++ b/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl@@ -1,4 +1,72 @@database "posthog" {+ table "kafka_logs_avro" {+ settings = {+ input_format_avro_allow_missing_fields = "1"+ }+ column "uuid" {+ type = "String"+ }+ column "trace_id" {+ type = "String"+ }+ column "span_id" {+ type = "String"+ }+ column "trace_flags" {+ type = "Int32"+ }+ column "timestamp" {+ type = "DateTime64(6)"+ }+ column "observed_timestamp" {+ type = "DateTime64(6)"+ }+ column "body" {+ type = "String"+ }+ column "severity_text" {+ type = "String"+ }+ column "severity_number" {+ type = "Int32"+ }+ column "service_name" {+ type = "String"+ }+ column "resource_attributes" {+ type = "Map(LowCardinality(String), String)"+ }+ column "instrumentation_scope" {+ type = "String"+ }+ column "event_name" {+ type = "String"+ }+ column "attributes" {+ type = "Map(LowCardinality(String), String)"+ }+ column "retention_days" {+ type = "Nullable(Int32)"+ }+ column "pattern" {+ type = "Nullable(String)"+ }+ column "pattern_version" {+ type = "Nullable(Int32)"+ }+ engine "kafka" {+ collection = "warpstream_logs"+ topic_list = "clickhouse_logs"+ group_name = "clickhouse-logs-avro-new"+ format = "Avro"+ num_consumers = 4+ skip_broken_messages = 100+ poll_timeout_ms = 3000+ poll_max_batch_size = 1000+ thread_per_consumer = true+ }+ }+table "kafka_metrics_avro4" {settings = {input_format_avro_allow_missing_fields = "1"@@ -170,6 +238,126 @@ database "posthog" {}}+ table "writable_logs34" {+ settings = {+ background_insert_batch = "1"+ }+ column "time_bucket" {+ type = "DateTime"+ materialized = "toStartOfDay(timestamp)"+ }+ column "original_expiry_timestamp" {+ type = "DateTime64(6)"+ }+ column "uuid" {+ type = "String"+ }+ column "team_id" {+ type = "Int32"+ }+ column "trace_id" {+ type = "String"+ }+ column "span_id" {+ type = "String"+ }+ column "trace_flags" {+ type = "Int32"+ }+ column "timestamp" {+ type = "DateTime64(6)"+ codec = "DoubleDelta"+ }+ column "observed_timestamp" {+ type = "DateTime64(6)"+ }+ column "created_at" {+ type = "DateTime64(6)"+ materialized = "now()"+ }+ column "body" {+ type = "String"+ }+ column "severity_text" {+ type = "LowCardinality(String)"+ }+ column "severity_number" {+ type = "Int32"+ }+ column "service_name" {+ type = "LowCardinality(String)"+ }+ column "resource_attributes" {+ type = "Map(LowCardinality(String), String)"+ }+ column "resource_fingerprint" {+ type = "UInt64"+ materialized = "cityHash64(resource_attributes)"+ }+ column "instrumentation_scope" {+ type = "String"+ }+ column "event_name" {+ type = "String"+ }+ column "attributes_map_str" {+ type = "Map(LowCardinality(String), String)"+ }+ column "level" {+ type = "String"+ alias = "severity_text"+ }+ column "mat_body_ipv4_matches" {+ type = "Array(String)"+ alias = "extractAll(body, '(\\\\d\\\\.((25[0-5]|(2[0-4]|1(0, 1)[0-9])(0, 1)[0-9])\\\\.)(2, 2)([0-9]))')"+ }+ column "time_minute" {+ type = "DateTime"+ alias = "toStartOfMinute(timestamp)"+ }+ column "attributes" {+ type = "Map(LowCardinality(String), String)"+ alias = "mapApply((k, v) -> (left(k, -5), v), attributes_map_str)"+ }+ column "attributes_map_float" {+ type = "Map(LowCardinality(String), Float64)"+ materialized = "mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__float'), toFloat64OrNull(v)), attributes_map_str))"+ }+ column "attributes_map_datetime" {+ type = "Map(LowCardinality(String), DateTime64(6))"+ materialized = "mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__datetime'), parseDateTimeBestEffortOrNull(v, 6)), attributes_map_str))"+ }+ column "_partition" {+ type = "UInt32"+ }+ column "_topic" {+ type = "String"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_bytes_uncompressed" {+ type = "UInt64"+ }+ column "_bytes_compressed" {+ type = "UInt64"+ }+ column "_record_count" {+ type = "UInt64"+ }+ column "pattern" {+ type = "String"+ }+ column "pattern_version" {+ type = "UInt8"+ }+ engine "distributed" {+ cluster_name = "logs"+ remote_database = "posthog"+ remote_table = "logs34"+ }+ }+table "writable_metrics4_attributes" {column "team_id" {type = "Int32"@@ -723,6 +911,118 @@ database "posthog" {}}+ materialized_view "kafka_logs34_avro_mv" {+ to_table = "posthog.writable_logs34"+ query = <<SQL+SELECT+ uuid,+ trace_id,+ span_id,+ trace_flags,+ timestamp,+ observed_timestamp,+ body,+ severity_text,+ severity_number,+ service_name,+ instrumentation_scope,+ event_name,+ mapSort(mapApply((k, v) -> (concat(k, '__str'), JSONExtractString(v)), attributes)) AS attributes_map_str,+ mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), resource_attributes)) AS resource_attributes,+ toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,+ timestamp+ + toIntervalDay(+ if(+ (retention_days IS NOT NULL) AND (retention_days > 0),+ retention_days,+ toInt32OrDefault(_headers.value[indexOf(_headers.name, 'retention-days')], toInt32(15))+ )+ ) AS original_expiry_timestamp,+ _partition,+ _topic,+ _offset,+ toInt64OrDefault(_headers.value[indexOf(_headers.name, 'record_count')], toInt64(1)) AS _record_count,+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_uncompressed')]) / _record_count AS _bytes_uncompressed,+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_compressed')]) / _record_count AS _bytes_compressed,+ ifNull(pattern, '') AS pattern,+ toUInt8(ifNull(pattern_version, 0)) AS pattern_version+FROM posthog.kafka_logs_avro+SQL++ column "uuid" {+ type = "String"+ }+ column "trace_id" {+ type = "String"+ }+ column "span_id" {+ type = "String"+ }+ column "trace_flags" {+ type = "Int32"+ }+ column "timestamp" {+ type = "DateTime64(6)"+ }+ column "observed_timestamp" {+ type = "DateTime64(6)"+ }+ column "body" {+ type = "String"+ }+ column "severity_text" {+ type = "String"+ }+ column "severity_number" {+ type = "Int32"+ }+ column "service_name" {+ type = "String"+ }+ column "instrumentation_scope" {+ type = "String"+ }+ column "event_name" {+ type = "String"+ }+ column "attributes_map_str" {+ type = "Map(String, String)"+ }+ column "resource_attributes" {+ type = "Map(String, String)"+ }+ column "team_id" {+ type = "Int32"+ }+ column "original_expiry_timestamp" {+ type = "DateTime64(6)"+ }+ column "_partition" {+ type = "UInt64"+ }+ column "_topic" {+ type = "LowCardinality(String)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_record_count" {+ type = "Int64"+ }+ column "_bytes_uncompressed" {+ type = "Nullable(Int64)"+ }+ column "_bytes_compressed" {+ type = "Nullable(Int64)"+ }+ column "pattern" {+ type = "String"+ }+ column "pattern_version" {+ type = "UInt8"+ }+ }+materialized_view "kafka_metrics_avro4_mv" {to_table = "posthog.metrics4_input"query = <<SQLdiff --git a/posthog/clickhouse/hcl/golden/prod-us/apm.hcl b/posthog/clickhouse/hcl/golden/prod-us/apm.hclindex 197ea16b9716..4341625ce848 100644--- a/posthog/clickhouse/hcl/golden/prod-us/apm.hcl+++ b/posthog/clickhouse/hcl/golden/prod-us/apm.hcl@@ -1,4 +1,72 @@database "posthog" {+ table "kafka_logs_avro" {+ settings = {+ input_format_avro_allow_missing_fields = "1"+ }+ column "uuid" {+ type = "String"+ }+ column "trace_id" {+ type = "String"+ }+ column "span_id" {+ type = "String"+ }+ column "trace_flags" {+ type = "Int32"+ }+ column "timestamp" {+ type = "DateTime64(6)"+ }+ column "observed_timestamp" {+ type = "DateTime64(6)"+ }+ column "body" {+ type = "String"+ }+ column "severity_text" {+ type = "String"+ }+ column "severity_number" {+ type = "Int32"+ }+ column "service_name" {+ type = "String"+ }+ column "resource_attributes" {+ type = "Map(LowCardinality(String), String)"+ }+ column "instrumentation_scope" {+ type = "String"+ }+ column "event_name" {+ type = "String"+ }+ column "attributes" {+ type = "Map(LowCardinality(String), String)"+ }+ column "retention_days" {+ type = "Nullable(Int32)"+ }+ column "pattern" {+ type = "Nullable(String)"+ }+ column "pattern_version" {+ type = "Nullable(Int32)"+ }+ engine "kafka" {+ collection = "warpstream_logs"+ topic_list = "clickhouse_logs"+ group_name = "clickhouse-logs-avro-new"+ format = "Avro"+ num_consumers = 4+ skip_broken_messages = 100+ poll_timeout_ms = 3000+ poll_max_batch_size = 1000+ thread_per_consumer = true+ }+ }+table "kafka_metrics_avro4" {settings = {input_format_avro_allow_missing_fields = "1"@@ -170,6 +238,126 @@ database "posthog" {}}+ table "writable_logs34" {+ settings = {+ background_insert_batch = "1"+ }+ column "time_bucket" {+ type = "DateTime"+ materialized = "toStartOfDay(timestamp)"+ }+ column "original_expiry_timestamp" {+ type = "DateTime64(6)"+ }+ column "uuid" {+ type = "String"+ }+ column "team_id" {+ type = "Int32"+ }+ column "trace_id" {+ type = "String"+ }+ column "span_id" {+ type = "String"+ }+ column "trace_flags" {+ type = "Int32"+ }+ column "timestamp" {+ type = "DateTime64(6)"+ codec = "DoubleDelta"+ }+ column "observed_timestamp" {+ type = "DateTime64(6)"+ }+ column "created_at" {+ type = "DateTime64(6)"+ materialized = "now()"+ }+ column "body" {+ type = "String"+ }+ column "severity_text" {+ type = "LowCardinality(String)"+ }+ column "severity_number" {+ type = "Int32"+ }+ column "service_name" {+ type = "LowCardinality(String)"+ }+ column "resource_attributes" {+ type = "Map(LowCardinality(String), String)"+ }+ column "resource_fingerprint" {+ type = "UInt64"+ materialized = "cityHash64(resource_attributes)"+ }+ column "instrumentation_scope" {+ type = "String"+ }+ column "event_name" {+ type = "String"+ }+ column "attributes_map_str" {+ type = "Map(LowCardinality(String), String)"+ }+ column "level" {+ type = "String"+ alias = "severity_text"+ }+ column "mat_body_ipv4_matches" {+ type = "Array(String)"+ alias = "extractAll(body, '(\\\\d\\\\.((25[0-5]|(2[0-4]|1(0, 1)[0-9])(0, 1)[0-9])\\\\.)(2, 2)([0-9]))')"+ }+ column "time_minute" {+ type = "DateTime"+ alias = "toStartOfMinute(timestamp)"+ }+ column "attributes" {+ type = "Map(LowCardinality(String), String)"+ alias = "mapApply((k, v) -> (left(k, -5), v), attributes_map_str)"+ }+ column "attributes_map_float" {+ type = "Map(LowCardinality(String), Float64)"+ materialized = "mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__float'), toFloat64OrNull(v)), attributes_map_str))"+ }+ column "attributes_map_datetime" {+ type = "Map(LowCardinality(String), DateTime64(6))"+ materialized = "mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__datetime'), parseDateTimeBestEffortOrNull(v, 6)), attributes_map_str))"+ }+ column "_partition" {+ type = "UInt32"+ }+ column "_topic" {+ type = "String"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_bytes_uncompressed" {+ type = "UInt64"+ }+ column "_bytes_compressed" {+ type = "UInt64"+ }+ column "_record_count" {+ type = "UInt64"+ }+ column "pattern" {+ type = "String"+ }+ column "pattern_version" {+ type = "UInt8"+ }+ engine "distributed" {+ cluster_name = "logs"+ remote_database = "posthog"+ remote_table = "logs34"+ }+ }+table "writable_metrics4_attributes" {column "team_id" {type = "Int32"@@ -723,6 +911,118 @@ database "posthog" {}}+ materialized_view "kafka_logs34_avro_mv" {+ to_table = "posthog.writable_logs34"+ query = <<SQL+SELECT+ uuid,+ trace_id,+ span_id,+ trace_flags,+ timestamp,+ observed_timestamp,+ body,+ severity_text,+ severity_number,+ service_name,+ instrumentation_scope,+ event_name,+ mapSort(mapApply((k, v) -> (concat(k, '__str'), JSONExtractString(v)), attributes)) AS attributes_map_str,+ mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), resource_attributes)) AS resource_attributes,+ toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,+ timestamp+ + toIntervalDay(+ if(+ (retention_days IS NOT NULL) AND (retention_days > 0),+ retention_days,+ toInt32OrDefault(_headers.value[indexOf(_headers.name, 'retention-days')], toInt32(15))+ )+ ) AS original_expiry_timestamp,+ _partition,+ _topic,+ _offset,+ toInt64OrDefault(_headers.value[indexOf(_headers.name, 'record_count')], toInt64(1)) AS _record_count,+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_uncompressed')]) / _record_count AS _bytes_uncompressed,+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_compressed')]) / _record_count AS _bytes_compressed,+ ifNull(pattern, '') AS pattern,+ toUInt8(ifNull(pattern_version, 0)) AS pattern_version+FROM posthog.kafka_logs_avro+SQL++ column "uuid" {+ type = "String"+ }+ column "trace_id" {+ type = "String"+ }+ column "span_id" {+ type = "String"+ }+ column "trace_flags" {+ type = "Int32"+ }+ column "timestamp" {+ type = "DateTime64(6)"+ }+ column "observed_timestamp" {+ type = "DateTime64(6)"+ }+ column "body" {+ type = "String"+ }+ column "severity_text" {+ type = "String"+ }+ column "severity_number" {+ type = "Int32"+ }+ column "service_name" {+ type = "String"+ }+ column "instrumentation_scope" {+ type = "String"+ }+ column "event_name" {+ type = "String"+ }+ column "attributes_map_str" {+ type = "Map(String, String)"+ }+ column "resource_attributes" {+ type = "Map(String, String)"+ }+ column "team_id" {+ type = "Int32"+ }+ column "original_expiry_timestamp" {+ type = "DateTime64(6)"+ }+ column "_partition" {+ type = "UInt64"+ }+ column "_topic" {+ type = "LowCardinality(String)"+ }+ column "_offset" {+ type = "UInt64"+ }+ column "_record_count" {+ type = "Int64"+ }+ column "_bytes_uncompressed" {+ type = "Nullable(Int64)"+ }+ column "_bytes_compressed" {+ type = "Nullable(Int64)"+ }+ column "pattern" {+ type = "String"+ }+ column "pattern_version" {+ type = "UInt8"+ }+ }+materialized_view "kafka_metrics_avro4_mv" {to_table = "posthog.metrics4_input"query = <<SQLdiff --git a/posthog/clickhouse/hcl/manifest.hcl b/posthog/clickhouse/hcl/manifest.hclindex 34e5a1a1a1c6..581c9249a671 100644--- a/posthog/clickhouse/hcl/manifest.hcl+++ b/posthog/clickhouse/hcl/manifest.hcl@@ -44,12 +44,12 @@ role "logs" {# APM satellite: the ingestion-apm nodes. Every environment runs the shared# custom_metrics suite and writes to the query log archive. Every environment also-# runs the metrics4 ingest chain. Development runs the legacy APM ingest chain too.-# The production logs nodes still run that legacy chain.+# runs the metrics4 ingest chain and the logs Kafka ingest. Development runs the+# legacy APM ingest chain too. The production logs nodes still run that legacy chain.role "apm" {env "dev" { layers = ["roles/shared/qla_write.hcl", "roles/coshared/named_collections", "roles/coshared/custom_metrics", "roles/coshared/apm_ingest", "roles/coshared/logs_avro_ingest", "roles/apm/shared", "roles/apm/dev"] }- env "prod-us" { layers = ["roles/shared/qla_write.hcl", "roles/coshared/named_collections", "roles/coshared/custom_metrics", "roles/apm/shared"] }- env "prod-eu" { layers = ["roles/shared/qla_write.hcl", "roles/coshared/named_collections", "roles/coshared/custom_metrics", "roles/apm/shared"] }+ env "prod-us" { layers = ["roles/shared/qla_write.hcl", "roles/coshared/named_collections", "roles/coshared/custom_metrics", "roles/coshared/logs_avro_ingest", "roles/apm/shared"] }+ env "prod-eu" { layers = ["roles/shared/qla_write.hcl", "roles/coshared/named_collections", "roles/coshared/custom_metrics", "roles/coshared/logs_avro_ingest", "roles/apm/shared"] }}# AI_EVENTS satellite (LLM analytics). local/hobby run the MSK variantdiff --git a/posthog/clickhouse/hcl/roles/coshared/logs_avro_ingest/tables.hcl b/posthog/clickhouse/hcl/roles/coshared/logs_avro_ingest/tables.hclindex 2eb8d14faf71..8dcda1fa9318 100644--- a/posthog/clickhouse/hcl/roles/coshared/logs_avro_ingest/tables.hcl+++ b/posthog/clickhouse/hcl/roles/coshared/logs_avro_ingest/tables.hcl@@ -1,5 +1,5 @@-# The Kafka ingest for logs. dev runs this on its ingestion-apm nodes, the prod-# clusters on their ingestion-events nodes.+# The Kafka ingest for logs. The cloud envs run it on their ingestion-apm nodes,+# the multinode stack on its ingestion-events node.database "posthog" {table "kafka_logs_avro" {column "uuid" {@@ -58,7 +58,7 @@ database "posthog" {topic_list = "clickhouse_logs"group_name = "clickhouse-logs-avro-new"format = "Avro"- num_consumers = 8+ num_consumers = 4skip_broken_messages = 100poll_timeout_ms = 3000poll_max_batch_size = 1000diff --git a/posthog/clickhouse/hcl/sql/prod-eu/apm.sql b/posthog/clickhouse/hcl/sql/prod-eu/apm.sqlindex 116fb599fbb3..d68e4b981af1 100644--- a/posthog/clickhouse/hcl/sql/prod-eu/apm.sql+++ b/posthog/clickhouse/hcl/sql/prod-eu/apm.sql@@ -1,6 +1,25 @@-- AUTO-GENERATED from the declarative HCL by ops/gen-sql.sh — do not edit.-- Full CREATE schema for the prod-eu/apm node. Apply to a fresh ClickHouse to build it.+CREATE TABLE posthog.kafka_logs_avro (+ uuid String,+ trace_id String,+ span_id String,+ trace_flags Int32,+ timestamp DateTime64(6),+ observed_timestamp DateTime64(6),+ body String,+ severity_text String,+ severity_number Int32,+ service_name String,+ resource_attributes Map(LowCardinality(String), String),+ instrumentation_scope String,+ event_name String,+ attributes Map(LowCardinality(String), String),+ retention_days Nullable(Int32),+ pattern Nullable(String),+ pattern_version Nullable(Int32)+) ENGINE = Kafka(warpstream_logs) SETTINGS input_format_avro_allow_missing_fields = 1, kafka_format = 'Avro', kafka_group_name = 'clickhouse-logs-avro-new', kafka_num_consumers = 4, kafka_poll_max_batch_size = 1000, kafka_poll_timeout_ms = 3000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_logs';CREATE TABLE posthog.kafka_metrics_avro4 (uuid String,trace_id String,@@ -54,6 +73,41 @@ CREATE TABLE posthog.metrics4_input (_topic String,_offset UInt64) ENGINE = Null();+CREATE TABLE posthog.writable_logs34 (+ time_bucket DateTime MATERIALIZED toStartOfDay(timestamp),+ original_expiry_timestamp DateTime64(6),+ uuid String,+ team_id Int32,+ trace_id String,+ span_id String,+ trace_flags Int32,+ timestamp DateTime64(6) CODEC(DoubleDelta),+ observed_timestamp DateTime64(6),+ created_at DateTime64(6) MATERIALIZED now(),+ body String,+ severity_text LowCardinality(String),+ severity_number Int32,+ service_name LowCardinality(String),+ resource_attributes Map(LowCardinality(String), String),+ resource_fingerprint UInt64 MATERIALIZED cityHash64(resource_attributes),+ instrumentation_scope String,+ event_name String,+ attributes_map_str Map(LowCardinality(String), String),+ level String ALIAS severity_text,+ mat_body_ipv4_matches Array(String) ALIAS extractAll(body, '(\\d\\.((25[0-5]|(2[0-4]|1(0, 1)[0-9])(0, 1)[0-9])\\.)(2, 2)([0-9]))'),+ time_minute DateTime ALIAS toStartOfMinute(timestamp),+ attributes Map(LowCardinality(String), String) ALIAS mapApply((k, v) -> (left(k, -5), v), attributes_map_str),+ attributes_map_float Map(LowCardinality(String), Float64) MATERIALIZED mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__float'), toFloat64OrNull(v)), attributes_map_str)),+ attributes_map_datetime Map(LowCardinality(String), DateTime64(6)) MATERIALIZED mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__datetime'), parseDateTimeBestEffortOrNull(v, 6)), attributes_map_str)),+ _partition UInt32,+ _topic String,+ _offset UInt64,+ _bytes_uncompressed UInt64,+ _bytes_compressed UInt64,+ _record_count UInt64,+ pattern String,+ pattern_version UInt8+) ENGINE = Distributed('logs', 'posthog', 'logs34') SETTINGS background_insert_batch = 1;CREATE TABLE posthog.writable_metrics4_attributes (team_id Int32,metric_name LowCardinality(String),@@ -234,6 +288,39 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (lc_estimated_rows SimpleAggregateFunction(max, Int64),lc_estimated_bytes SimpleAggregateFunction(max, Int64)) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');+CREATE MATERIALIZED VIEW posthog.kafka_logs34_avro_mv TO posthog.writable_logs34 (uuid String, trace_id String, span_id String, trace_flags Int32, timestamp DateTime64(6), observed_timestamp DateTime64(6), body String, severity_text String, severity_number Int32, service_name String, instrumentation_scope String, event_name String, attributes_map_str Map(String, String), resource_attributes Map(String, String), team_id Int32, original_expiry_timestamp DateTime64(6), _partition UInt64, _topic LowCardinality(String), _offset UInt64, _record_count Int64, _bytes_uncompressed Nullable(Int64), _bytes_compressed Nullable(Int64), pattern String, pattern_version UInt8) AS SELECT+ uuid,+ trace_id,+ span_id,+ trace_flags,+ timestamp,+ observed_timestamp,+ body,+ severity_text,+ severity_number,+ service_name,+ instrumentation_scope,+ event_name,+ mapSort(mapApply((k, v) -> (concat(k, '__str'), JSONExtractString(v)), attributes)) AS attributes_map_str,+ mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), resource_attributes)) AS resource_attributes,+ toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,+ timestamp+ + toIntervalDay(+ if(+ (retention_days IS NOT NULL) AND (retention_days > 0),+ retention_days,+ toInt32OrDefault(_headers.value[indexOf(_headers.name, 'retention-days')], toInt32(15))+ )+ ) AS original_expiry_timestamp,+ _partition,+ _topic,+ _offset,+ toInt64OrDefault(_headers.value[indexOf(_headers.name, 'record_count')], toInt64(1)) AS _record_count,+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_uncompressed')]) / _record_count AS _bytes_uncompressed,+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_compressed')]) / _record_count AS _bytes_compressed,+ ifNull(pattern, '') AS pattern,+ toUInt8(ifNull(pattern_version, 0)) AS pattern_version+FROM posthog.kafka_logs_avro;CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64) AS SELECTuuid,toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,diff --git a/posthog/clickhouse/hcl/sql/prod-us/apm.sql b/posthog/clickhouse/hcl/sql/prod-us/apm.sqlindex 08c12fa44e5f..3d0aef40c59b 100644--- a/posthog/clickhouse/hcl/sql/prod-us/apm.sql+++ b/posthog/clickhouse/hcl/sql/prod-us/apm.sql@@ -1,6 +1,25 @@-- AUTO-GENERATED from the declarative HCL by ops/gen-sql.sh — do not edit.-- Full CREATE schema for the prod-us/apm node. Apply to a fresh ClickHouse to build it.+CREATE TABLE posthog.kafka_logs_avro (+ uuid String,+ trace_id String,+ span_id String,+ trace_flags Int32,+ timestamp DateTime64(6),+ observed_timestamp DateTime64(6),+ body String,+ severity_text String,+ severity_number Int32,+ service_name String,+ resource_attributes Map(LowCardinality(String), String),+ instrumentation_scope String,+ event_name String,+ attributes Map(LowCardinality(String), String),+ retention_days Nullable(Int32),+ pattern Nullable(String),+ pattern_version Nullable(Int32)+) ENGINE = Kafka(warpstream_logs) SETTINGS input_format_avro_allow_missing_fields = 1, kafka_format = 'Avro', kafka_group_name = 'clickhouse-logs-avro-new', kafka_num_consumers = 4, kafka_poll_max_batch_size = 1000, kafka_poll_timeout_ms = 3000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_logs';CREATE TABLE posthog.kafka_metrics_avro4 (uuid String,trace_id String,@@ -54,6 +73,41 @@ CREATE TABLE posthog.metrics4_input (_topic String,_offset UInt64) ENGINE = Null();+CREATE TABLE posthog.writable_logs34 (+ time_bucket DateTime MATERIALIZED toStartOfDay(timestamp),+ original_expiry_timestamp DateTime64(6),+ uuid String,+ team_id Int32,+ trace_id String,+ span_id String,+ trace_flags Int32,+ timestamp DateTime64(6) CODEC(DoubleDelta),+ observed_timestamp DateTime64(6),+ created_at DateTime64(6) MATERIALIZED now(),+ body String,+ severity_text LowCardinality(String),+ severity_number Int32,+ service_name LowCardinality(String),+ resource_attributes Map(LowCardinality(String), String),+ resource_fingerprint UInt64 MATERIALIZED cityHash64(resource_attributes),+ instrumentation_scope String,+ event_name String,+ attributes_map_str Map(LowCardinality(String), String),+ level String ALIAS severity_text,+ mat_body_ipv4_matches Array(String) ALIAS extractAll(body, '(\\d\\.((25[0-5]|(2[0-4]|1(0, 1)[0-9])(0, 1)[0-9])\\.)(2, 2)([0-9]))'),+ time_minute DateTime ALIAS toStartOfMinute(timestamp),+ attributes Map(LowCardinality(String), String) ALIAS mapApply((k, v) -> (left(k, -5), v), attributes_map_str),+ attributes_map_float Map(LowCardinality(String), Float64) MATERIALIZED mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__float'), toFloat64OrNull(v)), attributes_map_str)),+ attributes_map_datetime Map(LowCardinality(String), DateTime64(6)) MATERIALIZED mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__datetime'), parseDateTimeBestEffortOrNull(v, 6)), attributes_map_str)),+ _partition UInt32,+ _topic String,+ _offset UInt64,+ _bytes_uncompressed UInt64,+ _bytes_compressed UInt64,+ _record_count UInt64,+ pattern String,+ pattern_version UInt8+) ENGINE = Distributed('logs', 'posthog', 'logs34') SETTINGS background_insert_batch = 1;CREATE TABLE posthog.writable_metrics4_attributes (team_id Int32,metric_name LowCardinality(String),@@ -234,6 +288,39 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (lc_estimated_rows SimpleAggregateFunction(max, Int64),lc_estimated_bytes SimpleAggregateFunction(max, Int64)) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');+CREATE MATERIALIZED VIEW posthog.kafka_logs34_avro_mv TO posthog.writable_logs34 (uuid String, trace_id String, span_id String, trace_flags Int32, timestamp DateTime64(6), observed_timestamp DateTime64(6), body String, severity_text String, severity_number Int32, service_name String, instrumentation_scope String, event_name String, attributes_map_str Map(String, String), resource_attributes Map(String, String), team_id Int32, original_expiry_timestamp DateTime64(6), _partition UInt64, _topic LowCardinality(String), _offset UInt64, _record_count Int64, _bytes_uncompressed Nullable(Int64), _bytes_compressed Nullable(Int64), pattern String, pattern_version UInt8) AS SELECT+ uuid,+ trace_id,+ span_id,+ trace_flags,+ timestamp,+ observed_timestamp,+ body,+ severity_text,+ severity_number,+ service_name,+ instrumentation_scope,+ event_name,+ mapSort(mapApply((k, v) -> (concat(k, '__str'), JSONExtractString(v)), attributes)) AS attributes_map_str,+ mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), resource_attributes)) AS resource_attributes,+ toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,+ timestamp+ + toIntervalDay(+ if(+ (retention_days IS NOT NULL) AND (retention_days > 0),+ retention_days,+ toInt32OrDefault(_headers.value[indexOf(_headers.name, 'retention-days')], toInt32(15))+ )+ ) AS original_expiry_timestamp,+ _partition,+ _topic,+ _offset,+ toInt64OrDefault(_headers.value[indexOf(_headers.name, 'record_count')], toInt64(1)) AS _record_count,+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_uncompressed')]) / _record_count AS _bytes_uncompressed,+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_compressed')]) / _record_count AS _bytes_compressed,+ ifNull(pattern, '') AS pattern,+ toUInt8(ifNull(pattern_version, 0)) AS pattern_version+FROM posthog.kafka_logs_avro;CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64) AS SELECTuuid,toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,diff --git a/posthog/clickhouse/logs/logs34.py b/posthog/clickhouse/logs/logs34.pyindex 738e46500586..9ed815e7f0e2 100644--- a/posthog/clickhouse/logs/logs34.py+++ b/posthog/clickhouse/logs/logs34.py@@ -105,11 +105,11 @@ CREATE OR REPLACE TABLE {database}.logs_distributed AS {database}.{table_name} Edef WRITABLE_LOGS34_TABLE_SQL():- # The write path for the Kafka MV, hosted on the ingestion-events nodes: a Distributed table over+ # The write path for the Kafka MV, hosted on the ingestion nodes: a Distributed table over# the logs cluster's `logs34`. The columns are spelled out (rather than `AS logs34`) because- # `logs34` is not local on the events nodes — only this Distributed front is. Keep the column list+ # `logs34` is not local on the ingestion nodes — only this Distributed front is. Keep the column list# byte-identical to `LOGS34_TABLE_SQL` (indexes/projection are storage-only and omitted here).- # `CLICKHOUSE_LOGS_WRITE_CLUSTER`, not `CLICKHOUSE_LOGS_CLUSTER`: the events node reaches the logs+ # `CLICKHOUSE_LOGS_WRITE_CLUSTER`, not `CLICKHOUSE_LOGS_CLUSTER`: the ingestion node reaches the logs# cluster under its own name, and the latter's default resolves to the data node from there.db = settings.CLICKHOUSE_LOGS_CLUSTER_DATABASEreturn f"""@@ -286,7 +286,7 @@ CREATE TABLE IF NOT EXISTS {settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE}.{KAFKA_TAENGINE = {kafka_engine(topic=KAFKA_TOPIC, group=KAFKA_GROUP, serialization="Avro", named_collection=KAFKA_NAMED_COLLECTION)}SETTINGSkafka_skip_broken_messages = 100,- kafka_num_consumers = {kafka_num_consumers(8)},+ kafka_num_consumers = {kafka_num_consumers(4)},kafka_poll_timeout_ms = 3000,kafka_poll_max_batch_size = 1000,kafka_thread_per_consumer = 1,@@ -327,8 +327,8 @@ FROM {db}.{KAFKA_TABLE_NAME}"""def KAFKA_LOGS34_AVRO_MV(to_table: str = TABLE_NAME):db = settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE# `to_table` defaults to `logs34` so the historical migration 0280 (which creates this MV on the- # logs nodes, where `logs34` is local) keeps working when replayed. Migration 0305 passes- # `writable_logs34` — the Distributed(logs, posthog, logs34) table on the ingestion-events nodes —+ # logs nodes, where `logs34` is local) keeps working when replayed. Migrations 0305 and 0359 pass+ # `writable_logs34` — the Distributed(logs, posthog, logs34) table on the ingestion nodes —# because `logs34` itself is not local there.return f"""CREATE MATERIALIZED VIEW IF NOT EXISTS {db}.kafka_logs34_avro_mv TO {db}.{to_table}diff --git a/posthog/clickhouse/migrations/0359_logs_ingest_to_apm.py b/posthog/clickhouse/migrations/0359_logs_ingest_to_apm.pynew file mode 100644index 000000000000..0c075b5a0ef9--- /dev/null+++ b/posthog/clickhouse/migrations/0359_logs_ingest_to_apm.py@@ -0,0 +1,42 @@+from django.conf import settings++from posthog.clickhouse.client.connection import NodeRole+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions+from posthog.clickhouse.logs import KAFKA_LOGS34_AVRO_MV, KAFKA_LOGS_AVRO_TABLE_SQL, WRITABLE_LOGS34_TABLE_SQL+from posthog.run_mode import run_mode++DB = settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE++# The cloud envs run the logs Kafka ingest on the ingestion-apm nodes, as the HCL `apm` role composes+# `coshared/logs_avro_ingest` for dev, prod-us and prod-eu. 0305, 0307 and 0348 put it on the+# ingestion-events nodes in prod, so a cluster built from the migration history would consume there.+# On a cluster that already runs the ingest on apm, every statement is a no-op.+#+# The apm side is created first. Both sides use the same `kafka_group_name`, so the apm consumers+# join the group and take partitions before the ingestion-events consumers leave, and the topic always+# has a consumer. The ingestion-events Kafka table drops before its MV so that its consumers leave the+# group at once. With the MV dropped first, they would keep their partitions and consume nothing.+# The apm creates require hosts, so that a topology with no apm hosts fails the migration before the+# drops instead of leaving no consumer.+#+# The ingestion-events `writable_logs34` stays. A dropped Distributed table loses the blocks it still+# queues for the logs cluster, and no migration statement can drain it only where it exists. With the+# MV gone nothing writes to it, so it only sends what it already holds. Drop it by hand once+# `system.distribution_queue` shows it empty.+#+# The multinode stack keeps the ingest on its ingestion-events node, so this migration is cloud-only.+operations = (+ [+ run_sql_with_exceptions(WRITABLE_LOGS34_TABLE_SQL(), node_roles=[NodeRole.APM], require_hosts=True),+ run_sql_with_exceptions(KAFKA_LOGS_AVRO_TABLE_SQL(), node_roles=[NodeRole.APM], require_hosts=True),+ run_sql_with_exceptions(+ KAFKA_LOGS34_AVRO_MV(to_table="writable_logs34"), node_roles=[NodeRole.APM], require_hosts=True+ ),+ run_sql_with_exceptions(f"DROP TABLE IF EXISTS {DB}.kafka_logs_avro", node_roles=[NodeRole.INGESTION_EVENTS]),+ run_sql_with_exceptions(+ f"DROP TABLE IF EXISTS {DB}.kafka_logs34_avro_mv", node_roles=[NodeRole.INGESTION_EVENTS]+ ),+ ]+ if run_mode().is_deployed_cloud+ else []+)diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txtindex 774f2a7b49ed..aa9442ad5cb9 100644--- a/posthog/clickhouse/migrations/max_migration.txt+++ b/posthog/clickhouse/migrations/max_migration.txt@@ -1 +1 @@-0358_platform_alert_events_source_detail+0359_logs_ingest_to_apm