Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(clickhouse): run logs kafka ingest on ingestion-apm in prod (#122283)

Bryan Ciaraldi3c382acdd32d
diff --git a/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl b/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl
index 197ea16b9716..4341625ce848 100644
--- a/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl
@@ -1,4 +1,72 @@
database "posthog" {
+ table "kafka_logs_avro" {
+ settings = {
+ input_format_avro_allow_missing_fields = "1"
+ }
+ column "uuid" {
+ type = "String"
+ }
+ column "trace_id" {
+ type = "String"
+ }
+ column "span_id" {
+ type = "String"
+ }
+ column "trace_flags" {
+ type = "Int32"
+ }
+ column "timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "observed_timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "body" {
+ type = "String"
+ }
+ column "severity_text" {
+ type = "String"
+ }
+ column "severity_number" {
+ type = "Int32"
+ }
+ column "service_name" {
+ type = "String"
+ }
+ column "resource_attributes" {
+ type = "Map(LowCardinality(String), String)"
+ }
+ column "instrumentation_scope" {
+ type = "String"
+ }
+ column "event_name" {
+ type = "String"
+ }
+ column "attributes" {
+ type = "Map(LowCardinality(String), String)"
+ }
+ column "retention_days" {
+ type = "Nullable(Int32)"
+ }
+ column "pattern" {
+ type = "Nullable(String)"
+ }
+ column "pattern_version" {
+ type = "Nullable(Int32)"
+ }
+ engine "kafka" {
+ collection = "warpstream_logs"
+ topic_list = "clickhouse_logs"
+ group_name = "clickhouse-logs-avro-new"
+ format = "Avro"
+ num_consumers = 4
+ skip_broken_messages = 100
+ poll_timeout_ms = 3000
+ poll_max_batch_size = 1000
+ thread_per_consumer = true
+ }
+ }
+
table "kafka_metrics_avro4" {
settings = {
input_format_avro_allow_missing_fields = "1"
@@ -170,6 +238,126 @@ database "posthog" {
}
}
+ table "writable_logs34" {
+ settings = {
+ background_insert_batch = "1"
+ }
+ column "time_bucket" {
+ type = "DateTime"
+ materialized = "toStartOfDay(timestamp)"
+ }
+ column "original_expiry_timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "uuid" {
+ type = "String"
+ }
+ column "team_id" {
+ type = "Int32"
+ }
+ column "trace_id" {
+ type = "String"
+ }
+ column "span_id" {
+ type = "String"
+ }
+ column "trace_flags" {
+ type = "Int32"
+ }
+ column "timestamp" {
+ type = "DateTime64(6)"
+ codec = "DoubleDelta"
+ }
+ column "observed_timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "created_at" {
+ type = "DateTime64(6)"
+ materialized = "now()"
+ }
+ column "body" {
+ type = "String"
+ }
+ column "severity_text" {
+ type = "LowCardinality(String)"
+ }
+ column "severity_number" {
+ type = "Int32"
+ }
+ column "service_name" {
+ type = "LowCardinality(String)"
+ }
+ column "resource_attributes" {
+ type = "Map(LowCardinality(String), String)"
+ }
+ column "resource_fingerprint" {
+ type = "UInt64"
+ materialized = "cityHash64(resource_attributes)"
+ }
+ column "instrumentation_scope" {
+ type = "String"
+ }
+ column "event_name" {
+ type = "String"
+ }
+ column "attributes_map_str" {
+ type = "Map(LowCardinality(String), String)"
+ }
+ column "level" {
+ type = "String"
+ alias = "severity_text"
+ }
+ column "mat_body_ipv4_matches" {
+ type = "Array(String)"
+ alias = "extractAll(body, '(\\\\d\\\\.((25[0-5]|(2[0-4]|1(0, 1)[0-9])(0, 1)[0-9])\\\\.)(2, 2)([0-9]))')"
+ }
+ column "time_minute" {
+ type = "DateTime"
+ alias = "toStartOfMinute(timestamp)"
+ }
+ column "attributes" {
+ type = "Map(LowCardinality(String), String)"
+ alias = "mapApply((k, v) -> (left(k, -5), v), attributes_map_str)"
+ }
+ column "attributes_map_float" {
+ type = "Map(LowCardinality(String), Float64)"
+ materialized = "mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__float'), toFloat64OrNull(v)), attributes_map_str))"
+ }
+ column "attributes_map_datetime" {
+ type = "Map(LowCardinality(String), DateTime64(6))"
+ materialized = "mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__datetime'), parseDateTimeBestEffortOrNull(v, 6)), attributes_map_str))"
+ }
+ column "_partition" {
+ type = "UInt32"
+ }
+ column "_topic" {
+ type = "String"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_bytes_uncompressed" {
+ type = "UInt64"
+ }
+ column "_bytes_compressed" {
+ type = "UInt64"
+ }
+ column "_record_count" {
+ type = "UInt64"
+ }
+ column "pattern" {
+ type = "String"
+ }
+ column "pattern_version" {
+ type = "UInt8"
+ }
+ engine "distributed" {
+ cluster_name = "logs"
+ remote_database = "posthog"
+ remote_table = "logs34"
+ }
+ }
+
table "writable_metrics4_attributes" {
column "team_id" {
type = "Int32"
@@ -723,6 +911,118 @@ database "posthog" {
}
}
+ materialized_view "kafka_logs34_avro_mv" {
+ to_table = "posthog.writable_logs34"
+ query = <<SQL
+SELECT
+ uuid,
+ trace_id,
+ span_id,
+ trace_flags,
+ timestamp,
+ observed_timestamp,
+ body,
+ severity_text,
+ severity_number,
+ service_name,
+ instrumentation_scope,
+ event_name,
+ mapSort(mapApply((k, v) -> (concat(k, '__str'), JSONExtractString(v)), attributes)) AS attributes_map_str,
+ mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), resource_attributes)) AS resource_attributes,
+ toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,
+ timestamp
+ + toIntervalDay(
+ if(
+ (retention_days IS NOT NULL) AND (retention_days > 0),
+ retention_days,
+ toInt32OrDefault(_headers.value[indexOf(_headers.name, 'retention-days')], toInt32(15))
+ )
+ ) AS original_expiry_timestamp,
+ _partition,
+ _topic,
+ _offset,
+ toInt64OrDefault(_headers.value[indexOf(_headers.name, 'record_count')], toInt64(1)) AS _record_count,
+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_uncompressed')]) / _record_count AS _bytes_uncompressed,
+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_compressed')]) / _record_count AS _bytes_compressed,
+ ifNull(pattern, '') AS pattern,
+ toUInt8(ifNull(pattern_version, 0)) AS pattern_version
+FROM posthog.kafka_logs_avro
+SQL
+
+ column "uuid" {
+ type = "String"
+ }
+ column "trace_id" {
+ type = "String"
+ }
+ column "span_id" {
+ type = "String"
+ }
+ column "trace_flags" {
+ type = "Int32"
+ }
+ column "timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "observed_timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "body" {
+ type = "String"
+ }
+ column "severity_text" {
+ type = "String"
+ }
+ column "severity_number" {
+ type = "Int32"
+ }
+ column "service_name" {
+ type = "String"
+ }
+ column "instrumentation_scope" {
+ type = "String"
+ }
+ column "event_name" {
+ type = "String"
+ }
+ column "attributes_map_str" {
+ type = "Map(String, String)"
+ }
+ column "resource_attributes" {
+ type = "Map(String, String)"
+ }
+ column "team_id" {
+ type = "Int32"
+ }
+ column "original_expiry_timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ column "_topic" {
+ type = "LowCardinality(String)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_record_count" {
+ type = "Int64"
+ }
+ column "_bytes_uncompressed" {
+ type = "Nullable(Int64)"
+ }
+ column "_bytes_compressed" {
+ type = "Nullable(Int64)"
+ }
+ column "pattern" {
+ type = "String"
+ }
+ column "pattern_version" {
+ type = "UInt8"
+ }
+ }
+
materialized_view "kafka_metrics_avro4_mv" {
to_table = "posthog.metrics4_input"
query = <<SQL
diff --git a/posthog/clickhouse/hcl/golden/prod-us/apm.hcl b/posthog/clickhouse/hcl/golden/prod-us/apm.hcl
index 197ea16b9716..4341625ce848 100644
--- a/posthog/clickhouse/hcl/golden/prod-us/apm.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-us/apm.hcl
@@ -1,4 +1,72 @@
database "posthog" {
+ table "kafka_logs_avro" {
+ settings = {
+ input_format_avro_allow_missing_fields = "1"
+ }
+ column "uuid" {
+ type = "String"
+ }
+ column "trace_id" {
+ type = "String"
+ }
+ column "span_id" {
+ type = "String"
+ }
+ column "trace_flags" {
+ type = "Int32"
+ }
+ column "timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "observed_timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "body" {
+ type = "String"
+ }
+ column "severity_text" {
+ type = "String"
+ }
+ column "severity_number" {
+ type = "Int32"
+ }
+ column "service_name" {
+ type = "String"
+ }
+ column "resource_attributes" {
+ type = "Map(LowCardinality(String), String)"
+ }
+ column "instrumentation_scope" {
+ type = "String"
+ }
+ column "event_name" {
+ type = "String"
+ }
+ column "attributes" {
+ type = "Map(LowCardinality(String), String)"
+ }
+ column "retention_days" {
+ type = "Nullable(Int32)"
+ }
+ column "pattern" {
+ type = "Nullable(String)"
+ }
+ column "pattern_version" {
+ type = "Nullable(Int32)"
+ }
+ engine "kafka" {
+ collection = "warpstream_logs"
+ topic_list = "clickhouse_logs"
+ group_name = "clickhouse-logs-avro-new"
+ format = "Avro"
+ num_consumers = 4
+ skip_broken_messages = 100
+ poll_timeout_ms = 3000
+ poll_max_batch_size = 1000
+ thread_per_consumer = true
+ }
+ }
+
table "kafka_metrics_avro4" {
settings = {
input_format_avro_allow_missing_fields = "1"
@@ -170,6 +238,126 @@ database "posthog" {
}
}
+ table "writable_logs34" {
+ settings = {
+ background_insert_batch = "1"
+ }
+ column "time_bucket" {
+ type = "DateTime"
+ materialized = "toStartOfDay(timestamp)"
+ }
+ column "original_expiry_timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "uuid" {
+ type = "String"
+ }
+ column "team_id" {
+ type = "Int32"
+ }
+ column "trace_id" {
+ type = "String"
+ }
+ column "span_id" {
+ type = "String"
+ }
+ column "trace_flags" {
+ type = "Int32"
+ }
+ column "timestamp" {
+ type = "DateTime64(6)"
+ codec = "DoubleDelta"
+ }
+ column "observed_timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "created_at" {
+ type = "DateTime64(6)"
+ materialized = "now()"
+ }
+ column "body" {
+ type = "String"
+ }
+ column "severity_text" {
+ type = "LowCardinality(String)"
+ }
+ column "severity_number" {
+ type = "Int32"
+ }
+ column "service_name" {
+ type = "LowCardinality(String)"
+ }
+ column "resource_attributes" {
+ type = "Map(LowCardinality(String), String)"
+ }
+ column "resource_fingerprint" {
+ type = "UInt64"
+ materialized = "cityHash64(resource_attributes)"
+ }
+ column "instrumentation_scope" {
+ type = "String"
+ }
+ column "event_name" {
+ type = "String"
+ }
+ column "attributes_map_str" {
+ type = "Map(LowCardinality(String), String)"
+ }
+ column "level" {
+ type = "String"
+ alias = "severity_text"
+ }
+ column "mat_body_ipv4_matches" {
+ type = "Array(String)"
+ alias = "extractAll(body, '(\\\\d\\\\.((25[0-5]|(2[0-4]|1(0, 1)[0-9])(0, 1)[0-9])\\\\.)(2, 2)([0-9]))')"
+ }
+ column "time_minute" {
+ type = "DateTime"
+ alias = "toStartOfMinute(timestamp)"
+ }
+ column "attributes" {
+ type = "Map(LowCardinality(String), String)"
+ alias = "mapApply((k, v) -> (left(k, -5), v), attributes_map_str)"
+ }
+ column "attributes_map_float" {
+ type = "Map(LowCardinality(String), Float64)"
+ materialized = "mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__float'), toFloat64OrNull(v)), attributes_map_str))"
+ }
+ column "attributes_map_datetime" {
+ type = "Map(LowCardinality(String), DateTime64(6))"
+ materialized = "mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__datetime'), parseDateTimeBestEffortOrNull(v, 6)), attributes_map_str))"
+ }
+ column "_partition" {
+ type = "UInt32"
+ }
+ column "_topic" {
+ type = "String"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_bytes_uncompressed" {
+ type = "UInt64"
+ }
+ column "_bytes_compressed" {
+ type = "UInt64"
+ }
+ column "_record_count" {
+ type = "UInt64"
+ }
+ column "pattern" {
+ type = "String"
+ }
+ column "pattern_version" {
+ type = "UInt8"
+ }
+ engine "distributed" {
+ cluster_name = "logs"
+ remote_database = "posthog"
+ remote_table = "logs34"
+ }
+ }
+
table "writable_metrics4_attributes" {
column "team_id" {
type = "Int32"
@@ -723,6 +911,118 @@ database "posthog" {
}
}
+ materialized_view "kafka_logs34_avro_mv" {
+ to_table = "posthog.writable_logs34"
+ query = <<SQL
+SELECT
+ uuid,
+ trace_id,
+ span_id,
+ trace_flags,
+ timestamp,
+ observed_timestamp,
+ body,
+ severity_text,
+ severity_number,
+ service_name,
+ instrumentation_scope,
+ event_name,
+ mapSort(mapApply((k, v) -> (concat(k, '__str'), JSONExtractString(v)), attributes)) AS attributes_map_str,
+ mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), resource_attributes)) AS resource_attributes,
+ toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,
+ timestamp
+ + toIntervalDay(
+ if(
+ (retention_days IS NOT NULL) AND (retention_days > 0),
+ retention_days,
+ toInt32OrDefault(_headers.value[indexOf(_headers.name, 'retention-days')], toInt32(15))
+ )
+ ) AS original_expiry_timestamp,
+ _partition,
+ _topic,
+ _offset,
+ toInt64OrDefault(_headers.value[indexOf(_headers.name, 'record_count')], toInt64(1)) AS _record_count,
+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_uncompressed')]) / _record_count AS _bytes_uncompressed,
+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_compressed')]) / _record_count AS _bytes_compressed,
+ ifNull(pattern, '') AS pattern,
+ toUInt8(ifNull(pattern_version, 0)) AS pattern_version
+FROM posthog.kafka_logs_avro
+SQL
+
+ column "uuid" {
+ type = "String"
+ }
+ column "trace_id" {
+ type = "String"
+ }
+ column "span_id" {
+ type = "String"
+ }
+ column "trace_flags" {
+ type = "Int32"
+ }
+ column "timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "observed_timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "body" {
+ type = "String"
+ }
+ column "severity_text" {
+ type = "String"
+ }
+ column "severity_number" {
+ type = "Int32"
+ }
+ column "service_name" {
+ type = "String"
+ }
+ column "instrumentation_scope" {
+ type = "String"
+ }
+ column "event_name" {
+ type = "String"
+ }
+ column "attributes_map_str" {
+ type = "Map(String, String)"
+ }
+ column "resource_attributes" {
+ type = "Map(String, String)"
+ }
+ column "team_id" {
+ type = "Int32"
+ }
+ column "original_expiry_timestamp" {
+ type = "DateTime64(6)"
+ }
+ column "_partition" {
+ type = "UInt64"
+ }
+ column "_topic" {
+ type = "LowCardinality(String)"
+ }
+ column "_offset" {
+ type = "UInt64"
+ }
+ column "_record_count" {
+ type = "Int64"
+ }
+ column "_bytes_uncompressed" {
+ type = "Nullable(Int64)"
+ }
+ column "_bytes_compressed" {
+ type = "Nullable(Int64)"
+ }
+ column "pattern" {
+ type = "String"
+ }
+ column "pattern_version" {
+ type = "UInt8"
+ }
+ }
+
materialized_view "kafka_metrics_avro4_mv" {
to_table = "posthog.metrics4_input"
query = <<SQL
diff --git a/posthog/clickhouse/hcl/manifest.hcl b/posthog/clickhouse/hcl/manifest.hcl
index 34e5a1a1a1c6..581c9249a671 100644
--- a/posthog/clickhouse/hcl/manifest.hcl
+++ b/posthog/clickhouse/hcl/manifest.hcl
@@ -44,12 +44,12 @@ role "logs" {
# APM satellite: the ingestion-apm nodes. Every environment runs the shared
# custom_metrics suite and writes to the query log archive. Every environment also
-# runs the metrics4 ingest chain. Development runs the legacy APM ingest chain too.
-# The production logs nodes still run that legacy chain.
+# runs the metrics4 ingest chain and the logs Kafka ingest. Development runs the
+# legacy APM ingest chain too. The production logs nodes still run that legacy chain.
role "apm" {
env "dev" { layers = ["roles/shared/qla_write.hcl", "roles/coshared/named_collections", "roles/coshared/custom_metrics", "roles/coshared/apm_ingest", "roles/coshared/logs_avro_ingest", "roles/apm/shared", "roles/apm/dev"] }
- env "prod-us" { layers = ["roles/shared/qla_write.hcl", "roles/coshared/named_collections", "roles/coshared/custom_metrics", "roles/apm/shared"] }
- env "prod-eu" { layers = ["roles/shared/qla_write.hcl", "roles/coshared/named_collections", "roles/coshared/custom_metrics", "roles/apm/shared"] }
+ env "prod-us" { layers = ["roles/shared/qla_write.hcl", "roles/coshared/named_collections", "roles/coshared/custom_metrics", "roles/coshared/logs_avro_ingest", "roles/apm/shared"] }
+ env "prod-eu" { layers = ["roles/shared/qla_write.hcl", "roles/coshared/named_collections", "roles/coshared/custom_metrics", "roles/coshared/logs_avro_ingest", "roles/apm/shared"] }
}
# AI_EVENTS satellite (LLM analytics). local/hobby run the MSK variant
diff --git a/posthog/clickhouse/hcl/roles/coshared/logs_avro_ingest/tables.hcl b/posthog/clickhouse/hcl/roles/coshared/logs_avro_ingest/tables.hcl
index 2eb8d14faf71..8dcda1fa9318 100644
--- a/posthog/clickhouse/hcl/roles/coshared/logs_avro_ingest/tables.hcl
+++ b/posthog/clickhouse/hcl/roles/coshared/logs_avro_ingest/tables.hcl
@@ -1,5 +1,5 @@
-# The Kafka ingest for logs. dev runs this on its ingestion-apm nodes, the prod
-# clusters on their ingestion-events nodes.
+# The Kafka ingest for logs. The cloud envs run it on their ingestion-apm nodes,
+# the multinode stack on its ingestion-events node.
database "posthog" {
table "kafka_logs_avro" {
column "uuid" {
@@ -58,7 +58,7 @@ database "posthog" {
topic_list = "clickhouse_logs"
group_name = "clickhouse-logs-avro-new"
format = "Avro"
- num_consumers = 8
+ num_consumers = 4
skip_broken_messages = 100
poll_timeout_ms = 3000
poll_max_batch_size = 1000
diff --git a/posthog/clickhouse/hcl/sql/prod-eu/apm.sql b/posthog/clickhouse/hcl/sql/prod-eu/apm.sql
index 116fb599fbb3..d68e4b981af1 100644
--- a/posthog/clickhouse/hcl/sql/prod-eu/apm.sql
+++ b/posthog/clickhouse/hcl/sql/prod-eu/apm.sql
@@ -1,6 +1,25 @@
-- AUTO-GENERATED from the declarative HCL by ops/gen-sql.sh — do not edit.
-- Full CREATE schema for the prod-eu/apm node. Apply to a fresh ClickHouse to build it.
+CREATE TABLE posthog.kafka_logs_avro (
+ uuid String,
+ trace_id String,
+ span_id String,
+ trace_flags Int32,
+ timestamp DateTime64(6),
+ observed_timestamp DateTime64(6),
+ body String,
+ severity_text String,
+ severity_number Int32,
+ service_name String,
+ resource_attributes Map(LowCardinality(String), String),
+ instrumentation_scope String,
+ event_name String,
+ attributes Map(LowCardinality(String), String),
+ retention_days Nullable(Int32),
+ pattern Nullable(String),
+ pattern_version Nullable(Int32)
+) ENGINE = Kafka(warpstream_logs) SETTINGS input_format_avro_allow_missing_fields = 1, kafka_format = 'Avro', kafka_group_name = 'clickhouse-logs-avro-new', kafka_num_consumers = 4, kafka_poll_max_batch_size = 1000, kafka_poll_timeout_ms = 3000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_logs';
CREATE TABLE posthog.kafka_metrics_avro4 (
uuid String,
trace_id String,
@@ -54,6 +73,41 @@ CREATE TABLE posthog.metrics4_input (
_topic String,
_offset UInt64
) ENGINE = Null();
+CREATE TABLE posthog.writable_logs34 (
+ time_bucket DateTime MATERIALIZED toStartOfDay(timestamp),
+ original_expiry_timestamp DateTime64(6),
+ uuid String,
+ team_id Int32,
+ trace_id String,
+ span_id String,
+ trace_flags Int32,
+ timestamp DateTime64(6) CODEC(DoubleDelta),
+ observed_timestamp DateTime64(6),
+ created_at DateTime64(6) MATERIALIZED now(),
+ body String,
+ severity_text LowCardinality(String),
+ severity_number Int32,
+ service_name LowCardinality(String),
+ resource_attributes Map(LowCardinality(String), String),
+ resource_fingerprint UInt64 MATERIALIZED cityHash64(resource_attributes),
+ instrumentation_scope String,
+ event_name String,
+ attributes_map_str Map(LowCardinality(String), String),
+ level String ALIAS severity_text,
+ mat_body_ipv4_matches Array(String) ALIAS extractAll(body, '(\\d\\.((25[0-5]|(2[0-4]|1(0, 1)[0-9])(0, 1)[0-9])\\.)(2, 2)([0-9]))'),
+ time_minute DateTime ALIAS toStartOfMinute(timestamp),
+ attributes Map(LowCardinality(String), String) ALIAS mapApply((k, v) -> (left(k, -5), v), attributes_map_str),
+ attributes_map_float Map(LowCardinality(String), Float64) MATERIALIZED mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__float'), toFloat64OrNull(v)), attributes_map_str)),
+ attributes_map_datetime Map(LowCardinality(String), DateTime64(6)) MATERIALIZED mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__datetime'), parseDateTimeBestEffortOrNull(v, 6)), attributes_map_str)),
+ _partition UInt32,
+ _topic String,
+ _offset UInt64,
+ _bytes_uncompressed UInt64,
+ _bytes_compressed UInt64,
+ _record_count UInt64,
+ pattern String,
+ pattern_version UInt8
+) ENGINE = Distributed('logs', 'posthog', 'logs34') SETTINGS background_insert_batch = 1;
CREATE TABLE posthog.writable_metrics4_attributes (
team_id Int32,
metric_name LowCardinality(String),
@@ -234,6 +288,39 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (
lc_estimated_rows SimpleAggregateFunction(max, Int64),
lc_estimated_bytes SimpleAggregateFunction(max, Int64)
) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');
+CREATE MATERIALIZED VIEW posthog.kafka_logs34_avro_mv TO posthog.writable_logs34 (uuid String, trace_id String, span_id String, trace_flags Int32, timestamp DateTime64(6), observed_timestamp DateTime64(6), body String, severity_text String, severity_number Int32, service_name String, instrumentation_scope String, event_name String, attributes_map_str Map(String, String), resource_attributes Map(String, String), team_id Int32, original_expiry_timestamp DateTime64(6), _partition UInt64, _topic LowCardinality(String), _offset UInt64, _record_count Int64, _bytes_uncompressed Nullable(Int64), _bytes_compressed Nullable(Int64), pattern String, pattern_version UInt8) AS SELECT
+ uuid,
+ trace_id,
+ span_id,
+ trace_flags,
+ timestamp,
+ observed_timestamp,
+ body,
+ severity_text,
+ severity_number,
+ service_name,
+ instrumentation_scope,
+ event_name,
+ mapSort(mapApply((k, v) -> (concat(k, '__str'), JSONExtractString(v)), attributes)) AS attributes_map_str,
+ mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), resource_attributes)) AS resource_attributes,
+ toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,
+ timestamp
+ + toIntervalDay(
+ if(
+ (retention_days IS NOT NULL) AND (retention_days > 0),
+ retention_days,
+ toInt32OrDefault(_headers.value[indexOf(_headers.name, 'retention-days')], toInt32(15))
+ )
+ ) AS original_expiry_timestamp,
+ _partition,
+ _topic,
+ _offset,
+ toInt64OrDefault(_headers.value[indexOf(_headers.name, 'record_count')], toInt64(1)) AS _record_count,
+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_uncompressed')]) / _record_count AS _bytes_uncompressed,
+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_compressed')]) / _record_count AS _bytes_compressed,
+ ifNull(pattern, '') AS pattern,
+ toUInt8(ifNull(pattern_version, 0)) AS pattern_version
+FROM posthog.kafka_logs_avro;
CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64) AS SELECT
uuid,
toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,
diff --git a/posthog/clickhouse/hcl/sql/prod-us/apm.sql b/posthog/clickhouse/hcl/sql/prod-us/apm.sql
index 08c12fa44e5f..3d0aef40c59b 100644
--- a/posthog/clickhouse/hcl/sql/prod-us/apm.sql
+++ b/posthog/clickhouse/hcl/sql/prod-us/apm.sql
@@ -1,6 +1,25 @@
-- AUTO-GENERATED from the declarative HCL by ops/gen-sql.sh — do not edit.
-- Full CREATE schema for the prod-us/apm node. Apply to a fresh ClickHouse to build it.
+CREATE TABLE posthog.kafka_logs_avro (
+ uuid String,
+ trace_id String,
+ span_id String,
+ trace_flags Int32,
+ timestamp DateTime64(6),
+ observed_timestamp DateTime64(6),
+ body String,
+ severity_text String,
+ severity_number Int32,
+ service_name String,
+ resource_attributes Map(LowCardinality(String), String),
+ instrumentation_scope String,
+ event_name String,
+ attributes Map(LowCardinality(String), String),
+ retention_days Nullable(Int32),
+ pattern Nullable(String),
+ pattern_version Nullable(Int32)
+) ENGINE = Kafka(warpstream_logs) SETTINGS input_format_avro_allow_missing_fields = 1, kafka_format = 'Avro', kafka_group_name = 'clickhouse-logs-avro-new', kafka_num_consumers = 4, kafka_poll_max_batch_size = 1000, kafka_poll_timeout_ms = 3000, kafka_skip_broken_messages = 100, kafka_thread_per_consumer = 1, kafka_topic_list = 'clickhouse_logs';
CREATE TABLE posthog.kafka_metrics_avro4 (
uuid String,
trace_id String,
@@ -54,6 +73,41 @@ CREATE TABLE posthog.metrics4_input (
_topic String,
_offset UInt64
) ENGINE = Null();
+CREATE TABLE posthog.writable_logs34 (
+ time_bucket DateTime MATERIALIZED toStartOfDay(timestamp),
+ original_expiry_timestamp DateTime64(6),
+ uuid String,
+ team_id Int32,
+ trace_id String,
+ span_id String,
+ trace_flags Int32,
+ timestamp DateTime64(6) CODEC(DoubleDelta),
+ observed_timestamp DateTime64(6),
+ created_at DateTime64(6) MATERIALIZED now(),
+ body String,
+ severity_text LowCardinality(String),
+ severity_number Int32,
+ service_name LowCardinality(String),
+ resource_attributes Map(LowCardinality(String), String),
+ resource_fingerprint UInt64 MATERIALIZED cityHash64(resource_attributes),
+ instrumentation_scope String,
+ event_name String,
+ attributes_map_str Map(LowCardinality(String), String),
+ level String ALIAS severity_text,
+ mat_body_ipv4_matches Array(String) ALIAS extractAll(body, '(\\d\\.((25[0-5]|(2[0-4]|1(0, 1)[0-9])(0, 1)[0-9])\\.)(2, 2)([0-9]))'),
+ time_minute DateTime ALIAS toStartOfMinute(timestamp),
+ attributes Map(LowCardinality(String), String) ALIAS mapApply((k, v) -> (left(k, -5), v), attributes_map_str),
+ attributes_map_float Map(LowCardinality(String), Float64) MATERIALIZED mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__float'), toFloat64OrNull(v)), attributes_map_str)),
+ attributes_map_datetime Map(LowCardinality(String), DateTime64(6)) MATERIALIZED mapFilter((k, v) -> (v IS NOT NULL), mapApply((k, v) -> (concat(left(k, -5), '__datetime'), parseDateTimeBestEffortOrNull(v, 6)), attributes_map_str)),
+ _partition UInt32,
+ _topic String,
+ _offset UInt64,
+ _bytes_uncompressed UInt64,
+ _bytes_compressed UInt64,
+ _record_count UInt64,
+ pattern String,
+ pattern_version UInt8
+) ENGINE = Distributed('logs', 'posthog', 'logs34') SETTINGS background_insert_batch = 1;
CREATE TABLE posthog.writable_metrics4_attributes (
team_id Int32,
metric_name LowCardinality(String),
@@ -234,6 +288,39 @@ CREATE TABLE posthog.writable_query_log_archive_v3 (
lc_estimated_rows SimpleAggregateFunction(max, Int64),
lc_estimated_bytes SimpleAggregateFunction(max, Int64)
) ENGINE = Distributed('ops', 'posthog', 'query_log_archive_v3_buffer');
+CREATE MATERIALIZED VIEW posthog.kafka_logs34_avro_mv TO posthog.writable_logs34 (uuid String, trace_id String, span_id String, trace_flags Int32, timestamp DateTime64(6), observed_timestamp DateTime64(6), body String, severity_text String, severity_number Int32, service_name String, instrumentation_scope String, event_name String, attributes_map_str Map(String, String), resource_attributes Map(String, String), team_id Int32, original_expiry_timestamp DateTime64(6), _partition UInt64, _topic LowCardinality(String), _offset UInt64, _record_count Int64, _bytes_uncompressed Nullable(Int64), _bytes_compressed Nullable(Int64), pattern String, pattern_version UInt8) AS SELECT
+ uuid,
+ trace_id,
+ span_id,
+ trace_flags,
+ timestamp,
+ observed_timestamp,
+ body,
+ severity_text,
+ severity_number,
+ service_name,
+ instrumentation_scope,
+ event_name,
+ mapSort(mapApply((k, v) -> (concat(k, '__str'), JSONExtractString(v)), attributes)) AS attributes_map_str,
+ mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), resource_attributes)) AS resource_attributes,
+ toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,
+ timestamp
+ + toIntervalDay(
+ if(
+ (retention_days IS NOT NULL) AND (retention_days > 0),
+ retention_days,
+ toInt32OrDefault(_headers.value[indexOf(_headers.name, 'retention-days')], toInt32(15))
+ )
+ ) AS original_expiry_timestamp,
+ _partition,
+ _topic,
+ _offset,
+ toInt64OrDefault(_headers.value[indexOf(_headers.name, 'record_count')], toInt64(1)) AS _record_count,
+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_uncompressed')]) / _record_count AS _bytes_uncompressed,
+ toInt64OrNull(_headers.value[indexOf(_headers.name, 'bytes_compressed')]) / _record_count AS _bytes_compressed,
+ ifNull(pattern, '') AS pattern,
+ toUInt8(ifNull(pattern_version, 0)) AS pattern_version
+FROM posthog.kafka_logs_avro;
CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64) AS SELECT
uuid,
toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,
diff --git a/posthog/clickhouse/logs/logs34.py b/posthog/clickhouse/logs/logs34.py
index 738e46500586..9ed815e7f0e2 100644
--- a/posthog/clickhouse/logs/logs34.py
+++ b/posthog/clickhouse/logs/logs34.py
@@ -105,11 +105,11 @@ CREATE OR REPLACE TABLE {database}.logs_distributed AS {database}.{table_name} E
def WRITABLE_LOGS34_TABLE_SQL():
- # The write path for the Kafka MV, hosted on the ingestion-events nodes: a Distributed table over
+ # The write path for the Kafka MV, hosted on the ingestion nodes: a Distributed table over
# the logs cluster's `logs34`. The columns are spelled out (rather than `AS logs34`) because
- # `logs34` is not local on the events nodes — only this Distributed front is. Keep the column list
+ # `logs34` is not local on the ingestion nodes — only this Distributed front is. Keep the column list
# byte-identical to `LOGS34_TABLE_SQL` (indexes/projection are storage-only and omitted here).
- # `CLICKHOUSE_LOGS_WRITE_CLUSTER`, not `CLICKHOUSE_LOGS_CLUSTER`: the events node reaches the logs
+ # `CLICKHOUSE_LOGS_WRITE_CLUSTER`, not `CLICKHOUSE_LOGS_CLUSTER`: the ingestion node reaches the logs
# cluster under its own name, and the latter's default resolves to the data node from there.
db = settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE
return f"""
@@ -286,7 +286,7 @@ CREATE TABLE IF NOT EXISTS {settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE}.{KAFKA_TA
ENGINE = {kafka_engine(topic=KAFKA_TOPIC, group=KAFKA_GROUP, serialization="Avro", named_collection=KAFKA_NAMED_COLLECTION)}
SETTINGS
kafka_skip_broken_messages = 100,
- kafka_num_consumers = {kafka_num_consumers(8)},
+ kafka_num_consumers = {kafka_num_consumers(4)},
kafka_poll_timeout_ms = 3000,
kafka_poll_max_batch_size = 1000,
kafka_thread_per_consumer = 1,
@@ -327,8 +327,8 @@ FROM {db}.{KAFKA_TABLE_NAME}"""
def KAFKA_LOGS34_AVRO_MV(to_table: str = TABLE_NAME):
db = settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE
# `to_table` defaults to `logs34` so the historical migration 0280 (which creates this MV on the
- # logs nodes, where `logs34` is local) keeps working when replayed. Migration 0305 passes
- # `writable_logs34` — the Distributed(logs, posthog, logs34) table on the ingestion-events nodes —
+ # logs nodes, where `logs34` is local) keeps working when replayed. Migrations 0305 and 0359 pass
+ # `writable_logs34` — the Distributed(logs, posthog, logs34) table on the ingestion nodes —
# because `logs34` itself is not local there.
return f"""
CREATE MATERIALIZED VIEW IF NOT EXISTS {db}.kafka_logs34_avro_mv TO {db}.{to_table}
diff --git a/posthog/clickhouse/migrations/0359_logs_ingest_to_apm.py b/posthog/clickhouse/migrations/0359_logs_ingest_to_apm.py
new file mode 100644
index 000000000000..0c075b5a0ef9
--- /dev/null
+++ b/posthog/clickhouse/migrations/0359_logs_ingest_to_apm.py
@@ -0,0 +1,42 @@
+from django.conf import settings
+
+from posthog.clickhouse.client.connection import NodeRole
+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions
+from posthog.clickhouse.logs import KAFKA_LOGS34_AVRO_MV, KAFKA_LOGS_AVRO_TABLE_SQL, WRITABLE_LOGS34_TABLE_SQL
+from posthog.run_mode import run_mode
+
+DB = settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE
+
+# The cloud envs run the logs Kafka ingest on the ingestion-apm nodes, as the HCL `apm` role composes
+# `coshared/logs_avro_ingest` for dev, prod-us and prod-eu. 0305, 0307 and 0348 put it on the
+# ingestion-events nodes in prod, so a cluster built from the migration history would consume there.
+# On a cluster that already runs the ingest on apm, every statement is a no-op.
+#
+# The apm side is created first. Both sides use the same `kafka_group_name`, so the apm consumers
+# join the group and take partitions before the ingestion-events consumers leave, and the topic always
+# has a consumer. The ingestion-events Kafka table drops before its MV so that its consumers leave the
+# group at once. With the MV dropped first, they would keep their partitions and consume nothing.
+# The apm creates require hosts, so that a topology with no apm hosts fails the migration before the
+# drops instead of leaving no consumer.
+#
+# The ingestion-events `writable_logs34` stays. A dropped Distributed table loses the blocks it still
+# queues for the logs cluster, and no migration statement can drain it only where it exists. With the
+# MV gone nothing writes to it, so it only sends what it already holds. Drop it by hand once
+# `system.distribution_queue` shows it empty.
+#
+# The multinode stack keeps the ingest on its ingestion-events node, so this migration is cloud-only.
+operations = (
+ [
+ run_sql_with_exceptions(WRITABLE_LOGS34_TABLE_SQL(), node_roles=[NodeRole.APM], require_hosts=True),
+ run_sql_with_exceptions(KAFKA_LOGS_AVRO_TABLE_SQL(), node_roles=[NodeRole.APM], require_hosts=True),
+ run_sql_with_exceptions(
+ KAFKA_LOGS34_AVRO_MV(to_table="writable_logs34"), node_roles=[NodeRole.APM], require_hosts=True
+ ),
+ run_sql_with_exceptions(f"DROP TABLE IF EXISTS {DB}.kafka_logs_avro", node_roles=[NodeRole.INGESTION_EVENTS]),
+ run_sql_with_exceptions(
+ f"DROP TABLE IF EXISTS {DB}.kafka_logs34_avro_mv", node_roles=[NodeRole.INGESTION_EVENTS]
+ ),
+ ]
+ if run_mode().is_deployed_cloud
+ else []
+)
diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txt
index 774f2a7b49ed..aa9442ad5cb9 100644
--- a/posthog/clickhouse/migrations/max_migration.txt
+++ b/posthog/clickhouse/migrations/max_migration.txt
@@ -1 +1 @@
-0358_platform_alert_events_source_detail
+0359_logs_ingest_to_apm