:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(metrics): write a two-hop partition checkpoint from the metrics4 chain (#127782)
Jimmy Zhuc6a144181b7b
diff --git a/posthog/clickhouse/hcl/golden/dev/apm.hcl b/posthog/clickhouse/hcl/golden/dev/apm.hclindex 98851c33ac1a..3a1dd2584fcb 100644--- a/posthog/clickhouse/hcl/golden/dev/apm.hcl+++ b/posthog/clickhouse/hcl/golden/dev/apm.hcl@@ -317,6 +317,12 @@ database "posthog" {column "_offset" {type = "UInt64"}+ column "_source_topic" {+ type = "String"+ }+ column "_source_partition" {+ type = "UInt32"+ }engine "null" {}}@@ -1499,7 +1505,9 @@ SELECT) AS attributes,_partition,_topic,- _offset+ _offset,+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partitionFROM posthog.kafka_metrics_avro4WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULLSETTINGS@@ -1588,6 +1596,12 @@ SQLcolumn "_offset" {type = "UInt64"}+ column "_source_topic" {+ type = "String"+ }+ column "_source_partition" {+ type = "UInt32"+ }}materialized_view "kafka_trace_spans_avro_mv" {@@ -2049,6 +2063,47 @@ SQL}}+ materialized_view "metrics4_input_to_metrics_kafka_metrics" {+ to_table = "posthog.writable_metrics_kafka_metrics"+ query = <<SQL+SELECT+ kafka_partition AS _partition,+ kafka_topic AS _topic,+ maxSimpleState(kafka_offset) AS max_offset,+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,+ maxSimpleState(now()) AS max_created_at+FROM+ (+ SELECT+ kafka_source.1 AS kafka_topic,+ kafka_source.2 AS kafka_partition,+ kafka_source.3 AS kafka_offset,+ observed_timestamp+ FROM+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source+ WHERE kafka_topic != ''+ )+GROUP BY+ kafka_partition, kafka_topic+SQL++ column "_partition" {+ type = "UInt32"+ }+ column "_topic" {+ type = "String"+ }+ column "max_offset" {+ type = "SimpleAggregateFunction(max, UInt64)"+ }+ column "max_observed_timestamp" {+ type = "SimpleAggregateFunction(max, DateTime64(6))"+ }+ column "max_created_at" {+ type = "SimpleAggregateFunction(max, DateTime)"+ }+ }+materialized_view "ops_query_log_archive_v3_mv" {to_table = "posthog.writable_query_log_archive_v3"query = <<SQLdiff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hclindex 84a59f0ec753..4ee171f381ea 100644--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl@@ -7865,6 +7865,12 @@ SQLcolumn "_offset" {type = "UInt64"}+ column "_source_topic" {+ type = "String"+ }+ column "_source_partition" {+ type = "UInt32"+ }engine "null" {}}@@ -18323,6 +18329,35 @@ SQL}}+ table "writable_metrics_kafka_metrics" {+ column "_partition" {+ type = "UInt32"+ }+ column "_topic" {+ type = "String"+ }+ column "max_offset" {+ type = "SimpleAggregateFunction(max, UInt64)"+ }+ column "max_observed_timestamp" {+ type = "SimpleAggregateFunction(max, DateTime64(9))"+ }+ column "max_timestamp" {+ type = "SimpleAggregateFunction(max, DateTime64(9))"+ }+ column "max_created_at" {+ type = "SimpleAggregateFunction(max, DateTime64(9))"+ }+ column "max_lag" {+ type = "SimpleAggregateFunction(max, UInt64)"+ }+ engine "distributed" {+ cluster_name = "logs"+ remote_database = "posthog"+ remote_table = "metrics_kafka_metrics"+ }+ }+table "writable_person" {column "id" {type = "UUID"@@ -22138,7 +22173,9 @@ SELECT) AS attributes,_partition,_topic,- _offset+ _offset,+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partitionFROM posthog.kafka_metrics_avro4WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULLSETTINGS@@ -22227,6 +22264,12 @@ SQLcolumn "_offset" {type = "UInt64"}+ column "_source_topic" {+ type = "String"+ }+ column "_source_partition" {+ type = "UInt32"+ }}materialized_view "kafka_trace_spans_avro_mv" {@@ -23320,6 +23363,47 @@ SQL}}+ materialized_view "metrics4_input_to_metrics_kafka_metrics" {+ to_table = "posthog.writable_metrics_kafka_metrics"+ query = <<SQL+SELECT+ kafka_partition AS _partition,+ kafka_topic AS _topic,+ maxSimpleState(kafka_offset) AS max_offset,+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,+ maxSimpleState(now()) AS max_created_at+FROM+ (+ SELECT+ kafka_source.1 AS kafka_topic,+ kafka_source.2 AS kafka_partition,+ kafka_source.3 AS kafka_offset,+ observed_timestamp+ FROM+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source+ WHERE kafka_topic != ''+ )+GROUP BY+ kafka_partition, kafka_topic+SQL++ column "_partition" {+ type = "UInt32"+ }+ column "_topic" {+ type = "String"+ }+ column "max_offset" {+ type = "SimpleAggregateFunction(max, UInt64)"+ }+ column "max_observed_timestamp" {+ type = "SimpleAggregateFunction(max, DateTime64(6))"+ }+ column "max_created_at" {+ type = "SimpleAggregateFunction(max, DateTime)"+ }+ }+materialized_view "ops_query_log_archive_v3_mv" {to_table = "posthog.writable_query_log_archive_v3"query = <<SQLdiff --git a/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl b/posthog/clickhouse/hcl/golden/prod-eu/apm.hclindex 7ca5a537b81f..4eb0db91d364 100644--- a/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl+++ b/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl@@ -234,6 +234,12 @@ database "posthog" {column "_offset" {type = "UInt64"}+ column "_source_topic" {+ type = "String"+ }+ column "_source_partition" {+ type = "UInt32"+ }engine "null" {}}@@ -559,6 +565,35 @@ database "posthog" {}}+ table "writable_metrics_kafka_metrics" {+ column "_partition" {+ type = "UInt32"+ }+ column "_topic" {+ type = "String"+ }+ column "max_offset" {+ type = "SimpleAggregateFunction(max, UInt64)"+ }+ column "max_observed_timestamp" {+ type = "SimpleAggregateFunction(max, DateTime64(9))"+ }+ column "max_timestamp" {+ type = "SimpleAggregateFunction(max, DateTime64(9))"+ }+ column "max_created_at" {+ type = "SimpleAggregateFunction(max, DateTime64(9))"+ }+ column "max_lag" {+ type = "SimpleAggregateFunction(max, UInt64)"+ }+ engine "distributed" {+ cluster_name = "logs"+ remote_database = "posthog"+ remote_table = "metrics_kafka_metrics"+ }+ }+table "writable_query_log_archive_v3" {column "initial_query_start_time" {type = "DateTime64(6, 'UTC')"@@ -1078,7 +1113,9 @@ SELECT) AS attributes,_partition,_topic,- _offset+ _offset,+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partitionFROM posthog.kafka_metrics_avro4WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULLSETTINGS@@ -1167,6 +1204,12 @@ SQLcolumn "_offset" {type = "UInt64"}+ column "_source_topic" {+ type = "String"+ }+ column "_source_partition" {+ type = "UInt32"+ }}materialized_view "metrics4_input_to_metrics4_attributes" {@@ -1496,6 +1539,47 @@ SQL}}+ materialized_view "metrics4_input_to_metrics_kafka_metrics" {+ to_table = "posthog.writable_metrics_kafka_metrics"+ query = <<SQL+SELECT+ kafka_partition AS _partition,+ kafka_topic AS _topic,+ maxSimpleState(kafka_offset) AS max_offset,+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,+ maxSimpleState(now()) AS max_created_at+FROM+ (+ SELECT+ kafka_source.1 AS kafka_topic,+ kafka_source.2 AS kafka_partition,+ kafka_source.3 AS kafka_offset,+ observed_timestamp+ FROM+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source+ WHERE kafka_topic != ''+ )+GROUP BY+ kafka_partition, kafka_topic+SQL++ column "_partition" {+ type = "UInt32"+ }+ column "_topic" {+ type = "String"+ }+ column "max_offset" {+ type = "SimpleAggregateFunction(max, UInt64)"+ }+ column "max_observed_timestamp" {+ type = "SimpleAggregateFunction(max, DateTime64(6))"+ }+ column "max_created_at" {+ type = "SimpleAggregateFunction(max, DateTime)"+ }+ }+materialized_view "ops_query_log_archive_v3_mv" {to_table = "posthog.writable_query_log_archive_v3"query = <<SQLdiff --git a/posthog/clickhouse/hcl/golden/prod-us/apm.hcl b/posthog/clickhouse/hcl/golden/prod-us/apm.hclindex 7ca5a537b81f..4eb0db91d364 100644--- a/posthog/clickhouse/hcl/golden/prod-us/apm.hcl+++ b/posthog/clickhouse/hcl/golden/prod-us/apm.hcl@@ -234,6 +234,12 @@ database "posthog" {column "_offset" {type = "UInt64"}+ column "_source_topic" {+ type = "String"+ }+ column "_source_partition" {+ type = "UInt32"+ }engine "null" {}}@@ -559,6 +565,35 @@ database "posthog" {}}+ table "writable_metrics_kafka_metrics" {+ column "_partition" {+ type = "UInt32"+ }+ column "_topic" {+ type = "String"+ }+ column "max_offset" {+ type = "SimpleAggregateFunction(max, UInt64)"+ }+ column "max_observed_timestamp" {+ type = "SimpleAggregateFunction(max, DateTime64(9))"+ }+ column "max_timestamp" {+ type = "SimpleAggregateFunction(max, DateTime64(9))"+ }+ column "max_created_at" {+ type = "SimpleAggregateFunction(max, DateTime64(9))"+ }+ column "max_lag" {+ type = "SimpleAggregateFunction(max, UInt64)"+ }+ engine "distributed" {+ cluster_name = "logs"+ remote_database = "posthog"+ remote_table = "metrics_kafka_metrics"+ }+ }+table "writable_query_log_archive_v3" {column "initial_query_start_time" {type = "DateTime64(6, 'UTC')"@@ -1078,7 +1113,9 @@ SELECT) AS attributes,_partition,_topic,- _offset+ _offset,+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partitionFROM posthog.kafka_metrics_avro4WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULLSETTINGS@@ -1167,6 +1204,12 @@ SQLcolumn "_offset" {type = "UInt64"}+ column "_source_topic" {+ type = "String"+ }+ column "_source_partition" {+ type = "UInt32"+ }}materialized_view "metrics4_input_to_metrics4_attributes" {@@ -1496,6 +1539,47 @@ SQL}}+ materialized_view "metrics4_input_to_metrics_kafka_metrics" {+ to_table = "posthog.writable_metrics_kafka_metrics"+ query = <<SQL+SELECT+ kafka_partition AS _partition,+ kafka_topic AS _topic,+ maxSimpleState(kafka_offset) AS max_offset,+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,+ maxSimpleState(now()) AS max_created_at+FROM+ (+ SELECT+ kafka_source.1 AS kafka_topic,+ kafka_source.2 AS kafka_partition,+ kafka_source.3 AS kafka_offset,+ observed_timestamp+ FROM+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source+ WHERE kafka_topic != ''+ )+GROUP BY+ kafka_partition, kafka_topic+SQL++ column "_partition" {+ type = "UInt32"+ }+ column "_topic" {+ type = "String"+ }+ column "max_offset" {+ type = "SimpleAggregateFunction(max, UInt64)"+ }+ column "max_observed_timestamp" {+ type = "SimpleAggregateFunction(max, DateTime64(6))"+ }+ column "max_created_at" {+ type = "SimpleAggregateFunction(max, DateTime)"+ }+ }+materialized_view "ops_query_log_archive_v3_mv" {to_table = "posthog.writable_query_log_archive_v3"query = <<SQLdiff --git a/posthog/clickhouse/hcl/roles/apm/dev/tables.hcl b/posthog/clickhouse/hcl/roles/apm/dev/tables.hclindex 2f6d2d70cd5d..d9e4d7b4b24c 100644--- a/posthog/clickhouse/hcl/roles/apm/dev/tables.hcl+++ b/posthog/clickhouse/hcl/roles/apm/dev/tables.hcl@@ -185,35 +185,6 @@ database "posthog" {}}- table "writable_metrics_kafka_metrics" {- column "_partition" {- type = "UInt32"- }- column "_topic" {- type = "String"- }- column "max_offset" {- type = "SimpleAggregateFunction(max, UInt64)"- }- column "max_observed_timestamp" {- type = "SimpleAggregateFunction(max, DateTime64(9))"- }- column "max_timestamp" {- type = "SimpleAggregateFunction(max, DateTime64(9))"- }- column "max_created_at" {- type = "SimpleAggregateFunction(max, DateTime64(9))"- }- column "max_lag" {- type = "SimpleAggregateFunction(max, UInt64)"- }- engine "distributed" {- cluster_name = "logs"- remote_database = "posthog"- remote_table = "metrics_kafka_metrics"- }- }-table "writable_trace_spans" {column "time_bucket" {type = "DateTime"diff --git a/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl b/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hclindex 4a8b28f426c8..b667a1291f79 100644--- a/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl+++ b/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl@@ -70,6 +70,8 @@ database "posthog" {column "_partition" { type = "UInt32" }column "_topic" { type = "String" }column "_offset" { type = "UInt64" }+ column "_source_topic" { type = "String" }+ column "_source_partition" { type = "UInt32" }engine "null" {}}@@ -168,6 +170,33 @@ database "posthog" {}}+ table "writable_metrics_kafka_metrics" {+ column "_partition" { type = "UInt32" }+ column "_topic" { type = "String" }+ column "max_offset" { type = "SimpleAggregateFunction(max, UInt64)" }+ column "max_observed_timestamp" { type = "SimpleAggregateFunction(max, DateTime64(9))" }+ column "max_timestamp" { type = "SimpleAggregateFunction(max, DateTime64(9))" }+ column "max_created_at" { type = "SimpleAggregateFunction(max, DateTime64(9))" }+ column "max_lag" { type = "SimpleAggregateFunction(max, UInt64)" }+ engine "distributed" {+ cluster_name = "logs"+ remote_database = "posthog"+ remote_table = "metrics_kafka_metrics"+ }+ }++ # The newest capture time each Kafka partition has delivered. The floor across partitions is+ # how far every partition has reached, which the metrics alert evaluator reads.+ materialized_view "metrics4_input_to_metrics_kafka_metrics" {+ to_table = "posthog.writable_metrics_kafka_metrics"+ query = file("sql/metrics4_input_to_metrics_kafka_metrics.sql")+ column "_partition" { type = "UInt32" }+ column "_topic" { type = "String" }+ column "max_offset" { type = "SimpleAggregateFunction(max, UInt64)" }+ column "max_observed_timestamp" { type = "SimpleAggregateFunction(max, DateTime64(6))" }+ column "max_created_at" { type = "SimpleAggregateFunction(max, DateTime)" }+ }+materialized_view "kafka_metrics_avro4_mv" {to_table = "posthog.metrics4_input"query = file("sql/kafka_metrics_avro4_mv.sql")@@ -198,6 +227,8 @@ database "posthog" {column "_partition" { type = "UInt64" }column "_topic" { type = "LowCardinality(String)" }column "_offset" { type = "UInt64" }+ column "_source_topic" { type = "String" }+ column "_source_partition" { type = "UInt32" }}materialized_view "metrics4_input_to_metrics4_samples" {diff --git a/posthog/clickhouse/hcl/roles/apm/shared/sql/kafka_metrics_avro4_mv.sql b/posthog/clickhouse/hcl/roles/apm/shared/sql/kafka_metrics_avro4_mv.sqlindex ce97139b2ac9..6237cb03c6f9 100644--- a/posthog/clickhouse/hcl/roles/apm/shared/sql/kafka_metrics_avro4_mv.sql+++ b/posthog/clickhouse/hcl/roles/apm/shared/sql/kafka_metrics_avro4_mv.sql@@ -25,7 +25,9 @@ SELECTif(toBool(ifNull(has_labels, 1)), mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), attributes)), CAST(map(), 'Map(String, String)')) AS attributes,_partition,_topic,- _offset+ _offset,+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partitionFROM posthog.kafka_metrics_avro4WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULLSETTINGSdiff --git a/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics_kafka_metrics.sql b/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics_kafka_metrics.sqlnew file mode 100644index 000000000000..5c39242f5710--- /dev/null+++ b/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics_kafka_metrics.sql@@ -0,0 +1,19 @@+SELECT+ kafka_partition AS _partition,+ kafka_topic AS _topic,+ maxSimpleState(kafka_offset) AS max_offset,+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,+ maxSimpleState(now()) AS max_created_at+FROM+ (+ SELECT+ kafka_source.1 AS kafka_topic,+ kafka_source.2 AS kafka_partition,+ kafka_source.3 AS kafka_offset,+ observed_timestamp+ FROM posthog.metrics4_input+ ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source+ WHERE kafka_topic != ''+ )+GROUP BY+ kafka_partition, kafka_topicdiff --git a/posthog/clickhouse/hcl/sql/dev/apm.sql b/posthog/clickhouse/hcl/sql/dev/apm.sqlindex 8b2d24d21ed7..28f0245e4767 100644--- a/posthog/clickhouse/hcl/sql/dev/apm.sql+++ b/posthog/clickhouse/hcl/sql/dev/apm.sql@@ -95,7 +95,9 @@ CREATE TABLE posthog.metrics4_input (attributes Map(LowCardinality(String), String),_partition UInt32,_topic String,- _offset UInt64+ _offset UInt64,+ _source_topic String,+ _source_partition UInt32) ENGINE = Null();CREATE TABLE posthog.writable_logs34 (time_bucket DateTime MATERIALIZED toStartOfDay(timestamp),@@ -449,7 +451,7 @@ CREATE MATERIALIZED VIEW posthog.kafka_logs34_avro_mv TO posthog.writable_logs34toUInt8(ifNull(pattern_version, 0)) AS pattern_version,_headers.value[indexOf(_headers.name, 'internal-source')] AS internal_sourceFROM posthog.kafka_logs_avro;-CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64) AS SELECT+CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64, _source_topic String, _source_partition UInt32) AS SELECTuuid,toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,ifNull(metric_name, '') AS metric_name,@@ -493,7 +495,9 @@ CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_inpu) AS attributes,_partition,_topic,- _offset+ _offset,+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partitionFROM posthog.kafka_metrics_avro4WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULLSETTINGS@@ -649,6 +653,25 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.wroriginal_expiry_timestampFROM posthog.metrics4_inputWHERE has_labels;+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics_kafka_metrics TO posthog.writable_metrics_kafka_metrics (_partition UInt32, _topic String, max_offset SimpleAggregateFunction(max, UInt64), max_observed_timestamp SimpleAggregateFunction(max, DateTime64(6)), max_created_at SimpleAggregateFunction(max, DateTime)) AS SELECT+ kafka_partition AS _partition,+ kafka_topic AS _topic,+ maxSimpleState(kafka_offset) AS max_offset,+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,+ maxSimpleState(now()) AS max_created_at+FROM+ (+ SELECT+ kafka_source.1 AS kafka_topic,+ kafka_source.2 AS kafka_partition,+ kafka_source.3 AS kafka_offset,+ observed_timestamp+ FROM+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source+ WHERE kafka_topic != ''+ )+GROUP BY+ kafka_partition, kafka_topic;CREATE MATERIALIZED VIEW posthog.ops_query_log_archive_v3_mv TO posthog.writable_query_log_archive_v3 (initial_query_start_time DateTime64(6, 'UTC'), team_id Int64, initial_query_id String, is_initial_query SimpleAggregateFunction(max, UInt8), hostname SimpleAggregateFunction(max, LowCardinality(String)), user SimpleAggregateFunction(max, LowCardinality(String)), type SimpleAggregateFunction(max, Enum8('QueryStart'=1, 'QueryFinish'=2, 'ExceptionBeforeStart'=3, 'ExceptionWhileProcessing'=4)), event_time_microseconds SimpleAggregateFunction(max, DateTime64(6, 'UTC')), query_duration_ms SimpleAggregateFunction(max, UInt64), read_rows SimpleAggregateFunction(sum, UInt64), read_bytes SimpleAggregateFunction(sum, UInt64), written_rows SimpleAggregateFunction(sum, UInt64), written_bytes SimpleAggregateFunction(sum, UInt64), result_rows SimpleAggregateFunction(sum, UInt64), result_bytes SimpleAggregateFunction(sum, UInt64), memory_usage SimpleAggregateFunction(max, UInt64), memory_usage_sum SimpleAggregateFunction(sum, UInt64), peak_threads_usage SimpleAggregateFunction(sum, UInt64), execution_count SimpleAggregateFunction(sum, UInt64), current_database SimpleAggregateFunction(max, LowCardinality(String)), query SimpleAggregateFunction(max, String), normalized_query_hash SimpleAggregateFunction(max, UInt64), query_kind_by_root SimpleAggregateFunction(max, Tuple(UInt8, String)), tables SimpleAggregateFunction(max, Array(String)), exception_code SimpleAggregateFunction(max, Int32), exception SimpleAggregateFunction(max, String), stack_trace SimpleAggregateFunction(max, String), ProfileEvents_SelectedRows SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_RealTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_OSCPUVirtualTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_UserTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_SystemTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedParts SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedMarks SimpleAggregateFunction(sum, UInt64), ProfileEvents_NetworkReceiveBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_NetworkSendBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_ReadBufferFromFileDescriptorReadBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_WriteBufferFromFileDescriptorWriteBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3Clients SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3DeleteObjects SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CopyObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3ListObjects SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3HeadObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3GetObjectAttributes SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CreateMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3UploadPartCopy SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3UploadPart SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3AbortMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CompleteMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3PutObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3GetObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_ReadBufferFromS3Bytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_WriteBufferFromS3Bytes SimpleAggregateFunction(sum, UInt64), lc_org_id SimpleAggregateFunction(max, String), lc_user_id SimpleAggregateFunction(max, Int64), lc_session_id SimpleAggregateFunction(max, String), lc_is_impersonated SimpleAggregateFunction(max, Bool), lc_workflow SimpleAggregateFunction(max, LowCardinality(String)), lc_kind SimpleAggregateFunction(max, LowCardinality(String)), lc_id SimpleAggregateFunction(max, String), lc_route_id SimpleAggregateFunction(max, String), lc_access_method SimpleAggregateFunction(max, LowCardinality(String)), lc_api_key_label SimpleAggregateFunction(max, String), lc_api_key_mask SimpleAggregateFunction(max, String), lc_query_type SimpleAggregateFunction(max, LowCardinality(String)), lc_product SimpleAggregateFunction(max, LowCardinality(String)), lc_chargeable SimpleAggregateFunction(max, Bool), lc_name SimpleAggregateFunction(max, String), lc_request_name SimpleAggregateFunction(max, String), lc_client_query_id SimpleAggregateFunction(max, String), lc_dashboard_id SimpleAggregateFunction(max, Int64), lc_insight_id SimpleAggregateFunction(max, Int64), lc_cohort_id SimpleAggregateFunction(max, Int64), lc_batch_export_id SimpleAggregateFunction(max, String), lc_experiment_id SimpleAggregateFunction(max, Int64), lc_experiment_feature_flag_key SimpleAggregateFunction(max, String), lc_alert_config_id SimpleAggregateFunction(max, String), lc_feature SimpleAggregateFunction(max, LowCardinality(String)), lc_table_id SimpleAggregateFunction(max, String), lc_warehouse_query SimpleAggregateFunction(max, Bool), lc_person_on_events_mode SimpleAggregateFunction(max, LowCardinality(String)), lc_service_name SimpleAggregateFunction(max, LowCardinality(String)), lc_workload SimpleAggregateFunction(max, LowCardinality(String)), lc_query__kind SimpleAggregateFunction(max, LowCardinality(String)), lc_query__query SimpleAggregateFunction(max, String), lc_query SimpleAggregateFunction(max, String), lc_modifiers SimpleAggregateFunction(max, String), lc_temporal__workflow_namespace SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__workflow_type SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__workflow_id SimpleAggregateFunction(max, String), lc_temporal__workflow_run_id SimpleAggregateFunction(max, String), lc_temporal__activity_type SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__activity_id SimpleAggregateFunction(max, String), lc_temporal__attempt SimpleAggregateFunction(max, Int64), lc_dagster__job_name SimpleAggregateFunction(max, LowCardinality(String)), lc_dagster__run_id SimpleAggregateFunction(max, String), lc_dagster__owner SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_name SimpleAggregateFunction(max, String), lc_experiment_metric_name SimpleAggregateFunction(max, String), lc_experiment_execution_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_metric_type SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_funnel_order_type SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_exposures_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_metric_events_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_query_surface SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_precompute_table SimpleAggregateFunction(max, String), lc_experiment_query_group_id SimpleAggregateFunction(max, String), lc_experiment_precompute_skip_reason SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_scan_date_from SimpleAggregateFunction(max, String), lc_experiment_scan_date_to SimpleAggregateFunction(max, String), lc_precompute_window_start SimpleAggregateFunction(max, String), lc_precompute_window_end SimpleAggregateFunction(max, String), lc_plan_fingerprint SimpleAggregateFunction(max, String), lc_estimated_rows SimpleAggregateFunction(max, Int64), lc_estimated_bytes SimpleAggregateFunction(max, Int64)) AS WITHJSONExtract(ql.log_comment, 'Tuple(team_id Int64, org_id String, user_id Int64, session_id String, is_impersonated Bool, workflow String, kind String, id String, route_id String, access_method String, api_key_label String, api_key_mask String, query_type String, product String, chargeable Bool, name String, request_name String, client_query_id String, dashboard_id Int64, insight_id Int64, cohort_id Int64, batch_export_id String, experiment_id Int64, experiment_feature_flag_key String, alert_config_id String, feature String, table_id String, warehouse_query Bool, person_on_events_mode String, service_name String, workload String, experiment_name String, experiment_metric_name String, experiment_execution_path String, experiment_metric_type String, experiment_funnel_order_type String, experiment_exposures_path String, experiment_metric_events_path String, experiment_query_surface String, experiment_precompute_table String, experiment_query_group_id String, experiment_precompute_skip_reason String, experiment_scan_date_from String, experiment_scan_date_to String, precompute_window_start String, precompute_window_end String, plan_fingerprint String, estimated_rows Int64, estimated_bytes Int64, temporal Tuple(workflow_namespace String, workflow_type String, workflow_id String, workflow_run_id String, activity_type String, activity_id String, attempt Int64), dagster Tuple(job_name String, run_id String, tags Tuple(owner String)), query Tuple(kind String, query String, source Tuple(kind String, query String)))') AS lcSELECTdiff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sqlindex b243d5ac1acd..d84cc77e6953 100644--- a/posthog/clickhouse/hcl/sql/local-single/all.sql+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql@@ -1581,7 +1581,9 @@ CREATE TABLE posthog.metrics4_input (attributes Map(LowCardinality(String), String),_partition UInt32,_topic String,- _offset UInt64+ _offset UInt64,+ _source_topic String,+ _source_partition UInt32) ENGINE = Null();CREATE TABLE posthog.metrics4_names (team_id Int32,@@ -4041,6 +4043,15 @@ CREATE TABLE posthog.writable_metrics4_series (time_bucket DateTime MATERIALIZED toStartOfHour(timestamp),original_expiry_timestamp DateTime64(6)) ENGINE = Distributed('logs', 'posthog', 'metrics4_series');+CREATE TABLE posthog.writable_metrics_kafka_metrics (+ _partition UInt32,+ _topic String,+ max_offset SimpleAggregateFunction(max, UInt64),+ max_observed_timestamp SimpleAggregateFunction(max, DateTime64(9)),+ max_timestamp SimpleAggregateFunction(max, DateTime64(9)),+ max_created_at SimpleAggregateFunction(max, DateTime64(9)),+ max_lag SimpleAggregateFunction(max, UInt64)+) ENGINE = Distributed('logs', 'posthog', 'metrics_kafka_metrics');CREATE TABLE posthog.writable_person (id UUID,created_at DateTime64(3),@@ -4965,7 +4976,7 @@ CREATE MATERIALIZED VIEW posthog.kafka_logs_avro_kafka_metrics_mv TO posthog.logFROM posthog.logs34GROUP BY_partition, _topic;-CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64) AS SELECT+CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64, _source_topic String, _source_partition UInt32) AS SELECTuuid,toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,ifNull(metric_name, '') AS metric_name,@@ -5009,7 +5020,9 @@ CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_inpu) AS attributes,_partition,_topic,- _offset+ _offset,+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partitionFROM posthog.kafka_metrics_avro4WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULLSETTINGS@@ -5433,6 +5446,25 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.wroriginal_expiry_timestampFROM posthog.metrics4_inputWHERE has_labels;+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics_kafka_metrics TO posthog.writable_metrics_kafka_metrics (_partition UInt32, _topic String, max_offset SimpleAggregateFunction(max, UInt64), max_observed_timestamp SimpleAggregateFunction(max, DateTime64(6)), max_created_at SimpleAggregateFunction(max, DateTime)) AS SELECT+ kafka_partition AS _partition,+ kafka_topic AS _topic,+ maxSimpleState(kafka_offset) AS max_offset,+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,+ maxSimpleState(now()) AS max_created_at+FROM+ (+ SELECT+ kafka_source.1 AS kafka_topic,+ kafka_source.2 AS kafka_partition,+ kafka_source.3 AS kafka_offset,+ observed_timestamp+ FROM+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source+ WHERE kafka_topic != ''+ )+GROUP BY+ kafka_partition, kafka_topic;CREATE MATERIALIZED VIEW posthog.performance_events_mv TO posthog.writeable_performance_events (uuid UUID, session_id String, window_id String, pageview_id String, distinct_id String, timestamp DateTime64(3), time_origin DateTime64(3, 'UTC'), entry_type LowCardinality(String), name String, team_id Int64, current_url String, start_time Float64, duration Float64, redirect_start Float64, redirect_end Float64, worker_start Float64, fetch_start Float64, domain_lookup_start Float64, domain_lookup_end Float64, connect_start Float64, secure_connection_start Float64, connect_end Float64, request_start Float64, response_start Float64, response_end Float64, decoded_body_size Int64, encoded_body_size Int64, initiator_type LowCardinality(String), next_hop_protocol LowCardinality(String), render_blocking_status LowCardinality(String), response_status Int64, transfer_size Int64, largest_contentful_paint_element String, largest_contentful_paint_render_time Float64, largest_contentful_paint_load_time Float64, largest_contentful_paint_size Float64, largest_contentful_paint_id String, largest_contentful_paint_url String, dom_complete Float64, dom_content_loaded_event Float64, dom_interactive Float64, load_event_end Float64, load_event_start Float64, redirect_count Int64, navigation_type LowCardinality(String), unload_event_end Float64, unload_event_start Float64, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECTuuid,session_id,diff --git a/posthog/clickhouse/hcl/sql/prod-eu/apm.sql b/posthog/clickhouse/hcl/sql/prod-eu/apm.sqlindex 2dd0f8791302..bbeec37e76e4 100644--- a/posthog/clickhouse/hcl/sql/prod-eu/apm.sql+++ b/posthog/clickhouse/hcl/sql/prod-eu/apm.sql@@ -71,7 +71,9 @@ CREATE TABLE posthog.metrics4_input (attributes Map(LowCardinality(String), String),_partition UInt32,_topic String,- _offset UInt64+ _offset UInt64,+ _source_topic String,+ _source_partition UInt32) ENGINE = Null();CREATE TABLE posthog.writable_logs34 (time_bucket DateTime MATERIALIZED toStartOfDay(timestamp),@@ -171,6 +173,15 @@ CREATE TABLE posthog.writable_metrics4_series (time_bucket DateTime MATERIALIZED toStartOfHour(timestamp),original_expiry_timestamp DateTime64(6)) ENGINE = Distributed('logs', 'posthog', 'metrics4_series');+CREATE TABLE posthog.writable_metrics_kafka_metrics (+ _partition UInt32,+ _topic String,+ max_offset SimpleAggregateFunction(max, UInt64),+ max_observed_timestamp SimpleAggregateFunction(max, DateTime64(9)),+ max_timestamp SimpleAggregateFunction(max, DateTime64(9)),+ max_created_at SimpleAggregateFunction(max, DateTime64(9)),+ max_lag SimpleAggregateFunction(max, UInt64)+) ENGINE = Distributed('logs', 'posthog', 'metrics_kafka_metrics');CREATE TABLE posthog.writable_query_log_archive_v3 (initial_query_start_time DateTime64(6, 'UTC'),team_id Int64,@@ -323,7 +334,7 @@ CREATE MATERIALIZED VIEW posthog.kafka_logs34_avro_mv TO posthog.writable_logs34toUInt8(ifNull(pattern_version, 0)) AS pattern_version,_headers.value[indexOf(_headers.name, 'internal-source')] AS internal_sourceFROM posthog.kafka_logs_avro;-CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64) AS SELECT+CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64, _source_topic String, _source_partition UInt32) AS SELECTuuid,toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,ifNull(metric_name, '') AS metric_name,@@ -367,7 +378,9 @@ CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_inpu) AS attributes,_partition,_topic,- _offset+ _offset,+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partitionFROM posthog.kafka_metrics_avro4WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULLSETTINGS@@ -485,6 +498,25 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.wroriginal_expiry_timestampFROM posthog.metrics4_inputWHERE has_labels;+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics_kafka_metrics TO posthog.writable_metrics_kafka_metrics (_partition UInt32, _topic String, max_offset SimpleAggregateFunction(max, UInt64), max_observed_timestamp SimpleAggregateFunction(max, DateTime64(6)), max_created_at SimpleAggregateFunction(max, DateTime)) AS SELECT+ kafka_partition AS _partition,+ kafka_topic AS _topic,+ maxSimpleState(kafka_offset) AS max_offset,+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,+ maxSimpleState(now()) AS max_created_at+FROM+ (+ SELECT+ kafka_source.1 AS kafka_topic,+ kafka_source.2 AS kafka_partition,+ kafka_source.3 AS kafka_offset,+ observed_timestamp+ FROM+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source+ WHERE kafka_topic != ''+ )+GROUP BY+ kafka_partition, kafka_topic;CREATE MATERIALIZED VIEW posthog.ops_query_log_archive_v3_mv TO posthog.writable_query_log_archive_v3 (initial_query_start_time DateTime64(6, 'UTC'), team_id Int64, initial_query_id String, is_initial_query SimpleAggregateFunction(max, UInt8), hostname SimpleAggregateFunction(max, LowCardinality(String)), user SimpleAggregateFunction(max, LowCardinality(String)), type SimpleAggregateFunction(max, Enum8('QueryStart'=1, 'QueryFinish'=2, 'ExceptionBeforeStart'=3, 'ExceptionWhileProcessing'=4)), event_time_microseconds SimpleAggregateFunction(max, DateTime64(6, 'UTC')), query_duration_ms SimpleAggregateFunction(max, UInt64), read_rows SimpleAggregateFunction(sum, UInt64), read_bytes SimpleAggregateFunction(sum, UInt64), written_rows SimpleAggregateFunction(sum, UInt64), written_bytes SimpleAggregateFunction(sum, UInt64), result_rows SimpleAggregateFunction(sum, UInt64), result_bytes SimpleAggregateFunction(sum, UInt64), memory_usage SimpleAggregateFunction(max, UInt64), memory_usage_sum SimpleAggregateFunction(sum, UInt64), peak_threads_usage SimpleAggregateFunction(sum, UInt64), execution_count SimpleAggregateFunction(sum, UInt64), current_database SimpleAggregateFunction(max, LowCardinality(String)), query SimpleAggregateFunction(max, String), normalized_query_hash SimpleAggregateFunction(max, UInt64), query_kind_by_root SimpleAggregateFunction(max, Tuple(UInt8, String)), tables SimpleAggregateFunction(max, Array(String)), exception_code SimpleAggregateFunction(max, Int32), exception SimpleAggregateFunction(max, String), stack_trace SimpleAggregateFunction(max, String), ProfileEvents_SelectedRows SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_RealTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_OSCPUVirtualTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_UserTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_SystemTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedParts SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedMarks SimpleAggregateFunction(sum, UInt64), ProfileEvents_NetworkReceiveBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_NetworkSendBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_ReadBufferFromFileDescriptorReadBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_WriteBufferFromFileDescriptorWriteBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3Clients SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3DeleteObjects SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CopyObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3ListObjects SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3HeadObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3GetObjectAttributes SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CreateMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3UploadPartCopy SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3UploadPart SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3AbortMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CompleteMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3PutObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3GetObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_ReadBufferFromS3Bytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_WriteBufferFromS3Bytes SimpleAggregateFunction(sum, UInt64), lc_org_id SimpleAggregateFunction(max, String), lc_user_id SimpleAggregateFunction(max, Int64), lc_session_id SimpleAggregateFunction(max, String), lc_is_impersonated SimpleAggregateFunction(max, Bool), lc_workflow SimpleAggregateFunction(max, LowCardinality(String)), lc_kind SimpleAggregateFunction(max, LowCardinality(String)), lc_id SimpleAggregateFunction(max, String), lc_route_id SimpleAggregateFunction(max, String), lc_access_method SimpleAggregateFunction(max, LowCardinality(String)), lc_api_key_label SimpleAggregateFunction(max, String), lc_api_key_mask SimpleAggregateFunction(max, String), lc_query_type SimpleAggregateFunction(max, LowCardinality(String)), lc_product SimpleAggregateFunction(max, LowCardinality(String)), lc_chargeable SimpleAggregateFunction(max, Bool), lc_name SimpleAggregateFunction(max, String), lc_request_name SimpleAggregateFunction(max, String), lc_client_query_id SimpleAggregateFunction(max, String), lc_dashboard_id SimpleAggregateFunction(max, Int64), lc_insight_id SimpleAggregateFunction(max, Int64), lc_cohort_id SimpleAggregateFunction(max, Int64), lc_batch_export_id SimpleAggregateFunction(max, String), lc_experiment_id SimpleAggregateFunction(max, Int64), lc_experiment_feature_flag_key SimpleAggregateFunction(max, String), lc_alert_config_id SimpleAggregateFunction(max, String), lc_feature SimpleAggregateFunction(max, LowCardinality(String)), lc_table_id SimpleAggregateFunction(max, String), lc_warehouse_query SimpleAggregateFunction(max, Bool), lc_person_on_events_mode SimpleAggregateFunction(max, LowCardinality(String)), lc_service_name SimpleAggregateFunction(max, LowCardinality(String)), lc_workload SimpleAggregateFunction(max, LowCardinality(String)), lc_query__kind SimpleAggregateFunction(max, LowCardinality(String)), lc_query__query SimpleAggregateFunction(max, String), lc_query SimpleAggregateFunction(max, String), lc_modifiers SimpleAggregateFunction(max, String), lc_temporal__workflow_namespace SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__workflow_type SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__workflow_id SimpleAggregateFunction(max, String), lc_temporal__workflow_run_id SimpleAggregateFunction(max, String), lc_temporal__activity_type SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__activity_id SimpleAggregateFunction(max, String), lc_temporal__attempt SimpleAggregateFunction(max, Int64), lc_dagster__job_name SimpleAggregateFunction(max, LowCardinality(String)), lc_dagster__run_id SimpleAggregateFunction(max, String), lc_dagster__owner SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_name SimpleAggregateFunction(max, String), lc_experiment_metric_name SimpleAggregateFunction(max, String), lc_experiment_execution_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_metric_type SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_funnel_order_type SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_exposures_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_metric_events_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_query_surface SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_precompute_table SimpleAggregateFunction(max, String), lc_experiment_query_group_id SimpleAggregateFunction(max, String), lc_experiment_precompute_skip_reason SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_scan_date_from SimpleAggregateFunction(max, String), lc_experiment_scan_date_to SimpleAggregateFunction(max, String), lc_precompute_window_start SimpleAggregateFunction(max, String), lc_precompute_window_end SimpleAggregateFunction(max, String), lc_plan_fingerprint SimpleAggregateFunction(max, String), lc_estimated_rows SimpleAggregateFunction(max, Int64), lc_estimated_bytes SimpleAggregateFunction(max, Int64)) AS WITHJSONExtract(ql.log_comment, 'Tuple(team_id Int64, org_id String, user_id Int64, session_id String, is_impersonated Bool, workflow String, kind String, id String, route_id String, access_method String, api_key_label String, api_key_mask String, query_type String, product String, chargeable Bool, name String, request_name String, client_query_id String, dashboard_id Int64, insight_id Int64, cohort_id Int64, batch_export_id String, experiment_id Int64, experiment_feature_flag_key String, alert_config_id String, feature String, table_id String, warehouse_query Bool, person_on_events_mode String, service_name String, workload String, experiment_name String, experiment_metric_name String, experiment_execution_path String, experiment_metric_type String, experiment_funnel_order_type String, experiment_exposures_path String, experiment_metric_events_path String, experiment_query_surface String, experiment_precompute_table String, experiment_query_group_id String, experiment_precompute_skip_reason String, experiment_scan_date_from String, experiment_scan_date_to String, precompute_window_start String, precompute_window_end String, plan_fingerprint String, estimated_rows Int64, estimated_bytes Int64, temporal Tuple(workflow_namespace String, workflow_type String, workflow_id String, workflow_run_id String, activity_type String, activity_id String, attempt Int64), dagster Tuple(job_name String, run_id String, tags Tuple(owner String)), query Tuple(kind String, query String, source Tuple(kind String, query String)))') AS lcSELECTdiff --git a/posthog/clickhouse/hcl/sql/prod-us/apm.sql b/posthog/clickhouse/hcl/sql/prod-us/apm.sqlindex 5e6187899412..92e6f9ae1e8b 100644--- a/posthog/clickhouse/hcl/sql/prod-us/apm.sql+++ b/posthog/clickhouse/hcl/sql/prod-us/apm.sql@@ -71,7 +71,9 @@ CREATE TABLE posthog.metrics4_input (attributes Map(LowCardinality(String), String),_partition UInt32,_topic String,- _offset UInt64+ _offset UInt64,+ _source_topic String,+ _source_partition UInt32) ENGINE = Null();CREATE TABLE posthog.writable_logs34 (time_bucket DateTime MATERIALIZED toStartOfDay(timestamp),@@ -171,6 +173,15 @@ CREATE TABLE posthog.writable_metrics4_series (time_bucket DateTime MATERIALIZED toStartOfHour(timestamp),original_expiry_timestamp DateTime64(6)) ENGINE = Distributed('logs', 'posthog', 'metrics4_series');+CREATE TABLE posthog.writable_metrics_kafka_metrics (+ _partition UInt32,+ _topic String,+ max_offset SimpleAggregateFunction(max, UInt64),+ max_observed_timestamp SimpleAggregateFunction(max, DateTime64(9)),+ max_timestamp SimpleAggregateFunction(max, DateTime64(9)),+ max_created_at SimpleAggregateFunction(max, DateTime64(9)),+ max_lag SimpleAggregateFunction(max, UInt64)+) ENGINE = Distributed('logs', 'posthog', 'metrics_kafka_metrics');CREATE TABLE posthog.writable_query_log_archive_v3 (initial_query_start_time DateTime64(6, 'UTC'),team_id Int64,@@ -323,7 +334,7 @@ CREATE MATERIALIZED VIEW posthog.kafka_logs34_avro_mv TO posthog.writable_logs34toUInt8(ifNull(pattern_version, 0)) AS pattern_version,_headers.value[indexOf(_headers.name, 'internal-source')] AS internal_sourceFROM posthog.kafka_logs_avro;-CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64) AS SELECT+CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64, _source_topic String, _source_partition UInt32) AS SELECTuuid,toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,ifNull(metric_name, '') AS metric_name,@@ -367,7 +378,9 @@ CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_inpu) AS attributes,_partition,_topic,- _offset+ _offset,+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partitionFROM posthog.kafka_metrics_avro4WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULLSETTINGS@@ -485,6 +498,25 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.wroriginal_expiry_timestampFROM posthog.metrics4_inputWHERE has_labels;+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics_kafka_metrics TO posthog.writable_metrics_kafka_metrics (_partition UInt32, _topic String, max_offset SimpleAggregateFunction(max, UInt64), max_observed_timestamp SimpleAggregateFunction(max, DateTime64(6)), max_created_at SimpleAggregateFunction(max, DateTime)) AS SELECT+ kafka_partition AS _partition,+ kafka_topic AS _topic,+ maxSimpleState(kafka_offset) AS max_offset,+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,+ maxSimpleState(now()) AS max_created_at+FROM+ (+ SELECT+ kafka_source.1 AS kafka_topic,+ kafka_source.2 AS kafka_partition,+ kafka_source.3 AS kafka_offset,+ observed_timestamp+ FROM+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source+ WHERE kafka_topic != ''+ )+GROUP BY+ kafka_partition, kafka_topic;CREATE MATERIALIZED VIEW posthog.ops_query_log_archive_v3_mv TO posthog.writable_query_log_archive_v3 (initial_query_start_time DateTime64(6, 'UTC'), team_id Int64, initial_query_id String, is_initial_query SimpleAggregateFunction(max, UInt8), hostname SimpleAggregateFunction(max, LowCardinality(String)), user SimpleAggregateFunction(max, LowCardinality(String)), type SimpleAggregateFunction(max, Enum8('QueryStart'=1, 'QueryFinish'=2, 'ExceptionBeforeStart'=3, 'ExceptionWhileProcessing'=4)), event_time_microseconds SimpleAggregateFunction(max, DateTime64(6, 'UTC')), query_duration_ms SimpleAggregateFunction(max, UInt64), read_rows SimpleAggregateFunction(sum, UInt64), read_bytes SimpleAggregateFunction(sum, UInt64), written_rows SimpleAggregateFunction(sum, UInt64), written_bytes SimpleAggregateFunction(sum, UInt64), result_rows SimpleAggregateFunction(sum, UInt64), result_bytes SimpleAggregateFunction(sum, UInt64), memory_usage SimpleAggregateFunction(max, UInt64), memory_usage_sum SimpleAggregateFunction(sum, UInt64), peak_threads_usage SimpleAggregateFunction(sum, UInt64), execution_count SimpleAggregateFunction(sum, UInt64), current_database SimpleAggregateFunction(max, LowCardinality(String)), query SimpleAggregateFunction(max, String), normalized_query_hash SimpleAggregateFunction(max, UInt64), query_kind_by_root SimpleAggregateFunction(max, Tuple(UInt8, String)), tables SimpleAggregateFunction(max, Array(String)), exception_code SimpleAggregateFunction(max, Int32), exception SimpleAggregateFunction(max, String), stack_trace SimpleAggregateFunction(max, String), ProfileEvents_SelectedRows SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_RealTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_OSCPUVirtualTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_UserTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_SystemTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedParts SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedMarks SimpleAggregateFunction(sum, UInt64), ProfileEvents_NetworkReceiveBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_NetworkSendBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_ReadBufferFromFileDescriptorReadBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_WriteBufferFromFileDescriptorWriteBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3Clients SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3DeleteObjects SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CopyObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3ListObjects SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3HeadObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3GetObjectAttributes SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CreateMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3UploadPartCopy SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3UploadPart SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3AbortMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CompleteMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3PutObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3GetObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_ReadBufferFromS3Bytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_WriteBufferFromS3Bytes SimpleAggregateFunction(sum, UInt64), lc_org_id SimpleAggregateFunction(max, String), lc_user_id SimpleAggregateFunction(max, Int64), lc_session_id SimpleAggregateFunction(max, String), lc_is_impersonated SimpleAggregateFunction(max, Bool), lc_workflow SimpleAggregateFunction(max, LowCardinality(String)), lc_kind SimpleAggregateFunction(max, LowCardinality(String)), lc_id SimpleAggregateFunction(max, String), lc_route_id SimpleAggregateFunction(max, String), lc_access_method SimpleAggregateFunction(max, LowCardinality(String)), lc_api_key_label SimpleAggregateFunction(max, String), lc_api_key_mask SimpleAggregateFunction(max, String), lc_query_type SimpleAggregateFunction(max, LowCardinality(String)), lc_product SimpleAggregateFunction(max, LowCardinality(String)), lc_chargeable SimpleAggregateFunction(max, Bool), lc_name SimpleAggregateFunction(max, String), lc_request_name SimpleAggregateFunction(max, String), lc_client_query_id SimpleAggregateFunction(max, String), lc_dashboard_id SimpleAggregateFunction(max, Int64), lc_insight_id SimpleAggregateFunction(max, Int64), lc_cohort_id SimpleAggregateFunction(max, Int64), lc_batch_export_id SimpleAggregateFunction(max, String), lc_experiment_id SimpleAggregateFunction(max, Int64), lc_experiment_feature_flag_key SimpleAggregateFunction(max, String), lc_alert_config_id SimpleAggregateFunction(max, String), lc_feature SimpleAggregateFunction(max, LowCardinality(String)), lc_table_id SimpleAggregateFunction(max, String), lc_warehouse_query SimpleAggregateFunction(max, Bool), lc_person_on_events_mode SimpleAggregateFunction(max, LowCardinality(String)), lc_service_name SimpleAggregateFunction(max, LowCardinality(String)), lc_workload SimpleAggregateFunction(max, LowCardinality(String)), lc_query__kind SimpleAggregateFunction(max, LowCardinality(String)), lc_query__query SimpleAggregateFunction(max, String), lc_query SimpleAggregateFunction(max, String), lc_modifiers SimpleAggregateFunction(max, String), lc_temporal__workflow_namespace SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__workflow_type SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__workflow_id SimpleAggregateFunction(max, String), lc_temporal__workflow_run_id SimpleAggregateFunction(max, String), lc_temporal__activity_type SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__activity_id SimpleAggregateFunction(max, String), lc_temporal__attempt SimpleAggregateFunction(max, Int64), lc_dagster__job_name SimpleAggregateFunction(max, LowCardinality(String)), lc_dagster__run_id SimpleAggregateFunction(max, String), lc_dagster__owner SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_name SimpleAggregateFunction(max, String), lc_experiment_metric_name SimpleAggregateFunction(max, String), lc_experiment_execution_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_metric_type SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_funnel_order_type SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_exposures_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_metric_events_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_query_surface SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_precompute_table SimpleAggregateFunction(max, String), lc_experiment_query_group_id SimpleAggregateFunction(max, String), lc_experiment_precompute_skip_reason SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_scan_date_from SimpleAggregateFunction(max, String), lc_experiment_scan_date_to SimpleAggregateFunction(max, String), lc_precompute_window_start SimpleAggregateFunction(max, String), lc_precompute_window_end SimpleAggregateFunction(max, String), lc_plan_fingerprint SimpleAggregateFunction(max, String), lc_estimated_rows SimpleAggregateFunction(max, Int64), lc_estimated_bytes SimpleAggregateFunction(max, Int64)) AS WITHJSONExtract(ql.log_comment, 'Tuple(team_id Int64, org_id String, user_id Int64, session_id String, is_impersonated Bool, workflow String, kind String, id String, route_id String, access_method String, api_key_label String, api_key_mask String, query_type String, product String, chargeable Bool, name String, request_name String, client_query_id String, dashboard_id Int64, insight_id Int64, cohort_id Int64, batch_export_id String, experiment_id Int64, experiment_feature_flag_key String, alert_config_id String, feature String, table_id String, warehouse_query Bool, person_on_events_mode String, service_name String, workload String, experiment_name String, experiment_metric_name String, experiment_execution_path String, experiment_metric_type String, experiment_funnel_order_type String, experiment_exposures_path String, experiment_metric_events_path String, experiment_query_surface String, experiment_precompute_table String, experiment_query_group_id String, experiment_precompute_skip_reason String, experiment_scan_date_from String, experiment_scan_date_to String, precompute_window_start String, precompute_window_end String, plan_fingerprint String, estimated_rows Int64, estimated_bytes Int64, temporal Tuple(workflow_namespace String, workflow_type String, workflow_id String, workflow_run_id String, activity_type String, activity_id String, attempt Int64), dagster Tuple(job_name String, run_id String, tags Tuple(owner String)), query Tuple(kind String, query String, source Tuple(kind String, query String)))') AS lcSELECTdiff --git a/posthog/clickhouse/metrics/__init__.py b/posthog/clickhouse/metrics/__init__.pyindex 9915279e3440..bbd29551d8e0 100644--- a/posthog/clickhouse/metrics/__init__.py+++ b/posthog/clickhouse/metrics/__init__.py@@ -53,6 +53,8 @@ from .metrics4 import (METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV,METRICS4_INPUT_TO_METRICS4_SAMPLES_MV,METRICS4_INPUT_TO_METRICS4_SERIES_MV,+ METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV,+ METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT,METRICS4_NAMES_TABLE_SQL,METRICS4_SAMPLES_TABLE_SQL,METRICS4_SERIES_TABLE_SQL,@@ -61,6 +63,7 @@ from .metrics4 import (WRITABLE_METRICS4_NAMES_TABLE_SQL,WRITABLE_METRICS4_SAMPLES_TABLE_SQL,WRITABLE_METRICS4_SERIES_TABLE_SQL,+ WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL,)__all__ = [@@ -80,6 +83,8 @@ __all__ = ["METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV","METRICS4_INPUT_TO_METRICS4_SAMPLES_MV","METRICS4_INPUT_TO_METRICS4_SERIES_MV",+ "METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV",+ "METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT","METRICS4_NAMES_TABLE_SQL","METRICS4_SAMPLES_TABLE_SQL","METRICS4_SERIES_TABLE_SQL",@@ -88,6 +93,7 @@ __all__ = ["WRITABLE_METRICS4_NAMES_TABLE_SQL","WRITABLE_METRICS4_SAMPLES_TABLE_SQL","WRITABLE_METRICS4_SERIES_TABLE_SQL",+ "WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL","KAFKA_METRICS_AVRO2_MV","KAFKA_METRICS_AVRO2_TABLE_SQL","METRIC_ATTRIBUTES2_DISTRIBUTED_TABLE_SQL",diff --git a/posthog/clickhouse/metrics/kafka_metrics.py b/posthog/clickhouse/metrics/kafka_metrics.pyindex 729a59d13110..589f17c48399 100644--- a/posthog/clickhouse/metrics/kafka_metrics.py+++ b/posthog/clickhouse/metrics/kafka_metrics.py@@ -140,17 +140,20 @@ SETTINGS"""-def METRICS_KAFKA_METRICS_TABLE_SQL():- return f"""-CREATE TABLE IF NOT EXISTS {settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE}.{KAFKA_METRICS_TABLE_NAME}-(- `_partition` UInt32,+KAFKA_METRICS_COLUMNS = """ `_partition` UInt32,`_topic` String,`max_offset` SimpleAggregateFunction(max, UInt64),`max_observed_timestamp` SimpleAggregateFunction(max, DateTime64(9)),`max_timestamp` SimpleAggregateFunction(max, DateTime64(9)),`max_created_at` SimpleAggregateFunction(max, DateTime64(9)),- `max_lag` SimpleAggregateFunction(max, UInt64)+ `max_lag` SimpleAggregateFunction(max, UInt64)"""+++def METRICS_KAFKA_METRICS_TABLE_SQL():+ return f"""+CREATE TABLE IF NOT EXISTS {settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE}.{KAFKA_METRICS_TABLE_NAME}+(+{KAFKA_METRICS_COLUMNS})ENGINE = {AggregatingMergeTree(KAFKA_METRICS_TABLE_NAME, replication_scheme=ReplicationScheme.REPLICATED)}ORDER BY (_topic, _partition)diff --git a/posthog/clickhouse/metrics/metrics2.py b/posthog/clickhouse/metrics/metrics2.pyindex df8a9c3a4466..fe0eb68bc491 100644--- a/posthog/clickhouse/metrics/metrics2.py+++ b/posthog/clickhouse/metrics/metrics2.py@@ -83,7 +83,11 @@ def KAFKA_METRICS_AVRO2_TABLE_SQL() -> str:return kafka_metrics_avro_table_sql(KAFKA_TABLE_NAME, KAFKA_GROUP)-def metrics_input_table_sql(table_name: str) -> str:+def metrics_input_table_sql(table_name: str, *, source_partition: bool = False) -> str:+ # Only the metrics4 chain records where the Node consumer read each row.+ source_partition_columns = (+ ",\n `_source_topic` String,\n `_source_partition` UInt32" if source_partition else ""+ )return f"""CREATE TABLE IF NOT EXISTS {_db()}.{table_name}(@@ -113,7 +117,7 @@ CREATE TABLE IF NOT EXISTS {_db()}.{table_name}`attributes` Map(LowCardinality(String), String),`_partition` UInt32,`_topic` String,- `_offset` UInt64+ `_offset` UInt64{source_partition_columns})ENGINE = Null"""@@ -271,8 +275,14 @@ def METRIC_ATTRIBUTES2_DISTRIBUTED_TABLE_SQL() -> str:return _distributed_sql(METRIC_ATTRIBUTES_DISTRIBUTED_TABLE_NAME, METRIC_ATTRIBUTES2_TABLE_NAME)-def kafka_metrics_avro_mv_select(table_name: str) -> str:+def kafka_metrics_avro_mv_select(table_name: str, *, source_partition: bool = False) -> str:db = _db()+ source_partition_columns = (+ ",\n _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,"+ "\n toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partition"+ if source_partition+ else ""+ )sorted_resource_attributes = "mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), resource_attributes))"sorted_attributes = "mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), attributes))"labelled = "toBool(ifNull(has_labels, 1))"@@ -306,7 +316,7 @@ def kafka_metrics_avro_mv_select(table_name: str) -> str:if({labelled}, {sorted_attributes}, CAST(map(), 'Map(String, String)')) AS attributes,_partition,_topic,- _offset+ _offset{source_partition_columns}FROM {db}.{table_name}WHERE {table_name}.series_fingerprint IS NOT NULLSETTINGSdiff --git a/posthog/clickhouse/metrics/metrics4.py b/posthog/clickhouse/metrics/metrics4.pyindex a5d08a8b1f34..373a20d56d05 100644--- a/posthog/clickhouse/metrics/metrics4.py+++ b/posthog/clickhouse/metrics/metrics4.py@@ -32,6 +32,7 @@ from django.conf import settingsfrom posthog.clickhouse.table_engines import AggregatingMergeTree, Distributed, ReplacingMergeTree, ReplicationScheme+from .kafka_metrics import KAFKA_METRICS_COLUMNS, KAFKA_METRICS_TABLE_NAMEfrom .metrics2 import kafka_metrics_avro_mv_select, kafka_metrics_avro_table_sql, metrics_input_table_sqlKAFKA_METRICS4_TABLE_NAME = "kafka_metrics_avro4"@@ -47,6 +48,7 @@ WRITABLE_METRICS4_SAMPLES_TABLE_NAME = "writable_metrics4_samples"WRITABLE_METRICS4_SERIES_TABLE_NAME = "writable_metrics4_series"WRITABLE_METRICS4_NAMES_TABLE_NAME = "writable_metrics4_names"WRITABLE_METRICS4_ATTRIBUTES_TABLE_NAME = "writable_metrics4_attributes"+WRITABLE_METRICS_KAFKA_METRICS_TABLE_NAME = f"writable_{KAFKA_METRICS_TABLE_NAME}"METRICS4_MAX_SAMPLES_PER_SERIES_HOUR = 10_000METRICS4_VIEW_NAME = "metrics4_view"# The first hour that metrics4_samples serves. The view skips earlier rows, which overlap the old metrics2 data.@@ -94,14 +96,14 @@ def KAFKA_METRICS_AVRO4_TABLE_SQL() -> str:def METRICS4_INPUT_TABLE_SQL() -> str:- return metrics_input_table_sql(METRICS4_INPUT_TABLE_NAME)+ return metrics_input_table_sql(METRICS4_INPUT_TABLE_NAME, source_partition=True)def KAFKA_METRICS_AVRO4_MV() -> str:db = _db()return f"""CREATE MATERIALIZED VIEW IF NOT EXISTS {db}.{KAFKA_METRICS4_TABLE_NAME}_mv TO {db}.{METRICS4_INPUT_TABLE_NAME}-AS {kafka_metrics_avro_mv_select(KAFKA_METRICS4_TABLE_NAME)}+AS {kafka_metrics_avro_mv_select(KAFKA_METRICS4_TABLE_NAME, source_partition=True)}"""@@ -194,6 +196,12 @@ def WRITABLE_METRICS4_ATTRIBUTES_TABLE_SQL() -> str:)+def WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL() -> str:+ return _writable_table_sql(+ WRITABLE_METRICS_KAFKA_METRICS_TABLE_NAME, KAFKA_METRICS_TABLE_NAME, KAFKA_METRICS_COLUMNS+ )++def METRICS4_SAMPLES_TABLE_SQL() -> str:arrays = ",\n".join(f" `{name}_arr` SimpleAggregateFunction(groupArrayArray({METRICS4_MAX_SAMPLES_PER_SERIES_HOUR}), Array({element_type})){_ARRAY_CODECS.get(name, '')}"@@ -388,6 +396,45 @@ GROUP BY"""+def METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT() -> str:+ # Each row counts toward its clickhouse_metrics partition and the ingestion-metrics partition+ # the Node consumer read it from. Both topics partition at random, so lag on either hop would+ # hide behind the other. A row without the source header has an empty source topic, and the+ # WHERE skips it, so it cannot add a partition that never moves. The source offset is unknown.+ return f"""SELECT+ kafka_partition AS _partition,+ kafka_topic AS _topic,+ maxSimpleState(kafka_offset) AS max_offset,+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,+ maxSimpleState(now()) AS max_created_at+FROM+(+ SELECT+ kafka_source.1 AS kafka_topic,+ kafka_source.2 AS kafka_partition,+ kafka_source.3 AS kafka_offset,+ observed_timestamp+ FROM {_db()}.{METRICS4_INPUT_TABLE_NAME}+ ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source+ WHERE kafka_topic != ''+)+GROUP BY kafka_partition, kafka_topic"""+++def METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV() -> str:+ """The newest capture time and offset each Kafka partition has delivered, one row per insert block.++ The floor across partitions is how far every partition has reached, which metrics alerts read.+ `max_timestamp` and `max_lag` stay unset because no reader uses them, and a sender that dates+ points ahead would pin `max_timestamp`.+ """+ db = _db()+ return f"""+CREATE MATERIALIZED VIEW IF NOT EXISTS {db}.{METRICS4_INPUT_TABLE_NAME}_to_{KAFKA_METRICS_TABLE_NAME} TO {db}.{WRITABLE_METRICS_KAFKA_METRICS_TABLE_NAME}+AS {METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT()}+"""++def METRICS4_INPUT_TO_METRICS4_SERIES_MV() -> str:db = _db()return f"""diff --git a/posthog/clickhouse/migrations/0370_metrics4_kafka_metrics_checkpoint.py b/posthog/clickhouse/migrations/0370_metrics4_kafka_metrics_checkpoint.pynew file mode 100644index 000000000000..814957e3fc2f--- /dev/null+++ b/posthog/clickhouse/migrations/0370_metrics4_kafka_metrics_checkpoint.py@@ -0,0 +1,34 @@+from django.conf import settings++from posthog.clickhouse.client.connection import NodeRole+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions+from posthog.clickhouse.metrics import (+ KAFKA_METRICS_AVRO4_MV,+ METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV,+ WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL,+)++DB = settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE++# metrics_kafka_metrics already exists on the logs cluster: 0309 created it, and 0340 dropped the+# only view that wrote to it. The new view counts each row toward both Kafka partitions it passed+# through. The Node consumer sends the ingestion-metrics partition in the `source_topic` and+# `source_partition` headers.+#+# The source columns go on metrics4_input before the views that read or write them, and the+# writable table goes before the view that inserts into it. The Kafka view gets new output columns,+# so it is dropped and created again. The Kafka table stays, with the same consumer group, so the+# gap causes lag and no data loss.+operations = [+ run_sql_with_exceptions(+ f"ALTER TABLE {DB}.metrics4_input ADD COLUMN IF NOT EXISTS _source_topic String, "+ "ADD COLUMN IF NOT EXISTS _source_partition UInt32",+ node_roles=[NodeRole.APM],+ sharded=False,+ is_alter_on_replicated_table=False,+ ),+ run_sql_with_exceptions(WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL(), node_roles=[NodeRole.APM]),+ run_sql_with_exceptions(METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV(), node_roles=[NodeRole.APM]),+ run_sql_with_exceptions(f"DROP TABLE IF EXISTS {DB}.kafka_metrics_avro4_mv", node_roles=[NodeRole.APM]),+ run_sql_with_exceptions(KAFKA_METRICS_AVRO4_MV(), node_roles=[NodeRole.APM]),+]diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txtindex 731ac00dcf4d..754af8720b55 100644--- a/posthog/clickhouse/migrations/max_migration.txt+++ b/posthog/clickhouse/migrations/max_migration.txt@@ -1 +1 @@-0369_person_group_membership_ingestion+0370_metrics4_kafka_metrics_checkpointdiff --git a/posthog/clickhouse/schema.py b/posthog/clickhouse/schema.pyindex bce58280a253..114f764c433b 100644--- a/posthog/clickhouse/schema.py+++ b/posthog/clickhouse/schema.py@@ -69,15 +69,18 @@ from posthog.clickhouse.metrics import (METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV,METRICS4_INPUT_TO_METRICS4_SAMPLES_MV,METRICS4_INPUT_TO_METRICS4_SERIES_MV,+ METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV,METRICS4_NAMES_TABLE_SQL,METRICS4_SAMPLES_TABLE_SQL,METRICS4_SERIES_TABLE_SQL,METRICS4_VIEW_SQL,METRICS_DISTRIBUTED_TABLE_SQL,+ METRICS_KAFKA_METRICS_TABLE_SQL,WRITABLE_METRICS4_ATTRIBUTES_TABLE_SQL,WRITABLE_METRICS4_NAMES_TABLE_SQL,WRITABLE_METRICS4_SAMPLES_TABLE_SQL,WRITABLE_METRICS4_SERIES_TABLE_SQL,+ WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL,)from posthog.clickhouse.plugin_log_entries import (KAFKA_PLUGIN_LOG_ENTRIES_TABLE_SQL,@@ -526,6 +529,7 @@ CREATE_MERGETREE_TABLE_QUERIES = (METRICS4_SERIES_TABLE_SQL,METRICS4_NAMES_TABLE_SQL,METRICS4_ATTRIBUTES_TABLE_SQL,+ METRICS_KAFKA_METRICS_TABLE_SQL,TOPHOG_DATA_TABLE_SQL,BILLING_USAGE_RECORDS_DATA_TABLE_SQL,AI_EVENTS_DATA_TABLE_SQL,@@ -631,6 +635,7 @@ CREATE_DISTRIBUTED_TABLE_QUERIES = (WRITABLE_METRICS4_SERIES_TABLE_SQL,WRITABLE_METRICS4_NAMES_TABLE_SQL,WRITABLE_METRICS4_ATTRIBUTES_TABLE_SQL,+ WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL,WRITABLE_TOPHOG_TABLE_SQL,DISTRIBUTED_TOPHOG_TABLE_SQL,DISTRIBUTED_BILLING_USAGE_RECORDS_TABLE_SQL,@@ -729,6 +734,7 @@ CREATE_MV_TABLE_QUERIES = (METRICS4_INPUT_TO_METRICS4_NAMES_MV,METRICS4_INPUT_TO_METRICS4_ATTRIBUTES_MV,METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV,+ METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV,KAFKA_METRICS_AVRO4_MV,TOPHOG_MV_SQL,BILLING_USAGE_RECORDS_MV_SQL,diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambrindex fec26e9b4e92..f4ddd900dc70 100644--- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr+++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr@@ -3292,7 +3292,9 @@if(toBool(ifNull(has_labels, 1)), mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), attributes)), CAST(map(), 'Map(String, String)')) AS attributes,_partition,_topic,- _offset+ _offset,+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partitionFROM posthog_test.kafka_metrics_avro4WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULLSETTINGS@@ -5196,7 +5198,9 @@`attributes` Map(LowCardinality(String), String),`_partition` UInt32,`_topic` String,- `_offset` UInt64+ `_offset` UInt64,+ `_source_topic` String,+ `_source_partition` UInt32)ENGINE = Null@@ -5393,6 +5397,31 @@'''# ---+# name: test_create_table_query[metrics4_input_to_metrics_kafka_metrics]+ '''++ CREATE MATERIALIZED VIEW IF NOT EXISTS posthog_test.metrics4_input_to_metrics_kafka_metrics TO posthog_test.writable_metrics_kafka_metrics+ AS SELECT+ kafka_partition AS _partition,+ kafka_topic AS _topic,+ maxSimpleState(kafka_offset) AS max_offset,+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,+ maxSimpleState(now()) AS max_created_at+ FROM+ (+ SELECT+ kafka_source.1 AS kafka_topic,+ kafka_source.2 AS kafka_partition,+ kafka_source.3 AS kafka_offset,+ observed_timestamp+ FROM posthog_test.metrics4_input+ ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source+ WHERE kafka_topic != ''+ )+ GROUP BY kafka_partition, kafka_topic++ '''+# ---# name: test_create_table_query[metrics4_names]'''@@ -5530,6 +5559,26 @@# name: test_create_table_query[metrics_distributed]"CREATE TABLE IF NOT EXISTS posthog_test.metrics_distributed AS posthog_test.metrics2 ENGINE = Distributed('posthog_single_shard', 'posthog_test', 'metrics2')"# ---+# name: test_create_table_query[metrics_kafka_metrics]+ '''++ CREATE TABLE IF NOT EXISTS posthog_test.metrics_kafka_metrics+ (+ `_partition` UInt32,+ `_topic` String,+ `max_offset` SimpleAggregateFunction(max, UInt64),+ `max_observed_timestamp` SimpleAggregateFunction(max, DateTime64(9)),+ `max_timestamp` SimpleAggregateFunction(max, DateTime64(9)),+ `max_created_at` SimpleAggregateFunction(max, DateTime64(9)),+ `max_lag` SimpleAggregateFunction(max, UInt64)+ )+ ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.metrics_kafka_metrics', '{replica}-{shard}')+ ORDER BY (_topic, _partition)+ SETTINGS+ index_granularity = 8192++ '''+# ---# name: test_create_table_query[ops_query_log_archive_v3_mv]'''CREATE MATERIALIZED VIEW IF NOT EXISTS ops_query_log_archive_v3_mv@@ -11235,6 +11284,23 @@'''# ---+# name: test_create_table_query[writable_metrics_kafka_metrics]+ '''++ CREATE TABLE IF NOT EXISTS posthog_test.writable_metrics_kafka_metrics+ (+ `_partition` UInt32,+ `_topic` String,+ `max_offset` SimpleAggregateFunction(max, UInt64),+ `max_observed_timestamp` SimpleAggregateFunction(max, DateTime64(9)),+ `max_timestamp` SimpleAggregateFunction(max, DateTime64(9)),+ `max_created_at` SimpleAggregateFunction(max, DateTime64(9)),+ `max_lag` SimpleAggregateFunction(max, UInt64)+ )+ ENGINE = Distributed('posthog_single_shard', 'posthog_test', 'metrics_kafka_metrics')++ '''+# ---# name: test_create_table_query[writable_person]'''@@ -13220,7 +13286,9 @@`attributes` Map(LowCardinality(String), String),`_partition` UInt32,`_topic` String,- `_offset` UInt64+ `_offset` UInt64,+ `_source_topic` String,+ `_source_partition` UInt32)ENGINE = Null@@ -13353,6 +13421,26 @@'''# ---+# name: test_create_table_query_replicated_and_storage[metrics_kafka_metrics]+ '''++ CREATE TABLE IF NOT EXISTS posthog_test.metrics_kafka_metrics+ (+ `_partition` UInt32,+ `_topic` String,+ `max_offset` SimpleAggregateFunction(max, UInt64),+ `max_observed_timestamp` SimpleAggregateFunction(max, DateTime64(9)),+ `max_timestamp` SimpleAggregateFunction(max, DateTime64(9)),+ `max_created_at` SimpleAggregateFunction(max, DateTime64(9)),+ `max_lag` SimpleAggregateFunction(max, UInt64)+ )+ ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.metrics_kafka_metrics', '{replica}-{shard}')+ ORDER BY (_topic, _partition)+ SETTINGS+ index_granularity = 8192++ '''+# ---# name: test_create_table_query_replicated_and_storage[partitioned_sharded_posthog_document_embeddings]'''diff --git a/posthog/clickhouse/test/test_schema.py b/posthog/clickhouse/test/test_schema.pyindex 00040a3fe191..7a2bd98c03a9 100644--- a/posthog/clickhouse/test/test_schema.py+++ b/posthog/clickhouse/test/test_schema.py@@ -15,6 +15,7 @@ from posthog.hogql.database.schema.flag_evaluations import FLAG_EVALUATIONS_CLICfrom posthog.clickhouse.client import sync_executefrom posthog.clickhouse.kafka_engine import CONSUMER_GROUP_EVENTS_JSON_NATIVE_JSON, KAFKA_COLUMNS_WITH_PARTITIONfrom posthog.clickhouse.logs import LOGS34_TO_PATTERN_BUCKETS_MV_SELECT, LOGS34_TO_VOLUME_BUCKETS_MV_SELECT+from posthog.clickhouse.metrics import METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECTfrom posthog.clickhouse.property_groups import flag_evaluations_property_group_definitions, iter_groupsfrom posthog.clickhouse.schema import (CREATE_KAFKA_TABLE_QUERIES,@@ -189,6 +190,41 @@ def test_logs_pattern_buckets_keep_stamped_empty_patterns_and_drop_unstamped_rowassert rows == [(5, "", "", 15, 1), (5, "GET <N>", "", 31, 2), (5, "GET <N>", "hog_flow", 15, 1)][email protected]("clickhouse_database")+def test_metrics_kafka_metrics_counts_rows_toward_both_hops() -> None:+ select = METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT().replace(+ f"FROM {django_settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE}.metrics4_input", "FROM checkpoint_input"+ )+ rows = sync_execute(+ """+ WITH checkpoint_input AS (+ SELECT+ row.1 AS _topic,+ toUInt32(row.2) AS _partition,+ toUInt64(row.3) AS _offset,+ row.4 AS _source_topic,+ toUInt32(row.5) AS _source_partition,+ toDateTime64(row.6, 6, 'UTC') AS observed_timestamp+ FROM (+ SELECT arrayJoin([+ ('clickhouse_metrics', 3, 10, 'ingestion_metrics', 7, '2026-01-02 12:00:00'),+ ('clickhouse_metrics', 4, 20, 'ingestion_metrics', 7, '2026-01-02 12:00:30'),+ ('clickhouse_metrics', 5, 11, '', 0, '2026-01-02 12:01:00')+ ]) AS row+ )+ )+ SELECT _topic, _partition, max_offset, toString(max_observed_timestamp) FROM ("""+ + select+ + ") ORDER BY _topic, _partition",+ )+ assert rows == [+ ("clickhouse_metrics", 3, 10, "2026-01-02 12:00:00.000000"),+ ("clickhouse_metrics", 4, 20, "2026-01-02 12:00:30.000000"),+ ("clickhouse_metrics", 5, 11, "2026-01-02 12:01:00.000000"),+ ("ingestion_metrics", 7, 0, "2026-01-02 12:00:30.000000"),+ ]++def _column_definition_lines(block: str) -> Iterator[str]:for raw_line in block.splitlines():line = raw_line.strip().lstrip(",").strip()