Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(metrics): write a two-hop partition checkpoint from the metrics4 chain (#127782)

Jimmy Zhuc6a144181b7b
diff --git a/posthog/clickhouse/hcl/golden/dev/apm.hcl b/posthog/clickhouse/hcl/golden/dev/apm.hcl
index 98851c33ac1a..3a1dd2584fcb 100644
--- a/posthog/clickhouse/hcl/golden/dev/apm.hcl
+++ b/posthog/clickhouse/hcl/golden/dev/apm.hcl
@@ -317,6 +317,12 @@ database "posthog" {
column "_offset" {
type = "UInt64"
}
+ column "_source_topic" {
+ type = "String"
+ }
+ column "_source_partition" {
+ type = "UInt32"
+ }
engine "null" {
}
}
@@ -1499,7 +1505,9 @@ SELECT
) AS attributes,
_partition,
_topic,
- _offset
+ _offset,
+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,
+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partition
FROM posthog.kafka_metrics_avro4
WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULL
SETTINGS
@@ -1588,6 +1596,12 @@ SQL
column "_offset" {
type = "UInt64"
}
+ column "_source_topic" {
+ type = "String"
+ }
+ column "_source_partition" {
+ type = "UInt32"
+ }
}
materialized_view "kafka_trace_spans_avro_mv" {
@@ -2049,6 +2063,47 @@ SQL
}
}
+ materialized_view "metrics4_input_to_metrics_kafka_metrics" {
+ to_table = "posthog.writable_metrics_kafka_metrics"
+ query = <<SQL
+SELECT
+ kafka_partition AS _partition,
+ kafka_topic AS _topic,
+ maxSimpleState(kafka_offset) AS max_offset,
+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,
+ maxSimpleState(now()) AS max_created_at
+FROM
+ (
+ SELECT
+ kafka_source.1 AS kafka_topic,
+ kafka_source.2 AS kafka_partition,
+ kafka_source.3 AS kafka_offset,
+ observed_timestamp
+ FROM
+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source
+ WHERE kafka_topic != ''
+ )
+GROUP BY
+ kafka_partition, kafka_topic
+SQL
+
+ column "_partition" {
+ type = "UInt32"
+ }
+ column "_topic" {
+ type = "String"
+ }
+ column "max_offset" {
+ type = "SimpleAggregateFunction(max, UInt64)"
+ }
+ column "max_observed_timestamp" {
+ type = "SimpleAggregateFunction(max, DateTime64(6))"
+ }
+ column "max_created_at" {
+ type = "SimpleAggregateFunction(max, DateTime)"
+ }
+ }
+
materialized_view "ops_query_log_archive_v3_mv" {
to_table = "posthog.writable_query_log_archive_v3"
query = <<SQL
diff --git a/posthog/clickhouse/hcl/golden/local-single/all.hcl b/posthog/clickhouse/hcl/golden/local-single/all.hcl
index 84a59f0ec753..4ee171f381ea 100644
--- a/posthog/clickhouse/hcl/golden/local-single/all.hcl
+++ b/posthog/clickhouse/hcl/golden/local-single/all.hcl
@@ -7865,6 +7865,12 @@ SQL
column "_offset" {
type = "UInt64"
}
+ column "_source_topic" {
+ type = "String"
+ }
+ column "_source_partition" {
+ type = "UInt32"
+ }
engine "null" {
}
}
@@ -18323,6 +18329,35 @@ SQL
}
}
+ table "writable_metrics_kafka_metrics" {
+ column "_partition" {
+ type = "UInt32"
+ }
+ column "_topic" {
+ type = "String"
+ }
+ column "max_offset" {
+ type = "SimpleAggregateFunction(max, UInt64)"
+ }
+ column "max_observed_timestamp" {
+ type = "SimpleAggregateFunction(max, DateTime64(9))"
+ }
+ column "max_timestamp" {
+ type = "SimpleAggregateFunction(max, DateTime64(9))"
+ }
+ column "max_created_at" {
+ type = "SimpleAggregateFunction(max, DateTime64(9))"
+ }
+ column "max_lag" {
+ type = "SimpleAggregateFunction(max, UInt64)"
+ }
+ engine "distributed" {
+ cluster_name = "logs"
+ remote_database = "posthog"
+ remote_table = "metrics_kafka_metrics"
+ }
+ }
+
table "writable_person" {
column "id" {
type = "UUID"
@@ -22138,7 +22173,9 @@ SELECT
) AS attributes,
_partition,
_topic,
- _offset
+ _offset,
+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,
+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partition
FROM posthog.kafka_metrics_avro4
WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULL
SETTINGS
@@ -22227,6 +22264,12 @@ SQL
column "_offset" {
type = "UInt64"
}
+ column "_source_topic" {
+ type = "String"
+ }
+ column "_source_partition" {
+ type = "UInt32"
+ }
}
materialized_view "kafka_trace_spans_avro_mv" {
@@ -23320,6 +23363,47 @@ SQL
}
}
+ materialized_view "metrics4_input_to_metrics_kafka_metrics" {
+ to_table = "posthog.writable_metrics_kafka_metrics"
+ query = <<SQL
+SELECT
+ kafka_partition AS _partition,
+ kafka_topic AS _topic,
+ maxSimpleState(kafka_offset) AS max_offset,
+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,
+ maxSimpleState(now()) AS max_created_at
+FROM
+ (
+ SELECT
+ kafka_source.1 AS kafka_topic,
+ kafka_source.2 AS kafka_partition,
+ kafka_source.3 AS kafka_offset,
+ observed_timestamp
+ FROM
+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source
+ WHERE kafka_topic != ''
+ )
+GROUP BY
+ kafka_partition, kafka_topic
+SQL
+
+ column "_partition" {
+ type = "UInt32"
+ }
+ column "_topic" {
+ type = "String"
+ }
+ column "max_offset" {
+ type = "SimpleAggregateFunction(max, UInt64)"
+ }
+ column "max_observed_timestamp" {
+ type = "SimpleAggregateFunction(max, DateTime64(6))"
+ }
+ column "max_created_at" {
+ type = "SimpleAggregateFunction(max, DateTime)"
+ }
+ }
+
materialized_view "ops_query_log_archive_v3_mv" {
to_table = "posthog.writable_query_log_archive_v3"
query = <<SQL
diff --git a/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl b/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl
index 7ca5a537b81f..4eb0db91d364 100644
--- a/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-eu/apm.hcl
@@ -234,6 +234,12 @@ database "posthog" {
column "_offset" {
type = "UInt64"
}
+ column "_source_topic" {
+ type = "String"
+ }
+ column "_source_partition" {
+ type = "UInt32"
+ }
engine "null" {
}
}
@@ -559,6 +565,35 @@ database "posthog" {
}
}
+ table "writable_metrics_kafka_metrics" {
+ column "_partition" {
+ type = "UInt32"
+ }
+ column "_topic" {
+ type = "String"
+ }
+ column "max_offset" {
+ type = "SimpleAggregateFunction(max, UInt64)"
+ }
+ column "max_observed_timestamp" {
+ type = "SimpleAggregateFunction(max, DateTime64(9))"
+ }
+ column "max_timestamp" {
+ type = "SimpleAggregateFunction(max, DateTime64(9))"
+ }
+ column "max_created_at" {
+ type = "SimpleAggregateFunction(max, DateTime64(9))"
+ }
+ column "max_lag" {
+ type = "SimpleAggregateFunction(max, UInt64)"
+ }
+ engine "distributed" {
+ cluster_name = "logs"
+ remote_database = "posthog"
+ remote_table = "metrics_kafka_metrics"
+ }
+ }
+
table "writable_query_log_archive_v3" {
column "initial_query_start_time" {
type = "DateTime64(6, 'UTC')"
@@ -1078,7 +1113,9 @@ SELECT
) AS attributes,
_partition,
_topic,
- _offset
+ _offset,
+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,
+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partition
FROM posthog.kafka_metrics_avro4
WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULL
SETTINGS
@@ -1167,6 +1204,12 @@ SQL
column "_offset" {
type = "UInt64"
}
+ column "_source_topic" {
+ type = "String"
+ }
+ column "_source_partition" {
+ type = "UInt32"
+ }
}
materialized_view "metrics4_input_to_metrics4_attributes" {
@@ -1496,6 +1539,47 @@ SQL
}
}
+ materialized_view "metrics4_input_to_metrics_kafka_metrics" {
+ to_table = "posthog.writable_metrics_kafka_metrics"
+ query = <<SQL
+SELECT
+ kafka_partition AS _partition,
+ kafka_topic AS _topic,
+ maxSimpleState(kafka_offset) AS max_offset,
+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,
+ maxSimpleState(now()) AS max_created_at
+FROM
+ (
+ SELECT
+ kafka_source.1 AS kafka_topic,
+ kafka_source.2 AS kafka_partition,
+ kafka_source.3 AS kafka_offset,
+ observed_timestamp
+ FROM
+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source
+ WHERE kafka_topic != ''
+ )
+GROUP BY
+ kafka_partition, kafka_topic
+SQL
+
+ column "_partition" {
+ type = "UInt32"
+ }
+ column "_topic" {
+ type = "String"
+ }
+ column "max_offset" {
+ type = "SimpleAggregateFunction(max, UInt64)"
+ }
+ column "max_observed_timestamp" {
+ type = "SimpleAggregateFunction(max, DateTime64(6))"
+ }
+ column "max_created_at" {
+ type = "SimpleAggregateFunction(max, DateTime)"
+ }
+ }
+
materialized_view "ops_query_log_archive_v3_mv" {
to_table = "posthog.writable_query_log_archive_v3"
query = <<SQL
diff --git a/posthog/clickhouse/hcl/golden/prod-us/apm.hcl b/posthog/clickhouse/hcl/golden/prod-us/apm.hcl
index 7ca5a537b81f..4eb0db91d364 100644
--- a/posthog/clickhouse/hcl/golden/prod-us/apm.hcl
+++ b/posthog/clickhouse/hcl/golden/prod-us/apm.hcl
@@ -234,6 +234,12 @@ database "posthog" {
column "_offset" {
type = "UInt64"
}
+ column "_source_topic" {
+ type = "String"
+ }
+ column "_source_partition" {
+ type = "UInt32"
+ }
engine "null" {
}
}
@@ -559,6 +565,35 @@ database "posthog" {
}
}
+ table "writable_metrics_kafka_metrics" {
+ column "_partition" {
+ type = "UInt32"
+ }
+ column "_topic" {
+ type = "String"
+ }
+ column "max_offset" {
+ type = "SimpleAggregateFunction(max, UInt64)"
+ }
+ column "max_observed_timestamp" {
+ type = "SimpleAggregateFunction(max, DateTime64(9))"
+ }
+ column "max_timestamp" {
+ type = "SimpleAggregateFunction(max, DateTime64(9))"
+ }
+ column "max_created_at" {
+ type = "SimpleAggregateFunction(max, DateTime64(9))"
+ }
+ column "max_lag" {
+ type = "SimpleAggregateFunction(max, UInt64)"
+ }
+ engine "distributed" {
+ cluster_name = "logs"
+ remote_database = "posthog"
+ remote_table = "metrics_kafka_metrics"
+ }
+ }
+
table "writable_query_log_archive_v3" {
column "initial_query_start_time" {
type = "DateTime64(6, 'UTC')"
@@ -1078,7 +1113,9 @@ SELECT
) AS attributes,
_partition,
_topic,
- _offset
+ _offset,
+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,
+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partition
FROM posthog.kafka_metrics_avro4
WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULL
SETTINGS
@@ -1167,6 +1204,12 @@ SQL
column "_offset" {
type = "UInt64"
}
+ column "_source_topic" {
+ type = "String"
+ }
+ column "_source_partition" {
+ type = "UInt32"
+ }
}
materialized_view "metrics4_input_to_metrics4_attributes" {
@@ -1496,6 +1539,47 @@ SQL
}
}
+ materialized_view "metrics4_input_to_metrics_kafka_metrics" {
+ to_table = "posthog.writable_metrics_kafka_metrics"
+ query = <<SQL
+SELECT
+ kafka_partition AS _partition,
+ kafka_topic AS _topic,
+ maxSimpleState(kafka_offset) AS max_offset,
+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,
+ maxSimpleState(now()) AS max_created_at
+FROM
+ (
+ SELECT
+ kafka_source.1 AS kafka_topic,
+ kafka_source.2 AS kafka_partition,
+ kafka_source.3 AS kafka_offset,
+ observed_timestamp
+ FROM
+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source
+ WHERE kafka_topic != ''
+ )
+GROUP BY
+ kafka_partition, kafka_topic
+SQL
+
+ column "_partition" {
+ type = "UInt32"
+ }
+ column "_topic" {
+ type = "String"
+ }
+ column "max_offset" {
+ type = "SimpleAggregateFunction(max, UInt64)"
+ }
+ column "max_observed_timestamp" {
+ type = "SimpleAggregateFunction(max, DateTime64(6))"
+ }
+ column "max_created_at" {
+ type = "SimpleAggregateFunction(max, DateTime)"
+ }
+ }
+
materialized_view "ops_query_log_archive_v3_mv" {
to_table = "posthog.writable_query_log_archive_v3"
query = <<SQL
diff --git a/posthog/clickhouse/hcl/roles/apm/dev/tables.hcl b/posthog/clickhouse/hcl/roles/apm/dev/tables.hcl
index 2f6d2d70cd5d..d9e4d7b4b24c 100644
--- a/posthog/clickhouse/hcl/roles/apm/dev/tables.hcl
+++ b/posthog/clickhouse/hcl/roles/apm/dev/tables.hcl
@@ -185,35 +185,6 @@ database "posthog" {
}
}
- table "writable_metrics_kafka_metrics" {
- column "_partition" {
- type = "UInt32"
- }
- column "_topic" {
- type = "String"
- }
- column "max_offset" {
- type = "SimpleAggregateFunction(max, UInt64)"
- }
- column "max_observed_timestamp" {
- type = "SimpleAggregateFunction(max, DateTime64(9))"
- }
- column "max_timestamp" {
- type = "SimpleAggregateFunction(max, DateTime64(9))"
- }
- column "max_created_at" {
- type = "SimpleAggregateFunction(max, DateTime64(9))"
- }
- column "max_lag" {
- type = "SimpleAggregateFunction(max, UInt64)"
- }
- engine "distributed" {
- cluster_name = "logs"
- remote_database = "posthog"
- remote_table = "metrics_kafka_metrics"
- }
- }
-
table "writable_trace_spans" {
column "time_bucket" {
type = "DateTime"
diff --git a/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl b/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl
index 4a8b28f426c8..b667a1291f79 100644
--- a/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl
+++ b/posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl
@@ -70,6 +70,8 @@ database "posthog" {
column "_partition" { type = "UInt32" }
column "_topic" { type = "String" }
column "_offset" { type = "UInt64" }
+ column "_source_topic" { type = "String" }
+ column "_source_partition" { type = "UInt32" }
engine "null" {}
}
@@ -168,6 +170,33 @@ database "posthog" {
}
}
+ table "writable_metrics_kafka_metrics" {
+ column "_partition" { type = "UInt32" }
+ column "_topic" { type = "String" }
+ column "max_offset" { type = "SimpleAggregateFunction(max, UInt64)" }
+ column "max_observed_timestamp" { type = "SimpleAggregateFunction(max, DateTime64(9))" }
+ column "max_timestamp" { type = "SimpleAggregateFunction(max, DateTime64(9))" }
+ column "max_created_at" { type = "SimpleAggregateFunction(max, DateTime64(9))" }
+ column "max_lag" { type = "SimpleAggregateFunction(max, UInt64)" }
+ engine "distributed" {
+ cluster_name = "logs"
+ remote_database = "posthog"
+ remote_table = "metrics_kafka_metrics"
+ }
+ }
+
+ # The newest capture time each Kafka partition has delivered. The floor across partitions is
+ # how far every partition has reached, which the metrics alert evaluator reads.
+ materialized_view "metrics4_input_to_metrics_kafka_metrics" {
+ to_table = "posthog.writable_metrics_kafka_metrics"
+ query = file("sql/metrics4_input_to_metrics_kafka_metrics.sql")
+ column "_partition" { type = "UInt32" }
+ column "_topic" { type = "String" }
+ column "max_offset" { type = "SimpleAggregateFunction(max, UInt64)" }
+ column "max_observed_timestamp" { type = "SimpleAggregateFunction(max, DateTime64(6))" }
+ column "max_created_at" { type = "SimpleAggregateFunction(max, DateTime)" }
+ }
+
materialized_view "kafka_metrics_avro4_mv" {
to_table = "posthog.metrics4_input"
query = file("sql/kafka_metrics_avro4_mv.sql")
@@ -198,6 +227,8 @@ database "posthog" {
column "_partition" { type = "UInt64" }
column "_topic" { type = "LowCardinality(String)" }
column "_offset" { type = "UInt64" }
+ column "_source_topic" { type = "String" }
+ column "_source_partition" { type = "UInt32" }
}
materialized_view "metrics4_input_to_metrics4_samples" {
diff --git a/posthog/clickhouse/hcl/roles/apm/shared/sql/kafka_metrics_avro4_mv.sql b/posthog/clickhouse/hcl/roles/apm/shared/sql/kafka_metrics_avro4_mv.sql
index ce97139b2ac9..6237cb03c6f9 100644
--- a/posthog/clickhouse/hcl/roles/apm/shared/sql/kafka_metrics_avro4_mv.sql
+++ b/posthog/clickhouse/hcl/roles/apm/shared/sql/kafka_metrics_avro4_mv.sql
@@ -25,7 +25,9 @@ SELECT
if(toBool(ifNull(has_labels, 1)), mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), attributes)), CAST(map(), 'Map(String, String)')) AS attributes,
_partition,
_topic,
- _offset
+ _offset,
+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,
+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partition
FROM posthog.kafka_metrics_avro4
WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULL
SETTINGS
diff --git a/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics_kafka_metrics.sql b/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics_kafka_metrics.sql
new file mode 100644
index 000000000000..5c39242f5710
--- /dev/null
+++ b/posthog/clickhouse/hcl/roles/apm/shared/sql/metrics4_input_to_metrics_kafka_metrics.sql
@@ -0,0 +1,19 @@
+SELECT
+ kafka_partition AS _partition,
+ kafka_topic AS _topic,
+ maxSimpleState(kafka_offset) AS max_offset,
+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,
+ maxSimpleState(now()) AS max_created_at
+FROM
+ (
+ SELECT
+ kafka_source.1 AS kafka_topic,
+ kafka_source.2 AS kafka_partition,
+ kafka_source.3 AS kafka_offset,
+ observed_timestamp
+ FROM posthog.metrics4_input
+ ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source
+ WHERE kafka_topic != ''
+ )
+GROUP BY
+ kafka_partition, kafka_topic
diff --git a/posthog/clickhouse/hcl/sql/dev/apm.sql b/posthog/clickhouse/hcl/sql/dev/apm.sql
index 8b2d24d21ed7..28f0245e4767 100644
--- a/posthog/clickhouse/hcl/sql/dev/apm.sql
+++ b/posthog/clickhouse/hcl/sql/dev/apm.sql
@@ -95,7 +95,9 @@ CREATE TABLE posthog.metrics4_input (
attributes Map(LowCardinality(String), String),
_partition UInt32,
_topic String,
- _offset UInt64
+ _offset UInt64,
+ _source_topic String,
+ _source_partition UInt32
) ENGINE = Null();
CREATE TABLE posthog.writable_logs34 (
time_bucket DateTime MATERIALIZED toStartOfDay(timestamp),
@@ -449,7 +451,7 @@ CREATE MATERIALIZED VIEW posthog.kafka_logs34_avro_mv TO posthog.writable_logs34
toUInt8(ifNull(pattern_version, 0)) AS pattern_version,
_headers.value[indexOf(_headers.name, 'internal-source')] AS internal_source
FROM posthog.kafka_logs_avro;
-CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64) AS SELECT
+CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64, _source_topic String, _source_partition UInt32) AS SELECT
uuid,
toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,
ifNull(metric_name, '') AS metric_name,
@@ -493,7 +495,9 @@ CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_inpu
) AS attributes,
_partition,
_topic,
- _offset
+ _offset,
+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,
+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partition
FROM posthog.kafka_metrics_avro4
WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULL
SETTINGS
@@ -649,6 +653,25 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.wr
original_expiry_timestamp
FROM posthog.metrics4_input
WHERE has_labels;
+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics_kafka_metrics TO posthog.writable_metrics_kafka_metrics (_partition UInt32, _topic String, max_offset SimpleAggregateFunction(max, UInt64), max_observed_timestamp SimpleAggregateFunction(max, DateTime64(6)), max_created_at SimpleAggregateFunction(max, DateTime)) AS SELECT
+ kafka_partition AS _partition,
+ kafka_topic AS _topic,
+ maxSimpleState(kafka_offset) AS max_offset,
+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,
+ maxSimpleState(now()) AS max_created_at
+FROM
+ (
+ SELECT
+ kafka_source.1 AS kafka_topic,
+ kafka_source.2 AS kafka_partition,
+ kafka_source.3 AS kafka_offset,
+ observed_timestamp
+ FROM
+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source
+ WHERE kafka_topic != ''
+ )
+GROUP BY
+ kafka_partition, kafka_topic;
CREATE MATERIALIZED VIEW posthog.ops_query_log_archive_v3_mv TO posthog.writable_query_log_archive_v3 (initial_query_start_time DateTime64(6, 'UTC'), team_id Int64, initial_query_id String, is_initial_query SimpleAggregateFunction(max, UInt8), hostname SimpleAggregateFunction(max, LowCardinality(String)), user SimpleAggregateFunction(max, LowCardinality(String)), type SimpleAggregateFunction(max, Enum8('QueryStart'=1, 'QueryFinish'=2, 'ExceptionBeforeStart'=3, 'ExceptionWhileProcessing'=4)), event_time_microseconds SimpleAggregateFunction(max, DateTime64(6, 'UTC')), query_duration_ms SimpleAggregateFunction(max, UInt64), read_rows SimpleAggregateFunction(sum, UInt64), read_bytes SimpleAggregateFunction(sum, UInt64), written_rows SimpleAggregateFunction(sum, UInt64), written_bytes SimpleAggregateFunction(sum, UInt64), result_rows SimpleAggregateFunction(sum, UInt64), result_bytes SimpleAggregateFunction(sum, UInt64), memory_usage SimpleAggregateFunction(max, UInt64), memory_usage_sum SimpleAggregateFunction(sum, UInt64), peak_threads_usage SimpleAggregateFunction(sum, UInt64), execution_count SimpleAggregateFunction(sum, UInt64), current_database SimpleAggregateFunction(max, LowCardinality(String)), query SimpleAggregateFunction(max, String), normalized_query_hash SimpleAggregateFunction(max, UInt64), query_kind_by_root SimpleAggregateFunction(max, Tuple(UInt8, String)), tables SimpleAggregateFunction(max, Array(String)), exception_code SimpleAggregateFunction(max, Int32), exception SimpleAggregateFunction(max, String), stack_trace SimpleAggregateFunction(max, String), ProfileEvents_SelectedRows SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_RealTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_OSCPUVirtualTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_UserTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_SystemTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedParts SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedMarks SimpleAggregateFunction(sum, UInt64), ProfileEvents_NetworkReceiveBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_NetworkSendBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_ReadBufferFromFileDescriptorReadBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_WriteBufferFromFileDescriptorWriteBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3Clients SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3DeleteObjects SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CopyObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3ListObjects SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3HeadObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3GetObjectAttributes SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CreateMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3UploadPartCopy SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3UploadPart SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3AbortMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CompleteMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3PutObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3GetObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_ReadBufferFromS3Bytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_WriteBufferFromS3Bytes SimpleAggregateFunction(sum, UInt64), lc_org_id SimpleAggregateFunction(max, String), lc_user_id SimpleAggregateFunction(max, Int64), lc_session_id SimpleAggregateFunction(max, String), lc_is_impersonated SimpleAggregateFunction(max, Bool), lc_workflow SimpleAggregateFunction(max, LowCardinality(String)), lc_kind SimpleAggregateFunction(max, LowCardinality(String)), lc_id SimpleAggregateFunction(max, String), lc_route_id SimpleAggregateFunction(max, String), lc_access_method SimpleAggregateFunction(max, LowCardinality(String)), lc_api_key_label SimpleAggregateFunction(max, String), lc_api_key_mask SimpleAggregateFunction(max, String), lc_query_type SimpleAggregateFunction(max, LowCardinality(String)), lc_product SimpleAggregateFunction(max, LowCardinality(String)), lc_chargeable SimpleAggregateFunction(max, Bool), lc_name SimpleAggregateFunction(max, String), lc_request_name SimpleAggregateFunction(max, String), lc_client_query_id SimpleAggregateFunction(max, String), lc_dashboard_id SimpleAggregateFunction(max, Int64), lc_insight_id SimpleAggregateFunction(max, Int64), lc_cohort_id SimpleAggregateFunction(max, Int64), lc_batch_export_id SimpleAggregateFunction(max, String), lc_experiment_id SimpleAggregateFunction(max, Int64), lc_experiment_feature_flag_key SimpleAggregateFunction(max, String), lc_alert_config_id SimpleAggregateFunction(max, String), lc_feature SimpleAggregateFunction(max, LowCardinality(String)), lc_table_id SimpleAggregateFunction(max, String), lc_warehouse_query SimpleAggregateFunction(max, Bool), lc_person_on_events_mode SimpleAggregateFunction(max, LowCardinality(String)), lc_service_name SimpleAggregateFunction(max, LowCardinality(String)), lc_workload SimpleAggregateFunction(max, LowCardinality(String)), lc_query__kind SimpleAggregateFunction(max, LowCardinality(String)), lc_query__query SimpleAggregateFunction(max, String), lc_query SimpleAggregateFunction(max, String), lc_modifiers SimpleAggregateFunction(max, String), lc_temporal__workflow_namespace SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__workflow_type SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__workflow_id SimpleAggregateFunction(max, String), lc_temporal__workflow_run_id SimpleAggregateFunction(max, String), lc_temporal__activity_type SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__activity_id SimpleAggregateFunction(max, String), lc_temporal__attempt SimpleAggregateFunction(max, Int64), lc_dagster__job_name SimpleAggregateFunction(max, LowCardinality(String)), lc_dagster__run_id SimpleAggregateFunction(max, String), lc_dagster__owner SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_name SimpleAggregateFunction(max, String), lc_experiment_metric_name SimpleAggregateFunction(max, String), lc_experiment_execution_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_metric_type SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_funnel_order_type SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_exposures_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_metric_events_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_query_surface SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_precompute_table SimpleAggregateFunction(max, String), lc_experiment_query_group_id SimpleAggregateFunction(max, String), lc_experiment_precompute_skip_reason SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_scan_date_from SimpleAggregateFunction(max, String), lc_experiment_scan_date_to SimpleAggregateFunction(max, String), lc_precompute_window_start SimpleAggregateFunction(max, String), lc_precompute_window_end SimpleAggregateFunction(max, String), lc_plan_fingerprint SimpleAggregateFunction(max, String), lc_estimated_rows SimpleAggregateFunction(max, Int64), lc_estimated_bytes SimpleAggregateFunction(max, Int64)) AS WITH
JSONExtract(ql.log_comment, 'Tuple(team_id Int64, org_id String, user_id Int64, session_id String, is_impersonated Bool, workflow String, kind String, id String, route_id String, access_method String, api_key_label String, api_key_mask String, query_type String, product String, chargeable Bool, name String, request_name String, client_query_id String, dashboard_id Int64, insight_id Int64, cohort_id Int64, batch_export_id String, experiment_id Int64, experiment_feature_flag_key String, alert_config_id String, feature String, table_id String, warehouse_query Bool, person_on_events_mode String, service_name String, workload String, experiment_name String, experiment_metric_name String, experiment_execution_path String, experiment_metric_type String, experiment_funnel_order_type String, experiment_exposures_path String, experiment_metric_events_path String, experiment_query_surface String, experiment_precompute_table String, experiment_query_group_id String, experiment_precompute_skip_reason String, experiment_scan_date_from String, experiment_scan_date_to String, precompute_window_start String, precompute_window_end String, plan_fingerprint String, estimated_rows Int64, estimated_bytes Int64, temporal Tuple(workflow_namespace String, workflow_type String, workflow_id String, workflow_run_id String, activity_type String, activity_id String, attempt Int64), dagster Tuple(job_name String, run_id String, tags Tuple(owner String)), query Tuple(kind String, query String, source Tuple(kind String, query String)))') AS lc
SELECT
diff --git a/posthog/clickhouse/hcl/sql/local-single/all.sql b/posthog/clickhouse/hcl/sql/local-single/all.sql
index b243d5ac1acd..d84cc77e6953 100644
--- a/posthog/clickhouse/hcl/sql/local-single/all.sql
+++ b/posthog/clickhouse/hcl/sql/local-single/all.sql
@@ -1581,7 +1581,9 @@ CREATE TABLE posthog.metrics4_input (
attributes Map(LowCardinality(String), String),
_partition UInt32,
_topic String,
- _offset UInt64
+ _offset UInt64,
+ _source_topic String,
+ _source_partition UInt32
) ENGINE = Null();
CREATE TABLE posthog.metrics4_names (
team_id Int32,
@@ -4041,6 +4043,15 @@ CREATE TABLE posthog.writable_metrics4_series (
time_bucket DateTime MATERIALIZED toStartOfHour(timestamp),
original_expiry_timestamp DateTime64(6)
) ENGINE = Distributed('logs', 'posthog', 'metrics4_series');
+CREATE TABLE posthog.writable_metrics_kafka_metrics (
+ _partition UInt32,
+ _topic String,
+ max_offset SimpleAggregateFunction(max, UInt64),
+ max_observed_timestamp SimpleAggregateFunction(max, DateTime64(9)),
+ max_timestamp SimpleAggregateFunction(max, DateTime64(9)),
+ max_created_at SimpleAggregateFunction(max, DateTime64(9)),
+ max_lag SimpleAggregateFunction(max, UInt64)
+) ENGINE = Distributed('logs', 'posthog', 'metrics_kafka_metrics');
CREATE TABLE posthog.writable_person (
id UUID,
created_at DateTime64(3),
@@ -4965,7 +4976,7 @@ CREATE MATERIALIZED VIEW posthog.kafka_logs_avro_kafka_metrics_mv TO posthog.log
FROM posthog.logs34
GROUP BY
_partition, _topic;
-CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64) AS SELECT
+CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64, _source_topic String, _source_partition UInt32) AS SELECT
uuid,
toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,
ifNull(metric_name, '') AS metric_name,
@@ -5009,7 +5020,9 @@ CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_inpu
) AS attributes,
_partition,
_topic,
- _offset
+ _offset,
+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,
+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partition
FROM posthog.kafka_metrics_avro4
WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULL
SETTINGS
@@ -5433,6 +5446,25 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.wr
original_expiry_timestamp
FROM posthog.metrics4_input
WHERE has_labels;
+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics_kafka_metrics TO posthog.writable_metrics_kafka_metrics (_partition UInt32, _topic String, max_offset SimpleAggregateFunction(max, UInt64), max_observed_timestamp SimpleAggregateFunction(max, DateTime64(6)), max_created_at SimpleAggregateFunction(max, DateTime)) AS SELECT
+ kafka_partition AS _partition,
+ kafka_topic AS _topic,
+ maxSimpleState(kafka_offset) AS max_offset,
+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,
+ maxSimpleState(now()) AS max_created_at
+FROM
+ (
+ SELECT
+ kafka_source.1 AS kafka_topic,
+ kafka_source.2 AS kafka_partition,
+ kafka_source.3 AS kafka_offset,
+ observed_timestamp
+ FROM
+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source
+ WHERE kafka_topic != ''
+ )
+GROUP BY
+ kafka_partition, kafka_topic;
CREATE MATERIALIZED VIEW posthog.performance_events_mv TO posthog.writeable_performance_events (uuid UUID, session_id String, window_id String, pageview_id String, distinct_id String, timestamp DateTime64(3), time_origin DateTime64(3, 'UTC'), entry_type LowCardinality(String), name String, team_id Int64, current_url String, start_time Float64, duration Float64, redirect_start Float64, redirect_end Float64, worker_start Float64, fetch_start Float64, domain_lookup_start Float64, domain_lookup_end Float64, connect_start Float64, secure_connection_start Float64, connect_end Float64, request_start Float64, response_start Float64, response_end Float64, decoded_body_size Int64, encoded_body_size Int64, initiator_type LowCardinality(String), next_hop_protocol LowCardinality(String), render_blocking_status LowCardinality(String), response_status Int64, transfer_size Int64, largest_contentful_paint_element String, largest_contentful_paint_render_time Float64, largest_contentful_paint_load_time Float64, largest_contentful_paint_size Float64, largest_contentful_paint_id String, largest_contentful_paint_url String, dom_complete Float64, dom_content_loaded_event Float64, dom_interactive Float64, load_event_end Float64, load_event_start Float64, redirect_count Int64, navigation_type LowCardinality(String), unload_event_end Float64, unload_event_start Float64, _timestamp Nullable(DateTime), _offset UInt64, _partition UInt64) AS SELECT
uuid,
session_id,
diff --git a/posthog/clickhouse/hcl/sql/prod-eu/apm.sql b/posthog/clickhouse/hcl/sql/prod-eu/apm.sql
index 2dd0f8791302..bbeec37e76e4 100644
--- a/posthog/clickhouse/hcl/sql/prod-eu/apm.sql
+++ b/posthog/clickhouse/hcl/sql/prod-eu/apm.sql
@@ -71,7 +71,9 @@ CREATE TABLE posthog.metrics4_input (
attributes Map(LowCardinality(String), String),
_partition UInt32,
_topic String,
- _offset UInt64
+ _offset UInt64,
+ _source_topic String,
+ _source_partition UInt32
) ENGINE = Null();
CREATE TABLE posthog.writable_logs34 (
time_bucket DateTime MATERIALIZED toStartOfDay(timestamp),
@@ -171,6 +173,15 @@ CREATE TABLE posthog.writable_metrics4_series (
time_bucket DateTime MATERIALIZED toStartOfHour(timestamp),
original_expiry_timestamp DateTime64(6)
) ENGINE = Distributed('logs', 'posthog', 'metrics4_series');
+CREATE TABLE posthog.writable_metrics_kafka_metrics (
+ _partition UInt32,
+ _topic String,
+ max_offset SimpleAggregateFunction(max, UInt64),
+ max_observed_timestamp SimpleAggregateFunction(max, DateTime64(9)),
+ max_timestamp SimpleAggregateFunction(max, DateTime64(9)),
+ max_created_at SimpleAggregateFunction(max, DateTime64(9)),
+ max_lag SimpleAggregateFunction(max, UInt64)
+) ENGINE = Distributed('logs', 'posthog', 'metrics_kafka_metrics');
CREATE TABLE posthog.writable_query_log_archive_v3 (
initial_query_start_time DateTime64(6, 'UTC'),
team_id Int64,
@@ -323,7 +334,7 @@ CREATE MATERIALIZED VIEW posthog.kafka_logs34_avro_mv TO posthog.writable_logs34
toUInt8(ifNull(pattern_version, 0)) AS pattern_version,
_headers.value[indexOf(_headers.name, 'internal-source')] AS internal_source
FROM posthog.kafka_logs_avro;
-CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64) AS SELECT
+CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64, _source_topic String, _source_partition UInt32) AS SELECT
uuid,
toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,
ifNull(metric_name, '') AS metric_name,
@@ -367,7 +378,9 @@ CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_inpu
) AS attributes,
_partition,
_topic,
- _offset
+ _offset,
+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,
+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partition
FROM posthog.kafka_metrics_avro4
WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULL
SETTINGS
@@ -485,6 +498,25 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.wr
original_expiry_timestamp
FROM posthog.metrics4_input
WHERE has_labels;
+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics_kafka_metrics TO posthog.writable_metrics_kafka_metrics (_partition UInt32, _topic String, max_offset SimpleAggregateFunction(max, UInt64), max_observed_timestamp SimpleAggregateFunction(max, DateTime64(6)), max_created_at SimpleAggregateFunction(max, DateTime)) AS SELECT
+ kafka_partition AS _partition,
+ kafka_topic AS _topic,
+ maxSimpleState(kafka_offset) AS max_offset,
+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,
+ maxSimpleState(now()) AS max_created_at
+FROM
+ (
+ SELECT
+ kafka_source.1 AS kafka_topic,
+ kafka_source.2 AS kafka_partition,
+ kafka_source.3 AS kafka_offset,
+ observed_timestamp
+ FROM
+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source
+ WHERE kafka_topic != ''
+ )
+GROUP BY
+ kafka_partition, kafka_topic;
CREATE MATERIALIZED VIEW posthog.ops_query_log_archive_v3_mv TO posthog.writable_query_log_archive_v3 (initial_query_start_time DateTime64(6, 'UTC'), team_id Int64, initial_query_id String, is_initial_query SimpleAggregateFunction(max, UInt8), hostname SimpleAggregateFunction(max, LowCardinality(String)), user SimpleAggregateFunction(max, LowCardinality(String)), type SimpleAggregateFunction(max, Enum8('QueryStart'=1, 'QueryFinish'=2, 'ExceptionBeforeStart'=3, 'ExceptionWhileProcessing'=4)), event_time_microseconds SimpleAggregateFunction(max, DateTime64(6, 'UTC')), query_duration_ms SimpleAggregateFunction(max, UInt64), read_rows SimpleAggregateFunction(sum, UInt64), read_bytes SimpleAggregateFunction(sum, UInt64), written_rows SimpleAggregateFunction(sum, UInt64), written_bytes SimpleAggregateFunction(sum, UInt64), result_rows SimpleAggregateFunction(sum, UInt64), result_bytes SimpleAggregateFunction(sum, UInt64), memory_usage SimpleAggregateFunction(max, UInt64), memory_usage_sum SimpleAggregateFunction(sum, UInt64), peak_threads_usage SimpleAggregateFunction(sum, UInt64), execution_count SimpleAggregateFunction(sum, UInt64), current_database SimpleAggregateFunction(max, LowCardinality(String)), query SimpleAggregateFunction(max, String), normalized_query_hash SimpleAggregateFunction(max, UInt64), query_kind_by_root SimpleAggregateFunction(max, Tuple(UInt8, String)), tables SimpleAggregateFunction(max, Array(String)), exception_code SimpleAggregateFunction(max, Int32), exception SimpleAggregateFunction(max, String), stack_trace SimpleAggregateFunction(max, String), ProfileEvents_SelectedRows SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_RealTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_OSCPUVirtualTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_UserTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_SystemTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedParts SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedMarks SimpleAggregateFunction(sum, UInt64), ProfileEvents_NetworkReceiveBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_NetworkSendBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_ReadBufferFromFileDescriptorReadBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_WriteBufferFromFileDescriptorWriteBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3Clients SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3DeleteObjects SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CopyObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3ListObjects SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3HeadObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3GetObjectAttributes SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CreateMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3UploadPartCopy SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3UploadPart SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3AbortMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CompleteMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3PutObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3GetObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_ReadBufferFromS3Bytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_WriteBufferFromS3Bytes SimpleAggregateFunction(sum, UInt64), lc_org_id SimpleAggregateFunction(max, String), lc_user_id SimpleAggregateFunction(max, Int64), lc_session_id SimpleAggregateFunction(max, String), lc_is_impersonated SimpleAggregateFunction(max, Bool), lc_workflow SimpleAggregateFunction(max, LowCardinality(String)), lc_kind SimpleAggregateFunction(max, LowCardinality(String)), lc_id SimpleAggregateFunction(max, String), lc_route_id SimpleAggregateFunction(max, String), lc_access_method SimpleAggregateFunction(max, LowCardinality(String)), lc_api_key_label SimpleAggregateFunction(max, String), lc_api_key_mask SimpleAggregateFunction(max, String), lc_query_type SimpleAggregateFunction(max, LowCardinality(String)), lc_product SimpleAggregateFunction(max, LowCardinality(String)), lc_chargeable SimpleAggregateFunction(max, Bool), lc_name SimpleAggregateFunction(max, String), lc_request_name SimpleAggregateFunction(max, String), lc_client_query_id SimpleAggregateFunction(max, String), lc_dashboard_id SimpleAggregateFunction(max, Int64), lc_insight_id SimpleAggregateFunction(max, Int64), lc_cohort_id SimpleAggregateFunction(max, Int64), lc_batch_export_id SimpleAggregateFunction(max, String), lc_experiment_id SimpleAggregateFunction(max, Int64), lc_experiment_feature_flag_key SimpleAggregateFunction(max, String), lc_alert_config_id SimpleAggregateFunction(max, String), lc_feature SimpleAggregateFunction(max, LowCardinality(String)), lc_table_id SimpleAggregateFunction(max, String), lc_warehouse_query SimpleAggregateFunction(max, Bool), lc_person_on_events_mode SimpleAggregateFunction(max, LowCardinality(String)), lc_service_name SimpleAggregateFunction(max, LowCardinality(String)), lc_workload SimpleAggregateFunction(max, LowCardinality(String)), lc_query__kind SimpleAggregateFunction(max, LowCardinality(String)), lc_query__query SimpleAggregateFunction(max, String), lc_query SimpleAggregateFunction(max, String), lc_modifiers SimpleAggregateFunction(max, String), lc_temporal__workflow_namespace SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__workflow_type SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__workflow_id SimpleAggregateFunction(max, String), lc_temporal__workflow_run_id SimpleAggregateFunction(max, String), lc_temporal__activity_type SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__activity_id SimpleAggregateFunction(max, String), lc_temporal__attempt SimpleAggregateFunction(max, Int64), lc_dagster__job_name SimpleAggregateFunction(max, LowCardinality(String)), lc_dagster__run_id SimpleAggregateFunction(max, String), lc_dagster__owner SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_name SimpleAggregateFunction(max, String), lc_experiment_metric_name SimpleAggregateFunction(max, String), lc_experiment_execution_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_metric_type SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_funnel_order_type SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_exposures_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_metric_events_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_query_surface SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_precompute_table SimpleAggregateFunction(max, String), lc_experiment_query_group_id SimpleAggregateFunction(max, String), lc_experiment_precompute_skip_reason SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_scan_date_from SimpleAggregateFunction(max, String), lc_experiment_scan_date_to SimpleAggregateFunction(max, String), lc_precompute_window_start SimpleAggregateFunction(max, String), lc_precompute_window_end SimpleAggregateFunction(max, String), lc_plan_fingerprint SimpleAggregateFunction(max, String), lc_estimated_rows SimpleAggregateFunction(max, Int64), lc_estimated_bytes SimpleAggregateFunction(max, Int64)) AS WITH
JSONExtract(ql.log_comment, 'Tuple(team_id Int64, org_id String, user_id Int64, session_id String, is_impersonated Bool, workflow String, kind String, id String, route_id String, access_method String, api_key_label String, api_key_mask String, query_type String, product String, chargeable Bool, name String, request_name String, client_query_id String, dashboard_id Int64, insight_id Int64, cohort_id Int64, batch_export_id String, experiment_id Int64, experiment_feature_flag_key String, alert_config_id String, feature String, table_id String, warehouse_query Bool, person_on_events_mode String, service_name String, workload String, experiment_name String, experiment_metric_name String, experiment_execution_path String, experiment_metric_type String, experiment_funnel_order_type String, experiment_exposures_path String, experiment_metric_events_path String, experiment_query_surface String, experiment_precompute_table String, experiment_query_group_id String, experiment_precompute_skip_reason String, experiment_scan_date_from String, experiment_scan_date_to String, precompute_window_start String, precompute_window_end String, plan_fingerprint String, estimated_rows Int64, estimated_bytes Int64, temporal Tuple(workflow_namespace String, workflow_type String, workflow_id String, workflow_run_id String, activity_type String, activity_id String, attempt Int64), dagster Tuple(job_name String, run_id String, tags Tuple(owner String)), query Tuple(kind String, query String, source Tuple(kind String, query String)))') AS lc
SELECT
diff --git a/posthog/clickhouse/hcl/sql/prod-us/apm.sql b/posthog/clickhouse/hcl/sql/prod-us/apm.sql
index 5e6187899412..92e6f9ae1e8b 100644
--- a/posthog/clickhouse/hcl/sql/prod-us/apm.sql
+++ b/posthog/clickhouse/hcl/sql/prod-us/apm.sql
@@ -71,7 +71,9 @@ CREATE TABLE posthog.metrics4_input (
attributes Map(LowCardinality(String), String),
_partition UInt32,
_topic String,
- _offset UInt64
+ _offset UInt64,
+ _source_topic String,
+ _source_partition UInt32
) ENGINE = Null();
CREATE TABLE posthog.writable_logs34 (
time_bucket DateTime MATERIALIZED toStartOfDay(timestamp),
@@ -171,6 +173,15 @@ CREATE TABLE posthog.writable_metrics4_series (
time_bucket DateTime MATERIALIZED toStartOfHour(timestamp),
original_expiry_timestamp DateTime64(6)
) ENGINE = Distributed('logs', 'posthog', 'metrics4_series');
+CREATE TABLE posthog.writable_metrics_kafka_metrics (
+ _partition UInt32,
+ _topic String,
+ max_offset SimpleAggregateFunction(max, UInt64),
+ max_observed_timestamp SimpleAggregateFunction(max, DateTime64(9)),
+ max_timestamp SimpleAggregateFunction(max, DateTime64(9)),
+ max_created_at SimpleAggregateFunction(max, DateTime64(9)),
+ max_lag SimpleAggregateFunction(max, UInt64)
+) ENGINE = Distributed('logs', 'posthog', 'metrics_kafka_metrics');
CREATE TABLE posthog.writable_query_log_archive_v3 (
initial_query_start_time DateTime64(6, 'UTC'),
team_id Int64,
@@ -323,7 +334,7 @@ CREATE MATERIALIZED VIEW posthog.kafka_logs34_avro_mv TO posthog.writable_logs34
toUInt8(ifNull(pattern_version, 0)) AS pattern_version,
_headers.value[indexOf(_headers.name, 'internal-source')] AS internal_source
FROM posthog.kafka_logs_avro;
-CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64) AS SELECT
+CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_input (uuid String, team_id Int32, metric_name String, series_fingerprint UInt64, resource_fingerprint UInt64, timestamp DateTime64(6), observed_timestamp DateTime64(6), original_expiry_timestamp DateTime64(6), service_name String, metric_type String, value Float64, count UInt64, histogram_bounds Array(Float64), histogram_counts Array(UInt64), trace_id String, span_id String, trace_flags Int32, has_labels Bool, unit String, aggregation_temporality String, is_monotonic UInt8, instrumentation_scope String, resource_attributes Map(String, String), attributes Map(String, String), _partition UInt64, _topic LowCardinality(String), _offset UInt64, _source_topic String, _source_partition UInt32) AS SELECT
uuid,
toInt32OrZero(_headers.value[indexOf(_headers.name, 'team_id')]) AS team_id,
ifNull(metric_name, '') AS metric_name,
@@ -367,7 +378,9 @@ CREATE MATERIALIZED VIEW posthog.kafka_metrics_avro4_mv TO posthog.metrics4_inpu
) AS attributes,
_partition,
_topic,
- _offset
+ _offset,
+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,
+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partition
FROM posthog.kafka_metrics_avro4
WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULL
SETTINGS
@@ -485,6 +498,25 @@ CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics4_series TO posthog.wr
original_expiry_timestamp
FROM posthog.metrics4_input
WHERE has_labels;
+CREATE MATERIALIZED VIEW posthog.metrics4_input_to_metrics_kafka_metrics TO posthog.writable_metrics_kafka_metrics (_partition UInt32, _topic String, max_offset SimpleAggregateFunction(max, UInt64), max_observed_timestamp SimpleAggregateFunction(max, DateTime64(6)), max_created_at SimpleAggregateFunction(max, DateTime)) AS SELECT
+ kafka_partition AS _partition,
+ kafka_topic AS _topic,
+ maxSimpleState(kafka_offset) AS max_offset,
+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,
+ maxSimpleState(now()) AS max_created_at
+FROM
+ (
+ SELECT
+ kafka_source.1 AS kafka_topic,
+ kafka_source.2 AS kafka_partition,
+ kafka_source.3 AS kafka_offset,
+ observed_timestamp
+ FROM
+ posthog.metrics4_input ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source
+ WHERE kafka_topic != ''
+ )
+GROUP BY
+ kafka_partition, kafka_topic;
CREATE MATERIALIZED VIEW posthog.ops_query_log_archive_v3_mv TO posthog.writable_query_log_archive_v3 (initial_query_start_time DateTime64(6, 'UTC'), team_id Int64, initial_query_id String, is_initial_query SimpleAggregateFunction(max, UInt8), hostname SimpleAggregateFunction(max, LowCardinality(String)), user SimpleAggregateFunction(max, LowCardinality(String)), type SimpleAggregateFunction(max, Enum8('QueryStart'=1, 'QueryFinish'=2, 'ExceptionBeforeStart'=3, 'ExceptionWhileProcessing'=4)), event_time_microseconds SimpleAggregateFunction(max, DateTime64(6, 'UTC')), query_duration_ms SimpleAggregateFunction(max, UInt64), read_rows SimpleAggregateFunction(sum, UInt64), read_bytes SimpleAggregateFunction(sum, UInt64), written_rows SimpleAggregateFunction(sum, UInt64), written_bytes SimpleAggregateFunction(sum, UInt64), result_rows SimpleAggregateFunction(sum, UInt64), result_bytes SimpleAggregateFunction(sum, UInt64), memory_usage SimpleAggregateFunction(max, UInt64), memory_usage_sum SimpleAggregateFunction(sum, UInt64), peak_threads_usage SimpleAggregateFunction(sum, UInt64), execution_count SimpleAggregateFunction(sum, UInt64), current_database SimpleAggregateFunction(max, LowCardinality(String)), query SimpleAggregateFunction(max, String), normalized_query_hash SimpleAggregateFunction(max, UInt64), query_kind_by_root SimpleAggregateFunction(max, Tuple(UInt8, String)), tables SimpleAggregateFunction(max, Array(String)), exception_code SimpleAggregateFunction(max, Int32), exception SimpleAggregateFunction(max, String), stack_trace SimpleAggregateFunction(max, String), ProfileEvents_SelectedRows SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_RealTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_OSCPUVirtualTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_UserTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_SystemTimeMicroseconds SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedParts SimpleAggregateFunction(sum, UInt64), ProfileEvents_SelectedMarks SimpleAggregateFunction(sum, UInt64), ProfileEvents_NetworkReceiveBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_NetworkSendBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_ReadBufferFromFileDescriptorReadBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_WriteBufferFromFileDescriptorWriteBytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3Clients SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3DeleteObjects SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CopyObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3ListObjects SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3HeadObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3GetObjectAttributes SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CreateMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3UploadPartCopy SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3UploadPart SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3AbortMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3CompleteMultipartUpload SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3PutObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_S3GetObject SimpleAggregateFunction(sum, UInt64), ProfileEvents_ReadBufferFromS3Bytes SimpleAggregateFunction(sum, UInt64), ProfileEvents_WriteBufferFromS3Bytes SimpleAggregateFunction(sum, UInt64), lc_org_id SimpleAggregateFunction(max, String), lc_user_id SimpleAggregateFunction(max, Int64), lc_session_id SimpleAggregateFunction(max, String), lc_is_impersonated SimpleAggregateFunction(max, Bool), lc_workflow SimpleAggregateFunction(max, LowCardinality(String)), lc_kind SimpleAggregateFunction(max, LowCardinality(String)), lc_id SimpleAggregateFunction(max, String), lc_route_id SimpleAggregateFunction(max, String), lc_access_method SimpleAggregateFunction(max, LowCardinality(String)), lc_api_key_label SimpleAggregateFunction(max, String), lc_api_key_mask SimpleAggregateFunction(max, String), lc_query_type SimpleAggregateFunction(max, LowCardinality(String)), lc_product SimpleAggregateFunction(max, LowCardinality(String)), lc_chargeable SimpleAggregateFunction(max, Bool), lc_name SimpleAggregateFunction(max, String), lc_request_name SimpleAggregateFunction(max, String), lc_client_query_id SimpleAggregateFunction(max, String), lc_dashboard_id SimpleAggregateFunction(max, Int64), lc_insight_id SimpleAggregateFunction(max, Int64), lc_cohort_id SimpleAggregateFunction(max, Int64), lc_batch_export_id SimpleAggregateFunction(max, String), lc_experiment_id SimpleAggregateFunction(max, Int64), lc_experiment_feature_flag_key SimpleAggregateFunction(max, String), lc_alert_config_id SimpleAggregateFunction(max, String), lc_feature SimpleAggregateFunction(max, LowCardinality(String)), lc_table_id SimpleAggregateFunction(max, String), lc_warehouse_query SimpleAggregateFunction(max, Bool), lc_person_on_events_mode SimpleAggregateFunction(max, LowCardinality(String)), lc_service_name SimpleAggregateFunction(max, LowCardinality(String)), lc_workload SimpleAggregateFunction(max, LowCardinality(String)), lc_query__kind SimpleAggregateFunction(max, LowCardinality(String)), lc_query__query SimpleAggregateFunction(max, String), lc_query SimpleAggregateFunction(max, String), lc_modifiers SimpleAggregateFunction(max, String), lc_temporal__workflow_namespace SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__workflow_type SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__workflow_id SimpleAggregateFunction(max, String), lc_temporal__workflow_run_id SimpleAggregateFunction(max, String), lc_temporal__activity_type SimpleAggregateFunction(max, LowCardinality(String)), lc_temporal__activity_id SimpleAggregateFunction(max, String), lc_temporal__attempt SimpleAggregateFunction(max, Int64), lc_dagster__job_name SimpleAggregateFunction(max, LowCardinality(String)), lc_dagster__run_id SimpleAggregateFunction(max, String), lc_dagster__owner SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_name SimpleAggregateFunction(max, String), lc_experiment_metric_name SimpleAggregateFunction(max, String), lc_experiment_execution_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_metric_type SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_funnel_order_type SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_exposures_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_metric_events_path SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_query_surface SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_precompute_table SimpleAggregateFunction(max, String), lc_experiment_query_group_id SimpleAggregateFunction(max, String), lc_experiment_precompute_skip_reason SimpleAggregateFunction(max, LowCardinality(String)), lc_experiment_scan_date_from SimpleAggregateFunction(max, String), lc_experiment_scan_date_to SimpleAggregateFunction(max, String), lc_precompute_window_start SimpleAggregateFunction(max, String), lc_precompute_window_end SimpleAggregateFunction(max, String), lc_plan_fingerprint SimpleAggregateFunction(max, String), lc_estimated_rows SimpleAggregateFunction(max, Int64), lc_estimated_bytes SimpleAggregateFunction(max, Int64)) AS WITH
JSONExtract(ql.log_comment, 'Tuple(team_id Int64, org_id String, user_id Int64, session_id String, is_impersonated Bool, workflow String, kind String, id String, route_id String, access_method String, api_key_label String, api_key_mask String, query_type String, product String, chargeable Bool, name String, request_name String, client_query_id String, dashboard_id Int64, insight_id Int64, cohort_id Int64, batch_export_id String, experiment_id Int64, experiment_feature_flag_key String, alert_config_id String, feature String, table_id String, warehouse_query Bool, person_on_events_mode String, service_name String, workload String, experiment_name String, experiment_metric_name String, experiment_execution_path String, experiment_metric_type String, experiment_funnel_order_type String, experiment_exposures_path String, experiment_metric_events_path String, experiment_query_surface String, experiment_precompute_table String, experiment_query_group_id String, experiment_precompute_skip_reason String, experiment_scan_date_from String, experiment_scan_date_to String, precompute_window_start String, precompute_window_end String, plan_fingerprint String, estimated_rows Int64, estimated_bytes Int64, temporal Tuple(workflow_namespace String, workflow_type String, workflow_id String, workflow_run_id String, activity_type String, activity_id String, attempt Int64), dagster Tuple(job_name String, run_id String, tags Tuple(owner String)), query Tuple(kind String, query String, source Tuple(kind String, query String)))') AS lc
SELECT
diff --git a/posthog/clickhouse/metrics/__init__.py b/posthog/clickhouse/metrics/__init__.py
index 9915279e3440..bbd29551d8e0 100644
--- a/posthog/clickhouse/metrics/__init__.py
+++ b/posthog/clickhouse/metrics/__init__.py
@@ -53,6 +53,8 @@ from .metrics4 import (
METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV,
METRICS4_INPUT_TO_METRICS4_SAMPLES_MV,
METRICS4_INPUT_TO_METRICS4_SERIES_MV,
+ METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV,
+ METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT,
METRICS4_NAMES_TABLE_SQL,
METRICS4_SAMPLES_TABLE_SQL,
METRICS4_SERIES_TABLE_SQL,
@@ -61,6 +63,7 @@ from .metrics4 import (
WRITABLE_METRICS4_NAMES_TABLE_SQL,
WRITABLE_METRICS4_SAMPLES_TABLE_SQL,
WRITABLE_METRICS4_SERIES_TABLE_SQL,
+ WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL,
)
__all__ = [
@@ -80,6 +83,8 @@ __all__ = [
"METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV",
"METRICS4_INPUT_TO_METRICS4_SAMPLES_MV",
"METRICS4_INPUT_TO_METRICS4_SERIES_MV",
+ "METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV",
+ "METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT",
"METRICS4_NAMES_TABLE_SQL",
"METRICS4_SAMPLES_TABLE_SQL",
"METRICS4_SERIES_TABLE_SQL",
@@ -88,6 +93,7 @@ __all__ = [
"WRITABLE_METRICS4_NAMES_TABLE_SQL",
"WRITABLE_METRICS4_SAMPLES_TABLE_SQL",
"WRITABLE_METRICS4_SERIES_TABLE_SQL",
+ "WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL",
"KAFKA_METRICS_AVRO2_MV",
"KAFKA_METRICS_AVRO2_TABLE_SQL",
"METRIC_ATTRIBUTES2_DISTRIBUTED_TABLE_SQL",
diff --git a/posthog/clickhouse/metrics/kafka_metrics.py b/posthog/clickhouse/metrics/kafka_metrics.py
index 729a59d13110..589f17c48399 100644
--- a/posthog/clickhouse/metrics/kafka_metrics.py
+++ b/posthog/clickhouse/metrics/kafka_metrics.py
@@ -140,17 +140,20 @@ SETTINGS
"""
-def METRICS_KAFKA_METRICS_TABLE_SQL():
- return f"""
-CREATE TABLE IF NOT EXISTS {settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE}.{KAFKA_METRICS_TABLE_NAME}
-(
- `_partition` UInt32,
+KAFKA_METRICS_COLUMNS = """ `_partition` UInt32,
`_topic` String,
`max_offset` SimpleAggregateFunction(max, UInt64),
`max_observed_timestamp` SimpleAggregateFunction(max, DateTime64(9)),
`max_timestamp` SimpleAggregateFunction(max, DateTime64(9)),
`max_created_at` SimpleAggregateFunction(max, DateTime64(9)),
- `max_lag` SimpleAggregateFunction(max, UInt64)
+ `max_lag` SimpleAggregateFunction(max, UInt64)"""
+
+
+def METRICS_KAFKA_METRICS_TABLE_SQL():
+ return f"""
+CREATE TABLE IF NOT EXISTS {settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE}.{KAFKA_METRICS_TABLE_NAME}
+(
+{KAFKA_METRICS_COLUMNS}
)
ENGINE = {AggregatingMergeTree(KAFKA_METRICS_TABLE_NAME, replication_scheme=ReplicationScheme.REPLICATED)}
ORDER BY (_topic, _partition)
diff --git a/posthog/clickhouse/metrics/metrics2.py b/posthog/clickhouse/metrics/metrics2.py
index df8a9c3a4466..fe0eb68bc491 100644
--- a/posthog/clickhouse/metrics/metrics2.py
+++ b/posthog/clickhouse/metrics/metrics2.py
@@ -83,7 +83,11 @@ def KAFKA_METRICS_AVRO2_TABLE_SQL() -> str:
return kafka_metrics_avro_table_sql(KAFKA_TABLE_NAME, KAFKA_GROUP)
-def metrics_input_table_sql(table_name: str) -> str:
+def metrics_input_table_sql(table_name: str, *, source_partition: bool = False) -> str:
+ # Only the metrics4 chain records where the Node consumer read each row.
+ source_partition_columns = (
+ ",\n `_source_topic` String,\n `_source_partition` UInt32" if source_partition else ""
+ )
return f"""
CREATE TABLE IF NOT EXISTS {_db()}.{table_name}
(
@@ -113,7 +117,7 @@ CREATE TABLE IF NOT EXISTS {_db()}.{table_name}
`attributes` Map(LowCardinality(String), String),
`_partition` UInt32,
`_topic` String,
- `_offset` UInt64
+ `_offset` UInt64{source_partition_columns}
)
ENGINE = Null
"""
@@ -271,8 +275,14 @@ def METRIC_ATTRIBUTES2_DISTRIBUTED_TABLE_SQL() -> str:
return _distributed_sql(METRIC_ATTRIBUTES_DISTRIBUTED_TABLE_NAME, METRIC_ATTRIBUTES2_TABLE_NAME)
-def kafka_metrics_avro_mv_select(table_name: str) -> str:
+def kafka_metrics_avro_mv_select(table_name: str, *, source_partition: bool = False) -> str:
db = _db()
+ source_partition_columns = (
+ ",\n _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,"
+ "\n toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partition"
+ if source_partition
+ else ""
+ )
sorted_resource_attributes = "mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), resource_attributes))"
sorted_attributes = "mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), attributes))"
labelled = "toBool(ifNull(has_labels, 1))"
@@ -306,7 +316,7 @@ def kafka_metrics_avro_mv_select(table_name: str) -> str:
if({labelled}, {sorted_attributes}, CAST(map(), 'Map(String, String)')) AS attributes,
_partition,
_topic,
- _offset
+ _offset{source_partition_columns}
FROM {db}.{table_name}
WHERE {table_name}.series_fingerprint IS NOT NULL
SETTINGS
diff --git a/posthog/clickhouse/metrics/metrics4.py b/posthog/clickhouse/metrics/metrics4.py
index a5d08a8b1f34..373a20d56d05 100644
--- a/posthog/clickhouse/metrics/metrics4.py
+++ b/posthog/clickhouse/metrics/metrics4.py
@@ -32,6 +32,7 @@ from django.conf import settings
from posthog.clickhouse.table_engines import AggregatingMergeTree, Distributed, ReplacingMergeTree, ReplicationScheme
+from .kafka_metrics import KAFKA_METRICS_COLUMNS, KAFKA_METRICS_TABLE_NAME
from .metrics2 import kafka_metrics_avro_mv_select, kafka_metrics_avro_table_sql, metrics_input_table_sql
KAFKA_METRICS4_TABLE_NAME = "kafka_metrics_avro4"
@@ -47,6 +48,7 @@ WRITABLE_METRICS4_SAMPLES_TABLE_NAME = "writable_metrics4_samples"
WRITABLE_METRICS4_SERIES_TABLE_NAME = "writable_metrics4_series"
WRITABLE_METRICS4_NAMES_TABLE_NAME = "writable_metrics4_names"
WRITABLE_METRICS4_ATTRIBUTES_TABLE_NAME = "writable_metrics4_attributes"
+WRITABLE_METRICS_KAFKA_METRICS_TABLE_NAME = f"writable_{KAFKA_METRICS_TABLE_NAME}"
METRICS4_MAX_SAMPLES_PER_SERIES_HOUR = 10_000
METRICS4_VIEW_NAME = "metrics4_view"
# The first hour that metrics4_samples serves. The view skips earlier rows, which overlap the old metrics2 data.
@@ -94,14 +96,14 @@ def KAFKA_METRICS_AVRO4_TABLE_SQL() -> str:
def METRICS4_INPUT_TABLE_SQL() -> str:
- return metrics_input_table_sql(METRICS4_INPUT_TABLE_NAME)
+ return metrics_input_table_sql(METRICS4_INPUT_TABLE_NAME, source_partition=True)
def KAFKA_METRICS_AVRO4_MV() -> str:
db = _db()
return f"""
CREATE MATERIALIZED VIEW IF NOT EXISTS {db}.{KAFKA_METRICS4_TABLE_NAME}_mv TO {db}.{METRICS4_INPUT_TABLE_NAME}
-AS {kafka_metrics_avro_mv_select(KAFKA_METRICS4_TABLE_NAME)}
+AS {kafka_metrics_avro_mv_select(KAFKA_METRICS4_TABLE_NAME, source_partition=True)}
"""
@@ -194,6 +196,12 @@ def WRITABLE_METRICS4_ATTRIBUTES_TABLE_SQL() -> str:
)
+def WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL() -> str:
+ return _writable_table_sql(
+ WRITABLE_METRICS_KAFKA_METRICS_TABLE_NAME, KAFKA_METRICS_TABLE_NAME, KAFKA_METRICS_COLUMNS
+ )
+
+
def METRICS4_SAMPLES_TABLE_SQL() -> str:
arrays = ",\n".join(
f" `{name}_arr` SimpleAggregateFunction(groupArrayArray({METRICS4_MAX_SAMPLES_PER_SERIES_HOUR}), Array({element_type})){_ARRAY_CODECS.get(name, '')}"
@@ -388,6 +396,45 @@ GROUP BY
"""
+def METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT() -> str:
+ # Each row counts toward its clickhouse_metrics partition and the ingestion-metrics partition
+ # the Node consumer read it from. Both topics partition at random, so lag on either hop would
+ # hide behind the other. A row without the source header has an empty source topic, and the
+ # WHERE skips it, so it cannot add a partition that never moves. The source offset is unknown.
+ return f"""SELECT
+ kafka_partition AS _partition,
+ kafka_topic AS _topic,
+ maxSimpleState(kafka_offset) AS max_offset,
+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,
+ maxSimpleState(now()) AS max_created_at
+FROM
+(
+ SELECT
+ kafka_source.1 AS kafka_topic,
+ kafka_source.2 AS kafka_partition,
+ kafka_source.3 AS kafka_offset,
+ observed_timestamp
+ FROM {_db()}.{METRICS4_INPUT_TABLE_NAME}
+ ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source
+ WHERE kafka_topic != ''
+)
+GROUP BY kafka_partition, kafka_topic"""
+
+
+def METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV() -> str:
+ """The newest capture time and offset each Kafka partition has delivered, one row per insert block.
+
+ The floor across partitions is how far every partition has reached, which metrics alerts read.
+ `max_timestamp` and `max_lag` stay unset because no reader uses them, and a sender that dates
+ points ahead would pin `max_timestamp`.
+ """
+ db = _db()
+ return f"""
+CREATE MATERIALIZED VIEW IF NOT EXISTS {db}.{METRICS4_INPUT_TABLE_NAME}_to_{KAFKA_METRICS_TABLE_NAME} TO {db}.{WRITABLE_METRICS_KAFKA_METRICS_TABLE_NAME}
+AS {METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT()}
+"""
+
+
def METRICS4_INPUT_TO_METRICS4_SERIES_MV() -> str:
db = _db()
return f"""
diff --git a/posthog/clickhouse/migrations/0370_metrics4_kafka_metrics_checkpoint.py b/posthog/clickhouse/migrations/0370_metrics4_kafka_metrics_checkpoint.py
new file mode 100644
index 000000000000..814957e3fc2f
--- /dev/null
+++ b/posthog/clickhouse/migrations/0370_metrics4_kafka_metrics_checkpoint.py
@@ -0,0 +1,34 @@
+from django.conf import settings
+
+from posthog.clickhouse.client.connection import NodeRole
+from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions
+from posthog.clickhouse.metrics import (
+ KAFKA_METRICS_AVRO4_MV,
+ METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV,
+ WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL,
+)
+
+DB = settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE
+
+# metrics_kafka_metrics already exists on the logs cluster: 0309 created it, and 0340 dropped the
+# only view that wrote to it. The new view counts each row toward both Kafka partitions it passed
+# through. The Node consumer sends the ingestion-metrics partition in the `source_topic` and
+# `source_partition` headers.
+#
+# The source columns go on metrics4_input before the views that read or write them, and the
+# writable table goes before the view that inserts into it. The Kafka view gets new output columns,
+# so it is dropped and created again. The Kafka table stays, with the same consumer group, so the
+# gap causes lag and no data loss.
+operations = [
+ run_sql_with_exceptions(
+ f"ALTER TABLE {DB}.metrics4_input ADD COLUMN IF NOT EXISTS _source_topic String, "
+ "ADD COLUMN IF NOT EXISTS _source_partition UInt32",
+ node_roles=[NodeRole.APM],
+ sharded=False,
+ is_alter_on_replicated_table=False,
+ ),
+ run_sql_with_exceptions(WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL(), node_roles=[NodeRole.APM]),
+ run_sql_with_exceptions(METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV(), node_roles=[NodeRole.APM]),
+ run_sql_with_exceptions(f"DROP TABLE IF EXISTS {DB}.kafka_metrics_avro4_mv", node_roles=[NodeRole.APM]),
+ run_sql_with_exceptions(KAFKA_METRICS_AVRO4_MV(), node_roles=[NodeRole.APM]),
+]
diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txt
index 731ac00dcf4d..754af8720b55 100644
--- a/posthog/clickhouse/migrations/max_migration.txt
+++ b/posthog/clickhouse/migrations/max_migration.txt
@@ -1 +1 @@
-0369_person_group_membership_ingestion
+0370_metrics4_kafka_metrics_checkpoint
diff --git a/posthog/clickhouse/schema.py b/posthog/clickhouse/schema.py
index bce58280a253..114f764c433b 100644
--- a/posthog/clickhouse/schema.py
+++ b/posthog/clickhouse/schema.py
@@ -69,15 +69,18 @@ from posthog.clickhouse.metrics import (
METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV,
METRICS4_INPUT_TO_METRICS4_SAMPLES_MV,
METRICS4_INPUT_TO_METRICS4_SERIES_MV,
+ METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV,
METRICS4_NAMES_TABLE_SQL,
METRICS4_SAMPLES_TABLE_SQL,
METRICS4_SERIES_TABLE_SQL,
METRICS4_VIEW_SQL,
METRICS_DISTRIBUTED_TABLE_SQL,
+ METRICS_KAFKA_METRICS_TABLE_SQL,
WRITABLE_METRICS4_ATTRIBUTES_TABLE_SQL,
WRITABLE_METRICS4_NAMES_TABLE_SQL,
WRITABLE_METRICS4_SAMPLES_TABLE_SQL,
WRITABLE_METRICS4_SERIES_TABLE_SQL,
+ WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL,
)
from posthog.clickhouse.plugin_log_entries import (
KAFKA_PLUGIN_LOG_ENTRIES_TABLE_SQL,
@@ -526,6 +529,7 @@ CREATE_MERGETREE_TABLE_QUERIES = (
METRICS4_SERIES_TABLE_SQL,
METRICS4_NAMES_TABLE_SQL,
METRICS4_ATTRIBUTES_TABLE_SQL,
+ METRICS_KAFKA_METRICS_TABLE_SQL,
TOPHOG_DATA_TABLE_SQL,
BILLING_USAGE_RECORDS_DATA_TABLE_SQL,
AI_EVENTS_DATA_TABLE_SQL,
@@ -631,6 +635,7 @@ CREATE_DISTRIBUTED_TABLE_QUERIES = (
WRITABLE_METRICS4_SERIES_TABLE_SQL,
WRITABLE_METRICS4_NAMES_TABLE_SQL,
WRITABLE_METRICS4_ATTRIBUTES_TABLE_SQL,
+ WRITABLE_METRICS_KAFKA_METRICS_TABLE_SQL,
WRITABLE_TOPHOG_TABLE_SQL,
DISTRIBUTED_TOPHOG_TABLE_SQL,
DISTRIBUTED_BILLING_USAGE_RECORDS_TABLE_SQL,
@@ -729,6 +734,7 @@ CREATE_MV_TABLE_QUERIES = (
METRICS4_INPUT_TO_METRICS4_NAMES_MV,
METRICS4_INPUT_TO_METRICS4_ATTRIBUTES_MV,
METRICS4_INPUT_TO_METRICS4_RESOURCE_ATTRIBUTES_MV,
+ METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV,
KAFKA_METRICS_AVRO4_MV,
TOPHOG_MV_SQL,
BILLING_USAGE_RECORDS_MV_SQL,
diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambr
index fec26e9b4e92..f4ddd900dc70 100644
--- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr
+++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr
@@ -3292,7 +3292,9 @@
if(toBool(ifNull(has_labels, 1)), mapSort(mapApply((k, v) -> (k, JSONExtractString(v)), attributes)), CAST(map(), 'Map(String, String)')) AS attributes,
_partition,
_topic,
- _offset
+ _offset,
+ _headers.value[indexOf(_headers.name, 'source_topic')] AS _source_topic,
+ toUInt32OrZero(_headers.value[indexOf(_headers.name, 'source_partition')]) AS _source_partition
FROM posthog_test.kafka_metrics_avro4
WHERE kafka_metrics_avro4.series_fingerprint IS NOT NULL
SETTINGS
@@ -5196,7 +5198,9 @@
`attributes` Map(LowCardinality(String), String),
`_partition` UInt32,
`_topic` String,
- `_offset` UInt64
+ `_offset` UInt64,
+ `_source_topic` String,
+ `_source_partition` UInt32
)
ENGINE = Null
@@ -5393,6 +5397,31 @@
'''
# ---
+# name: test_create_table_query[metrics4_input_to_metrics_kafka_metrics]
+ '''
+
+ CREATE MATERIALIZED VIEW IF NOT EXISTS posthog_test.metrics4_input_to_metrics_kafka_metrics TO posthog_test.writable_metrics_kafka_metrics
+ AS SELECT
+ kafka_partition AS _partition,
+ kafka_topic AS _topic,
+ maxSimpleState(kafka_offset) AS max_offset,
+ maxSimpleState(observed_timestamp) AS max_observed_timestamp,
+ maxSimpleState(now()) AS max_created_at
+ FROM
+ (
+ SELECT
+ kafka_source.1 AS kafka_topic,
+ kafka_source.2 AS kafka_partition,
+ kafka_source.3 AS kafka_offset,
+ observed_timestamp
+ FROM posthog_test.metrics4_input
+ ARRAY JOIN [(_topic, _partition, _offset), (_source_topic, _source_partition, 0)] AS kafka_source
+ WHERE kafka_topic != ''
+ )
+ GROUP BY kafka_partition, kafka_topic
+
+ '''
+# ---
# name: test_create_table_query[metrics4_names]
'''
@@ -5530,6 +5559,26 @@
# name: test_create_table_query[metrics_distributed]
"CREATE TABLE IF NOT EXISTS posthog_test.metrics_distributed AS posthog_test.metrics2 ENGINE = Distributed('posthog_single_shard', 'posthog_test', 'metrics2')"
# ---
+# name: test_create_table_query[metrics_kafka_metrics]
+ '''
+
+ CREATE TABLE IF NOT EXISTS posthog_test.metrics_kafka_metrics
+ (
+ `_partition` UInt32,
+ `_topic` String,
+ `max_offset` SimpleAggregateFunction(max, UInt64),
+ `max_observed_timestamp` SimpleAggregateFunction(max, DateTime64(9)),
+ `max_timestamp` SimpleAggregateFunction(max, DateTime64(9)),
+ `max_created_at` SimpleAggregateFunction(max, DateTime64(9)),
+ `max_lag` SimpleAggregateFunction(max, UInt64)
+ )
+ ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.metrics_kafka_metrics', '{replica}-{shard}')
+ ORDER BY (_topic, _partition)
+ SETTINGS
+ index_granularity = 8192
+
+ '''
+# ---
# name: test_create_table_query[ops_query_log_archive_v3_mv]
'''
CREATE MATERIALIZED VIEW IF NOT EXISTS ops_query_log_archive_v3_mv
@@ -11235,6 +11284,23 @@
'''
# ---
+# name: test_create_table_query[writable_metrics_kafka_metrics]
+ '''
+
+ CREATE TABLE IF NOT EXISTS posthog_test.writable_metrics_kafka_metrics
+ (
+ `_partition` UInt32,
+ `_topic` String,
+ `max_offset` SimpleAggregateFunction(max, UInt64),
+ `max_observed_timestamp` SimpleAggregateFunction(max, DateTime64(9)),
+ `max_timestamp` SimpleAggregateFunction(max, DateTime64(9)),
+ `max_created_at` SimpleAggregateFunction(max, DateTime64(9)),
+ `max_lag` SimpleAggregateFunction(max, UInt64)
+ )
+ ENGINE = Distributed('posthog_single_shard', 'posthog_test', 'metrics_kafka_metrics')
+
+ '''
+# ---
# name: test_create_table_query[writable_person]
'''
@@ -13220,7 +13286,9 @@
`attributes` Map(LowCardinality(String), String),
`_partition` UInt32,
`_topic` String,
- `_offset` UInt64
+ `_offset` UInt64,
+ `_source_topic` String,
+ `_source_partition` UInt32
)
ENGINE = Null
@@ -13353,6 +13421,26 @@
'''
# ---
+# name: test_create_table_query_replicated_and_storage[metrics_kafka_metrics]
+ '''
+
+ CREATE TABLE IF NOT EXISTS posthog_test.metrics_kafka_metrics
+ (
+ `_partition` UInt32,
+ `_topic` String,
+ `max_offset` SimpleAggregateFunction(max, UInt64),
+ `max_observed_timestamp` SimpleAggregateFunction(max, DateTime64(9)),
+ `max_timestamp` SimpleAggregateFunction(max, DateTime64(9)),
+ `max_created_at` SimpleAggregateFunction(max, DateTime64(9)),
+ `max_lag` SimpleAggregateFunction(max, UInt64)
+ )
+ ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_noshard/posthog.metrics_kafka_metrics', '{replica}-{shard}')
+ ORDER BY (_topic, _partition)
+ SETTINGS
+ index_granularity = 8192
+
+ '''
+# ---
# name: test_create_table_query_replicated_and_storage[partitioned_sharded_posthog_document_embeddings]
'''
diff --git a/posthog/clickhouse/test/test_schema.py b/posthog/clickhouse/test/test_schema.py
index 00040a3fe191..7a2bd98c03a9 100644
--- a/posthog/clickhouse/test/test_schema.py
+++ b/posthog/clickhouse/test/test_schema.py
@@ -15,6 +15,7 @@ from posthog.hogql.database.schema.flag_evaluations import FLAG_EVALUATIONS_CLIC
from posthog.clickhouse.client import sync_execute
from posthog.clickhouse.kafka_engine import CONSUMER_GROUP_EVENTS_JSON_NATIVE_JSON, KAFKA_COLUMNS_WITH_PARTITION
from posthog.clickhouse.logs import LOGS34_TO_PATTERN_BUCKETS_MV_SELECT, LOGS34_TO_VOLUME_BUCKETS_MV_SELECT
+from posthog.clickhouse.metrics import METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT
from posthog.clickhouse.property_groups import flag_evaluations_property_group_definitions, iter_groups
from posthog.clickhouse.schema import (
CREATE_KAFKA_TABLE_QUERIES,
@@ -189,6 +190,41 @@ def test_logs_pattern_buckets_keep_stamped_empty_patterns_and_drop_unstamped_row
assert rows == [(5, "", "", 15, 1), (5, "GET <N>", "", 31, 2), (5, "GET <N>", "hog_flow", 15, 1)]
[email protected]("clickhouse_database")
+def test_metrics_kafka_metrics_counts_rows_toward_both_hops() -> None:
+ select = METRICS4_INPUT_TO_METRICS_KAFKA_METRICS_MV_SELECT().replace(
+ f"FROM {django_settings.CLICKHOUSE_LOGS_CLUSTER_DATABASE}.metrics4_input", "FROM checkpoint_input"
+ )
+ rows = sync_execute(
+ """
+ WITH checkpoint_input AS (
+ SELECT
+ row.1 AS _topic,
+ toUInt32(row.2) AS _partition,
+ toUInt64(row.3) AS _offset,
+ row.4 AS _source_topic,
+ toUInt32(row.5) AS _source_partition,
+ toDateTime64(row.6, 6, 'UTC') AS observed_timestamp
+ FROM (
+ SELECT arrayJoin([
+ ('clickhouse_metrics', 3, 10, 'ingestion_metrics', 7, '2026-01-02 12:00:00'),
+ ('clickhouse_metrics', 4, 20, 'ingestion_metrics', 7, '2026-01-02 12:00:30'),
+ ('clickhouse_metrics', 5, 11, '', 0, '2026-01-02 12:01:00')
+ ]) AS row
+ )
+ )
+ SELECT _topic, _partition, max_offset, toString(max_observed_timestamp) FROM ("""
+ + select
+ + ") ORDER BY _topic, _partition",
+ )
+ assert rows == [
+ ("clickhouse_metrics", 3, 10, "2026-01-02 12:00:00.000000"),
+ ("clickhouse_metrics", 4, 20, "2026-01-02 12:00:30.000000"),
+ ("clickhouse_metrics", 5, 11, "2026-01-02 12:01:00.000000"),
+ ("ingestion_metrics", 7, 0, "2026-01-02 12:00:30.000000"),
+ ]
+
+
def _column_definition_lines(block: str) -> Iterator[str]:
for raw_line in block.splitlines():
line = raw_line.strip().lstrip(",").strip()