Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

22.5 KB
Raw Blame
use std::collections::HashMap;

use anyhow::Result;
use base64::{prelude::BASE64_STANDARD, Engine};
use chrono::serde::ts_microseconds;
use chrono::DateTime;
use chrono::Utc;
use clickhouse::Row;
use opentelemetry_proto::tonic::{
    common::v1::{AnyValue, InstrumentationScope},
    resource::v1::Resource,
    trace::v1::Span,
};
use serde::{Deserialize, Serialize};
use serde_json::json;
use tracing::debug;
use uuid::Uuid;

use crate::log_record::{
    any_value_to_string, extract_resource_attributes, extract_span_id, extract_trace_id,
    override_timestamp,
};

#[derive(Row, Debug, Serialize, Deserialize)]
pub struct KafkaTraceRow {
    pub uuid: String,
    pub trace_id: String,
    pub span_id: String,
    pub parent_span_id: String,
    pub trace_state: String,
    pub name: String,
    pub kind: i32,
    pub flags: i32,
    #[serde(with = "ts_microseconds")]
    pub timestamp: DateTime<Utc>,
    #[serde(with = "ts_microseconds")]
    pub end_time: DateTime<Utc>,
    #[serde(with = "ts_microseconds")]
    pub observed_timestamp: DateTime<Utc>,
    pub service_name: String,
    pub resource_attributes: HashMap<String, String>,
    pub instrumentation_scope: String,
    pub attributes: HashMap<String, String>,
    pub dropped_attributes_count: i32,
    pub events: Vec<String>,
    pub dropped_events_count: i32,
    pub links: Vec<String>,
    pub dropped_links_count: i32,
    pub status_code: i32,
    pub status_message: String,
    pub bytes_uncompressed: Option<i64>,
    // Per-row retention in days. capture-logs always writes None — the Node consumer stamps
    // this from team retention rules. Null lets ClickHouse fall back to the batch header.
    pub retention_days: Option<i32>,
}

/// Sum byte lengths of the row's string, map, and array content. Fixed-width
/// fields (timestamps, kind, flags, status_code, dropped_*_count) are excluded —
/// only variable-length payload is counted. Distinct from the Kafka header
/// `bytes_uncompressed`, which is the raw HTTP body size for the whole batch.
pub fn compute_kafka_trace_row_bytes(row: &KafkaTraceRow) -> i64 {
    let mut total: usize = 0;
    total = total.saturating_add(row.uuid.len());
    total = total.saturating_add(row.trace_id.len());
    total = total.saturating_add(row.span_id.len());
    total = total.saturating_add(row.parent_span_id.len());
    total = total.saturating_add(row.trace_state.len());
    total = total.saturating_add(row.name.len());
    total = total.saturating_add(row.service_name.len());
    total = total.saturating_add(row.instrumentation_scope.len());
    total = total.saturating_add(row.status_message.len());
    for (k, v) in &row.resource_attributes {
        total = total.saturating_add(k.len()).saturating_add(v.len());
    }
    for (k, v) in &row.attributes {
        total = total.saturating_add(k.len()).saturating_add(v.len());
    }
    for event in &row.events {
        total = total.saturating_add(event.len());
    }
    for link in &row.links {
        total = total.saturating_add(link.len());
    }
    i64::try_from(total).unwrap_or(i64::MAX)
}

impl KafkaTraceRow {
    /// Set `bytes_uncompressed` from the row's variable-length content. Consuming
    /// builder; the `mut self` is encapsulated and never escapes.
    fn with_computed_bytes(mut self) -> Self {
        self.bytes_uncompressed = Some(compute_kafka_trace_row_bytes(&self));
        self
    }

    pub fn new(
        span: Span,
        resource: Option<Resource>,
        scope: Option<InstrumentationScope>,
    ) -> Result<(Self, bool)> {
        let trace_id = BASE64_STANDARD.encode(extract_trace_id(&span.trace_id));
        let span_id = BASE64_STANDARD.encode(extract_span_id(&span.span_id));
        let parent_span_id = BASE64_STANDARD.encode(extract_span_id(&span.parent_span_id));

        let resource_attributes = extract_resource_attributes(resource);
        let service_name = extract_string_from_resource(&resource_attributes, "service.name");

        let mut attributes: HashMap<String, String> = span
            .attributes
            .into_iter()
            .map(|kv| {
                (
                    kv.key,
                    any_value_to_string(kv.value.unwrap_or(AnyValue {
                        value: Some(
                            opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(
                                "".to_string(),
                            ),
                        ),
                    })),
                )
            })
            .collect();

        let instrumentation_scope = match scope {
            Some(s) => format!("{}@{}", s.name, s.version),
            None => "".to_string(),
        };

        let raw_timestamp = match span.start_time_unix_nano {
            0 => Utc::now(),
            _ => DateTime::<Utc>::from_timestamp_nanos(span.start_time_unix_nano.try_into()?),
        };
        let (timestamp, original_timestamp) = override_timestamp(raw_timestamp);
        let was_overridden = original_timestamp.is_some();
        if let Some(original) = original_timestamp {
            attributes.insert("$originalTimestamp".to_string(), original.to_rfc3339());
        }

        let end_time = match span.end_time_unix_nano {
            0 => timestamp,
            _ => DateTime::<Utc>::from_timestamp_nanos(span.end_time_unix_nano.try_into()?),
        };

        let events: Vec<String> = span
            .events
            .into_iter()
            .map(|event| {
                let attrs: HashMap<String, String> = event
                    .attributes
                    .into_iter()
                    .map(|kv| {
                        (
                            kv.key,
                            any_value_to_string(kv.value.unwrap_or(AnyValue {
                                value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue("".to_string())),
                            })),
                        )
                    })
                    .collect();
                json!({
                    "time_unix_nano": event.time_unix_nano,
                    "name": event.name,
                    "attributes": attrs,
                    "dropped_attributes_count": event.dropped_attributes_count,
                })
                .to_string()
            })
            .collect();

        let links: Vec<String> = span
            .links
            .into_iter()
            .map(|link| {
                let attrs: HashMap<String, String> = link
                    .attributes
                    .into_iter()
                    .map(|kv| {
                        (
                            kv.key,
                            any_value_to_string(kv.value.unwrap_or(AnyValue {
                                value: Some(opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue("".to_string())),
                            })),
                        )
                    })
                    .collect();
                json!({
                    "trace_id": BASE64_STANDARD.encode(extract_trace_id(&link.trace_id)),
                    "span_id": BASE64_STANDARD.encode(extract_span_id(&link.span_id)),
                    "trace_state": link.trace_state,
                    "attributes": attrs,
                    "dropped_attributes_count": link.dropped_attributes_count,
                    "flags": link.flags,
                })
                .to_string()
            })
            .collect();

        let status_code = span.status.as_ref().map(|s| s.code).unwrap_or(0);
        let status_message = span
            .status
            .as_ref()
            .map(|s| s.message.clone())
            .unwrap_or_default();

        let row = Self {
            uuid: Uuid::now_v7().to_string(),
            trace_id,
            span_id,
            parent_span_id,
            trace_state: span.trace_state,
            name: span.name,
            kind: span.kind,
            flags: span.flags as i32,
            timestamp,
            end_time,
            observed_timestamp: Utc::now(),
            service_name,
            resource_attributes,
            instrumentation_scope,
            attributes,
            dropped_attributes_count: span.dropped_attributes_count as i32,
            events,
            dropped_events_count: span.dropped_events_count as i32,
            links,
            dropped_links_count: span.dropped_links_count as i32,
            status_code,
            status_message,
            bytes_uncompressed: None,
            retention_days: None,
        }
        .with_computed_bytes();
        debug!("trace span: {:?}", row);

        Ok((row, was_overridden))
    }
}

fn extract_string_from_resource(attributes: &HashMap<String, String>, key: &str) -> String {
    if let Some(value) = attributes.get(key) {
        if let Ok(serde_json::Value::String(value)) =
            serde_json::from_str::<serde_json::Value>(value)
        {
            value.to_string()
        } else {
            value.to_string()
        }
    } else {
        "".to_string()
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use apache_avro::{Codec, Reader, Schema, Writer};
    use chrono::{TimeDelta, Utc};
    use opentelemetry_proto::tonic::{
        common::v1::{any_value, AnyValue, KeyValue},
        trace::v1::{span::Event, span::Link, Span, Status},
    };

    use crate::traces_avro_schema::TRACES_AVRO_SCHEMA;

    fn make_span() -> Span {
        let now_nanos = Utc::now().timestamp_nanos_opt().unwrap() as u64;
        Span {
            trace_id: vec![1u8; 16],
            span_id: vec![2u8; 8],
            parent_span_id: vec![3u8; 8],
            trace_state: "vendor=foo".to_string(),
            name: "test.operation".to_string(),
            kind: 2, // SERVER
            flags: 1,
            start_time_unix_nano: now_nanos - 1_000_000_000,
            end_time_unix_nano: now_nanos,
            attributes: vec![KeyValue {
                key: "http.method".to_string(),
                value: Some(AnyValue {
                    value: Some(any_value::Value::StringValue("GET".to_string())),
                }),
            }],
            dropped_attributes_count: 0,
            events: vec![],
            dropped_events_count: 0,
            links: vec![],
            dropped_links_count: 0,
            status: Some(Status {
                message: "".to_string(),
                code: 1, // OK
            }),
        }
    }

    #[test]
    fn test_compute_kafka_trace_row_bytes_sums_string_map_and_array_lengths() {
        let mut resource_attributes = HashMap::new();
        resource_attributes.insert("host.name".to_string(), "localhost".to_string());
        let mut attributes = HashMap::new();
        attributes.insert("a".to_string(), "b".to_string());
        let row = KafkaTraceRow {
            uuid: "uuid-1".to_string(),
            trace_id: "tid".to_string(),
            span_id: "sid".to_string(),
            parent_span_id: "psid".to_string(),
            trace_state: "ts".to_string(),
            name: "op".to_string(),
            kind: 0,
            flags: 0,
            timestamp: Utc::now(),
            end_time: Utc::now(),
            observed_timestamp: Utc::now(),
            service_name: "svc".to_string(),
            resource_attributes,
            instrumentation_scope: "scope".to_string(),
            attributes,
            dropped_attributes_count: 0,
            events: vec!["{}".to_string()],
            dropped_events_count: 0,
            links: vec!["{}".to_string()],
            dropped_links_count: 0,
            status_code: 0,
            status_message: "ok".to_string(),
            bytes_uncompressed: None,
            retention_days: None,
        };
        // strings: uuid(6) + tid(3) + sid(3) + psid(4) + ts(2) + op(2) + svc(3) + scope(5) + ok(2) = 30
        // resource_attributes: host.name(9)+localhost(9)=18; attributes a(1)+b(1)=2
        // events 1 entry of "{}"(2); links 1 entry of "{}"(2) = 4
        // total = 30 + 18 + 2 + 4 = 54
        assert_eq!(compute_kafka_trace_row_bytes(&row), 54);
    }

    #[test]
    fn test_bytes_uncompressed_serialises_into_avro_payload() {
        // Decode to raw Avro Value rather than struct: apache_avro's `from_value` can't
        // resolve `["null", T]` unions into the existing non-Option fields on KafkaTraceRow.
        use apache_avro::types::Value;

        let (row, _) = KafkaTraceRow::new(make_span(), None, None).expect("ok");
        let expected = row.bytes_uncompressed.expect("populated");

        let schema = Schema::parse_str(TRACES_AVRO_SCHEMA).expect("schema parses");
        let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Null);
        writer.append_ser(&row).expect("append_ser ok");
        let payload = writer.into_inner().expect("flush ok");

        let reader = Reader::new(payload.as_slice()).expect("reader ok");
        let mut found_long: Option<i64> = None;
        for value in reader {
            let value = value.expect("decode ok");
            if let Value::Record(fields) = value {
                for (name, field_value) in fields {
                    if name == "bytes_uncompressed" {
                        if let Value::Union(_, inner) = field_value {
                            if let Value::Long(v) = *inner {
                                found_long = Some(v);
                            }
                        }
                    }
                }
            }
        }
        assert_eq!(found_long, Some(expected));
    }

    #[test]
    fn test_retention_days_serialises_into_avro_payload() {
        // The Node consumer stamps this field, so the schema must carry it even though
        // capture-logs always writes None.
        use apache_avro::types::Value;

        let (mut row, _) = KafkaTraceRow::new(make_span(), None, None).expect("ok");
        row.retention_days = Some(30);

        let schema = Schema::parse_str(TRACES_AVRO_SCHEMA).expect("schema parses");
        let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Null);
        writer.append_ser(&row).expect("append_ser ok");
        let payload = writer.into_inner().expect("flush ok");

        let reader = Reader::new(payload.as_slice()).expect("reader ok");
        let mut found_int: Option<i32> = None;
        for value in reader {
            let value = value.expect("decode ok");
            if let Value::Record(fields) = value {
                for (name, field_value) in fields {
                    if name == "retention_days" {
                        if let Value::Union(_, inner) = field_value {
                            if let Value::Int(v) = *inner {
                                found_int = Some(v);
                            }
                        }
                    }
                }
            }
        }
        assert_eq!(found_int, Some(30));
    }

    #[test]
    fn test_new_populates_bytes_uncompressed() {
        let (row, _) = KafkaTraceRow::new(make_span(), None, None).expect("ok");
        assert!(row.bytes_uncompressed.is_some());
        assert_eq!(
            row.bytes_uncompressed.unwrap(),
            compute_kafka_trace_row_bytes(&row),
        );
    }

    #[test]
    fn test_basic_span_conversion() {
        let span = make_span();
        let (row, was_overridden) = KafkaTraceRow::new(span, None, None).unwrap();

        assert!(!was_overridden);
        assert!(!row.uuid.is_empty());
        assert_eq!(row.trace_id, BASE64_STANDARD.encode([1u8; 16]));
        assert_eq!(row.span_id, BASE64_STANDARD.encode([2u8; 8]));
        assert_eq!(row.parent_span_id, BASE64_STANDARD.encode([3u8; 8]));
        assert_eq!(row.trace_state, "vendor=foo");
        assert_eq!(row.name, "test.operation");
        assert_eq!(row.kind, 2);
        assert_eq!(row.flags, 1);
        assert_eq!(row.status_code, 1);
        assert_eq!(
            row.attributes.get("http.method").map(|s| s.as_str()),
            Some("\"GET\"")
        );
    }

    #[test]
    fn test_empty_span_produces_valid_row() {
        let span = Span::default();
        let (row, was_overridden) = KafkaTraceRow::new(span, None, None).unwrap();

        // Zero timestamp defaults to now, which is within range — no override
        assert!(!was_overridden);
        assert!(!row.uuid.is_empty());
        assert_eq!(row.name, "");
        assert_eq!(row.kind, 0);
        assert_eq!(row.status_code, 0);
        assert!(row.events.is_empty());
        assert!(row.links.is_empty());
    }

    #[test]
    fn test_timestamp_override_far_past() {
        let far_past_nanos = (Utc::now() - TimeDelta::hours(48))
            .timestamp_nanos_opt()
            .unwrap() as u64;
        let span = Span {
            start_time_unix_nano: far_past_nanos,
            end_time_unix_nano: far_past_nanos + 1_000_000,
            ..Default::default()
        };
        let (row, was_overridden) = KafkaTraceRow::new(span, None, None).unwrap();

        assert!(was_overridden);
        assert!(row.attributes.contains_key("$originalTimestamp"));
    }

    #[test]
    fn test_timestamp_within_range_not_overridden() {
        let recent_nanos = (Utc::now() - TimeDelta::hours(1))
            .timestamp_nanos_opt()
            .unwrap() as u64;
        let span = Span {
            start_time_unix_nano: recent_nanos,
            end_time_unix_nano: recent_nanos + 1_000_000,
            ..Default::default()
        };
        let (row, was_overridden) = KafkaTraceRow::new(span, None, None).unwrap();

        assert!(!was_overridden);
        assert!(!row.attributes.contains_key("$originalTimestamp"));
    }

    #[test]
    fn test_events_serialized_as_array_of_json_strings() {
        let now_nanos = Utc::now().timestamp_nanos_opt().unwrap() as u64;
        let span = Span {
            start_time_unix_nano: now_nanos - 1_000_000,
            end_time_unix_nano: now_nanos,
            events: vec![
                Event {
                    time_unix_nano: now_nanos - 500_000,
                    name: "db.query".to_string(),
                    attributes: vec![KeyValue {
                        key: "db.statement".to_string(),
                        value: Some(AnyValue {
                            value: Some(any_value::Value::StringValue("SELECT 1".to_string())),
                        }),
                    }],
                    dropped_attributes_count: 0,
                },
                Event {
                    time_unix_nano: now_nanos - 100_000,
                    name: "exception".to_string(),
                    attributes: vec![],
                    dropped_attributes_count: 0,
                },
            ],
            ..Default::default()
        };
        let (row, _) = KafkaTraceRow::new(span, None, None).unwrap();

        assert_eq!(row.events.len(), 2);
        // Each element must be valid JSON
        let e0: serde_json::Value = serde_json::from_str(&row.events[0]).unwrap();
        assert_eq!(e0["name"], "db.query");
        let e1: serde_json::Value = serde_json::from_str(&row.events[1]).unwrap();
        assert_eq!(e1["name"], "exception");
    }

    #[test]
    fn test_links_serialized_as_array_of_json_strings() {
        let now_nanos = Utc::now().timestamp_nanos_opt().unwrap() as u64;
        let span = Span {
            start_time_unix_nano: now_nanos - 1_000_000,
            end_time_unix_nano: now_nanos,
            links: vec![Link {
                trace_id: vec![9u8; 16],
                span_id: vec![8u8; 8],
                trace_state: "linked=true".to_string(),
                attributes: vec![],
                dropped_attributes_count: 0,
                flags: 0,
            }],
            ..Default::default()
        };
        let (row, _) = KafkaTraceRow::new(span, None, None).unwrap();

        assert_eq!(row.links.len(), 1);
        let l0: serde_json::Value = serde_json::from_str(&row.links[0]).unwrap();
        assert_eq!(l0["trace_id"], BASE64_STANDARD.encode([9u8; 16]));
        assert_eq!(l0["span_id"], BASE64_STANDARD.encode([8u8; 8]));
        assert_eq!(l0["trace_state"], "linked=true");
    }

    #[test]
    fn test_status_code_extraction() {
        let now_nanos = Utc::now().timestamp_nanos_opt().unwrap() as u64;
        let span = Span {
            start_time_unix_nano: now_nanos - 1_000_000,
            end_time_unix_nano: now_nanos,
            status: Some(Status {
                message: "internal error".to_string(),
                code: 2, // ERROR
            }),
            ..Default::default()
        };
        let (row, _) = KafkaTraceRow::new(span, None, None).unwrap();
        assert_eq!(row.status_code, 2);
    }

    #[test]
    fn test_parent_span_id_encoding() {
        let now_nanos = Utc::now().timestamp_nanos_opt().unwrap() as u64;
        let parent = vec![0xAB, 0xCD, 0xEF, 0x01, 0x23, 0x45, 0x67, 0x89];
        let span = Span {
            start_time_unix_nano: now_nanos - 1_000_000,
            end_time_unix_nano: now_nanos,
            parent_span_id: parent.clone(),
            ..Default::default()
        };
        let (row, _) = KafkaTraceRow::new(span, None, None).unwrap();
        assert_eq!(row.parent_span_id, BASE64_STANDARD.encode(&parent));
    }

    #[test]
    fn test_resource_service_name_extracted() {
        use opentelemetry_proto::tonic::{
            common::v1::{any_value, AnyValue, KeyValue},
            resource::v1::Resource,
        };

        let now_nanos = Utc::now().timestamp_nanos_opt().unwrap() as u64;
        let span = Span {
            start_time_unix_nano: now_nanos - 1_000_000,
            end_time_unix_nano: now_nanos,
            ..Default::default()
        };
        let resource = Resource {
            attributes: vec![KeyValue {
                key: "service.name".to_string(),
                value: Some(AnyValue {
                    value: Some(any_value::Value::StringValue("my-service".to_string())),
                }),
            }],
            dropped_attributes_count: 0,
        };
        let (row, _) = KafkaTraceRow::new(span, Some(resource), None).unwrap();
        assert_eq!(row.service_name, "my-service");
    }

    #[test]
    fn test_instrumentation_scope_name_version() {
        use opentelemetry_proto::tonic::common::v1::InstrumentationScope;

        let now_nanos = Utc::now().timestamp_nanos_opt().unwrap() as u64;
        let span = Span {
            start_time_unix_nano: now_nanos - 1_000_000,
            end_time_unix_nano: now_nanos,
            ..Default::default()
        };
        let scope = InstrumentationScope {
            name: "opentelemetry-rust".to_string(),
            version: "0.22.0".to_string(),
            ..Default::default()
        };
        let (row, _) = KafkaTraceRow::new(span, None, Some(scope)).unwrap();
        assert_eq!(row.instrumentation_scope, "[email protected]");
    }
}