Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.0 KB
Raw Blame
use std::sync::Arc;

use ingestion_framework::{drop, ok, KafkaMessage, Outcome, PipelineResult, ProcessingStep};
use metrics::counter;

use crate::internal_logs::INTERNAL_LOGS_HEADER;

use super::context::{kafka_header, quarantine_log_message, LogsPipelineContext};
use super::deps::Team;
use super::errors::NamedError;
use super::types::{parse_min_timestamp_header, parse_size_header, LogsIngestionMessage};

pub struct ParseOutcome {
    pub parsed: Option<LogsIngestionMessage>,
    pub failure: Option<anyhow::Error>,
}

fn count_dropped(reason: &'static str) {
    counter!("logs_ingestion_message_dropped_count", "reason" => reason, "team_id" => "unknown")
        .increment(1);
}

async fn parse(
    context: &LogsPipelineContext,
    message: Arc<KafkaMessage>,
) -> anyhow::Result<PipelineResult<LogsIngestionMessage>> {
    let header = |name: &str| kafka_header(&message, name);
    let token = match header("token").as_deref() {
        Some(t) if !t.is_empty() => t.to_string(),
        _ => {
            tracing::warn!("missing_token");
            count_dropped("missing_token");
            return Ok(drop("missing_token"));
        }
    };

    let team: Option<Team> = match context.deps.team_manager.get_team_by_token(&token).await {
        Ok(team) => team,
        Err(error) => {
            tracing::error!(error = %error, "team_lookup_error");
            count_dropped("team_lookup_error");
            return Ok(drop("team_lookup_error"));
        }
    };
    let Some(team) = team else {
        tracing::warn!(token_with_no_team = %token, "team_not_found");
        count_dropped("team_not_found");
        return Ok(drop("team_not_found"));
    };

    let bytes_uncompressed = parse_size_header(header("bytes_uncompressed").as_deref());
    let bytes_uncompressed_records =
        parse_size_header(header("bytes_uncompressed_records").as_deref());
    let bytes_compressed = parse_size_header(header("bytes_compressed").as_deref());
    let record_count = parse_size_header(header("record_count").as_deref());
    let (
        Some(bytes_uncompressed),
        Some(bytes_uncompressed_records),
        Some(bytes_compressed),
        Some(record_count),
    ) = (
        bytes_uncompressed,
        bytes_uncompressed_records,
        bytes_compressed,
        record_count,
    )
    else {
        tracing::error!(
            team_id = team.id,
            partition = message.partition,
            offset = message.offset,
            "logs_ingestion_invalid_size_headers"
        );
        let error = anyhow::Error::new(NamedError::new("Error", "invalid_size_headers"));
        quarantine_log_message(context, &message, &error, Some(&token), Some(team.id)).await?;
        return Ok(drop("invalid_size_headers"));
    };

    if bytes_uncompressed_records > bytes_uncompressed {
        counter!("logs_ingestion_records_bytes_exceed_payload_total", "team_id" => team.id.to_string()).increment(1);
    }

    let internal_logs = context.internal_logs_verifier.verify(
        header(INTERNAL_LOGS_HEADER).as_deref(),
        team.id,
        header("created_at").as_deref(),
    );
    Ok(ok(LogsIngestionMessage {
        token,
        message: message.clone(),
        team_id: team.id,
        bytes_uncompressed,
        bytes_uncompressed_records,
        bytes_compressed,
        record_count,
        min_timestamp_micros: parse_min_timestamp_header(header("min_timestamp").as_deref()),
        backfill_requested: header("backfill_days").is_some(),
        internal_logs,
    }))
}

/// A failed quarantine is carried as a result, so the batch fails only once every message has settled.
pub fn create_settled_parse_step(
    context: Arc<LogsPipelineContext>,
) -> ProcessingStep<Arc<KafkaMessage>, ParseOutcome> {
    ProcessingStep::new("parseLogsMessageStep", move |message: Arc<KafkaMessage>| {
        let context = context.clone();
        async move {
            let result = match parse(&context, message.clone()).await {
                Ok(result) => result,
                Err(e) => {
                    // A failed quarantine of bad size headers: the second attempt names the DLQ error.
                    tracing::error!(error = %e, partition = message.partition, offset = message.offset, "logs_ingestion_parse_error");
                    count_dropped("parse_error");
                    if let Err(failure) =
                        quarantine_log_message(&context, &message, &e, None, None).await
                    {
                        return Ok(ok(ParseOutcome {
                            parsed: None,
                            failure: Some(failure),
                        }));
                    }
                    drop("parse_error")
                }
            };
            Ok(match result.outcome {
                Outcome::Ok(parsed) => ok(ParseOutcome {
                    parsed: Some(parsed),
                    failure: None,
                }),
                Outcome::NonOk(non_ok) => PipelineResult::new(Outcome::NonOk(non_ok)),
            })
        }
    })
}