Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.2 KB
Raw Blame
//! Shadow mode runs beside the Node consumer without affecting production.

pub mod config;
pub mod consumer_loop;
pub mod kafka_outputs;
pub mod kafka_source;
pub mod lazy_loader;
pub mod node_env;
pub mod postgres;
pub mod redis_deps;
pub mod shadow;
pub mod usage;

use std::collections::HashMap;
use std::future::ready;
use std::sync::Arc;
use std::time::Duration;

use axum::response::IntoResponse;
use axum::routing::get;
use axum::Router;
use lifecycle::{ComponentOptions, Manager};
use rdkafka::consumer::Consumer;
use rdkafka::metadata::Metadata;
use rdkafka::producer::{FutureProducer, Producer};
use sqlx::PgPool;

use crate::metric_rules::LogsMetricsEmitter;
use crate::pipeline::{
    DependencyRetry, LogsConsumerIdentity, LogsIngestionConsumer, LogsIngestionConsumerDeps,
};
use crate::rate_limiter::{system_clock, KeyRoot, RedisTokenBucket};

use config::{ConsumerKind, RunMode, ServiceConfig};
use consumer_loop::run_consumer_loop;
use kafka_outputs::{producer_client_config, KafkaOutputs};
use kafka_source::{consumer_client_config, FencingContext, KafkaBatchSource, LogsConsumer};
use node_env::NodeEnv;

const SERVICE_NAME: &str = "logs-ingestion";
/// How long a task may take to finish once the lifecycle monitor has returned.
const TASK_JOIN_GRACE: Duration = Duration::from_secs(2);

/// The shadow-mode rules, applied to any set of dependencies (README.md lists them).
pub fn shadow_deps(mut deps: LogsIngestionConsumerDeps) -> LogsIngestionConsumerDeps {
    deps.team_manager = Arc::new(shadow::ReadOnlyTeamManager(deps.team_manager));
    deps.usage_batch = Arc::new(shadow::DiscardUsageRecords);
    deps.metrics_emitter = deps
        .metrics_emitter
        .map(|_| Arc::new(shadow::SuppressedMetricsEmitter) as _);
    deps.logs_transformer = None;
    deps.key_root = KeyRoot::Shadow;
    deps.source_offset_header = true;
    deps
}

pub fn identity(kind: ConsumerKind) -> LogsConsumerIdentity {
    match kind {
        ConsumerKind::Logs => LogsConsumerIdentity::logs(),
        ConsumerKind::Traces => LogsConsumerIdentity::traces(),
    }
}

pub fn create_producers(config: &ServiceConfig) -> anyhow::Result<HashMap<String, FutureProducer>> {
    let mut out = HashMap::new();
    for (name, overrides) in &config.producers {
        let topics: Vec<&str> = config
            .routes
            .iter()
            .filter(|r| &r.producer == name)
            .map(|r| r.topic.as_str())
            .collect();
        if topics.is_empty() {
            continue;
        }
        let mut client = producer_client_config(overrides, config.kafka_client_rack.as_deref());
        if config.mode == RunMode::Shadow {
            // A mistyped shadow topic must fail here instead of creating a topic on the cluster.
            client.set("allow.auto.create.topics", "false");
        }
        let producer: FutureProducer = client.create()?;
        let metadata = producer
            .client()
            .fetch_metadata(None, Duration::from_secs(15))
            .map_err(|e| anyhow::anyhow!("producer {name} cannot reach Kafka: {e}"))?;
        if config.mode == RunMode::Shadow {
            for topic in topics {
                require_topic(&metadata, topic)?;
            }
        }
        out.insert(name.clone(), producer);
    }
    Ok(out)
}

/// Brokers below Metadata v4 ignore `allow.auto.create.topics` and create a named topic, so this checks the all-topics metadata.
fn require_topic(metadata: &Metadata, topic: &str) -> anyhow::Result<()> {
    let exists = metadata
        .topics()
        .iter()
        .any(|t| t.name() == topic && t.error().is_none() && !t.partitions().is_empty());
    anyhow::ensure!(exists, "shadow topic {topic} does not exist");
    Ok(())
}

pub fn create_consumer(config: &ServiceConfig) -> anyhow::Result<LogsConsumer> {
    let consumer: LogsConsumer = consumer_client_config(
        &config.group_id,
        &config.kafka_consumer,
        config.kafka_client_rack.as_deref(),
        config.node_env == NodeEnv::Test,
    )
    .create_with_context(FencingContext::default())?;
    consumer.subscribe(&[config.input_topic.as_str()])?;
    Ok(consumer)
}

fn pool(url: &str, size: u32) -> anyhow::Result<PgPool> {
    Ok(common_database::get_pool_with_config(
        url,
        common_database::PoolConfig {
            max_connections: size,
            pool_name: Some(SERVICE_NAME.to_string()),
            ..Default::default()
        },
    )?)
}

pub fn build_deps(
    config: &ServiceConfig,
    outputs: Arc<KafkaOutputs>,
) -> anyhow::Result<LogsIngestionConsumerDeps> {
    let reader = pool(&config.database_reader_url, config.pool_size)?;
    let writer = match (config.mode, &config.database_writer_url) {
        (RunMode::Primary, Some(url)) => Some(pool(url, config.pool_size)?),
        _ => None,
    };
    let clock = system_clock();
    let posthog_redis =
        redis_deps::ReconnectingRedis::new(redis_deps::connection_info(&config.posthog_redis)?)?;
    let logs_redis =
        redis_deps::ReconnectingRedis::new(redis_deps::connection_info(&config.logs_redis)?)?;
    let logs = config.kind == ConsumerKind::Logs;
    let metrics_enabled = !config.metrics_export_url.is_empty();
    let usage_batch: Arc<dyn crate::pipeline::UsageRecordBatch> = match config.mode {
        RunMode::Shadow => Arc::new(shadow::DiscardUsageRecords),
        RunMode::Primary => Arc::new(usage::GrpcUsageRecordBatch::new(
            &config.usage,
            if logs { "logs" } else { "apm-traces" },
            clock.clone(),
        )?),
    };
    let deps = LogsIngestionConsumerDeps {
        team_manager: Arc::new(postgres::PgTeamManager::new(reader.clone(), writer)),
        quota_limiting: Arc::new(redis_deps::RedisQuotaLimiting::new(
            posthog_redis,
            clock.clone(),
        )),
        outputs,
        usage_batch,
        sampling_rules: logs.then(|| Arc::new(postgres::PgSamplingRules::new(reader.clone())) as _),
        metric_rules: metrics_enabled
            .then(|| Arc::new(postgres::PgMetricRules::new(reader.clone())) as _),
        metrics_emitter: if metrics_enabled {
            Some(Arc::new(LogsMetricsEmitter::new(
                config.metrics_export_url.clone(),
                shadow::ReqwestOtlpTransport::new()?,
            )) as _)
        } else {
            None
        },
        logs_transformer: None,
        retention_rules: Some(Arc::new(postgres::PgRetentionRules::new(reader.clone()))),
        pattern_message_keys: logs
            .then(|| Arc::new(postgres::PgPatternMessageKeys::new(reader.clone())) as _),
        tracing_config: (!logs)
            .then(|| Arc::new(postgres::PgTracingConfig::new(reader.clone())) as _),
        dependency_retry: DependencyRetry::default(),
        internal_source_policies: None,
        token_bucket_redis: Arc::new(RedisTokenBucket::new(logs_redis)),
        key_root: if config.node_env == NodeEnv::Test {
            KeyRoot::Test
        } else {
            KeyRoot::Production
        },
        clock,
        source_offset_header: false,
    };
    Ok(match config.mode {
        RunMode::Primary => deps,
        RunMode::Shadow => shadow_deps(deps),
    })
}

pub fn health_router(
    readiness: lifecycle::ReadinessHandler,
    liveness: lifecycle::LivenessHandler,
    render_metrics: Arc<dyn Fn() -> String + Send + Sync>,
) -> Router {
    Router::new()
        .route("/", get(|| async { SERVICE_NAME }))
        .route(
            "/_health",
            get(move || {
                let liveness = liveness.clone();
                async move { liveness.check().into_response() }
            }),
        )
        .route(
            "/_ready",
            get(move || {
                let readiness = readiness.clone();
                async move { readiness.check().await }
            }),
        )
        .route("/metrics", get(move || ready(render_metrics())))
}

/// A failed batch ends the process with an error, as the Node consumer crashes, so the restart replays from the last commit.
pub async fn run(config: ServiceConfig) -> anyhow::Result<()> {
    tracing::info!(
        kind = ?config.kind,
        mode = ?config.mode,
        group_id = %config.group_id,
        input_topic = %config.input_topic,
        routes = ?config.routes,
        "starting logs-ingestion"
    );
    let recorder = serve_metrics::setup_metrics_recorder();
    let mut manager = Manager::builder(SERVICE_NAME)
        .with_global_shutdown_timeout(config.shutdown_timeout)
        .build();
    let metrics_handle =
        manager.register("metrics", ComponentOptions::new().is_observability(true));
    let consumer_handle = manager.register(
        "consumer",
        ComponentOptions::new()
            .with_graceful_shutdown(config.shutdown_timeout)
            .with_liveness_deadline(Duration::from_secs(300))
            .with_stall_threshold(2),
    );
    let readiness = manager.readiness_handler();
    let liveness = manager.liveness_handler();

    let producers = create_producers(&config)?;
    let outputs = Arc::new(KafkaOutputs::new(&config.routes, &producers)?);
    let deps = build_deps(&config, outputs.clone())?;
    let consumer = LogsIngestionConsumer::new(identity(config.kind), config.consumer.clone(), deps);
    let mut source = KafkaBatchSource::new(create_consumer(&config)?);
    let guard = manager.monitor_background();

    let loop_config = config.consumer_loop.clone();
    let consumer_task = tokio::spawn(async move {
        let handle = consumer_handle;
        let _scope = handle.process_scope();
        let tick_handle = handle.clone();
        let result = run_consumer_loop(
            &mut source,
            &consumer,
            &loop_config,
            handle.shutdown_signal(),
            &move || tick_handle.report_healthy(),
        )
        .await;
        outputs.flush(Duration::from_secs(10));
        match &result {
            Ok(stats) => tracing::info!(
                batches = stats.batches,
                messages = stats.messages,
                "consumer stopped"
            ),
            Err(error) => handle.signal_failure(format!("{error:#}")),
        }
        result
    });

    let app = health_router(readiness, liveness, Arc::new(move || recorder.render()));
    let listener = tokio::net::TcpListener::bind(("0.0.0.0", config.http_port)).await?;
    let server_task = tokio::spawn(async move {
        let result = axum::serve(listener, app)
            .with_graceful_shutdown(metrics_handle.shutdown_signal())
            .await;
        match &result {
            Ok(()) => metrics_handle.work_completed(),
            Err(error) => metrics_handle.signal_failure(format!("http server failed: {error}")),
        }
        result
    });
    // The monitor returns by the global shutdown deadline, so nothing after it waits unbounded.
    let monitor = guard.wait().await;
    let consumer_result = join_or_abort("consumer", consumer_task).await;
    let server_result = join_or_abort("http server", server_task).await;
    consumer_result??;
    server_result??;
    monitor?;
    Ok(())
}

async fn join_or_abort<T>(name: &str, mut task: tokio::task::JoinHandle<T>) -> anyhow::Result<T> {
    match tokio::time::timeout(TASK_JOIN_GRACE, &mut task).await {
        Ok(joined) => Ok(joined?),
        Err(_) => {
            task.abort();
            Err(anyhow::anyhow!(
                "{name} did not stop by the shutdown deadline"
            ))
        }
    }
}