Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.7 KB
Raw Blame
use std::sync::Arc;

use async_trait::async_trait;
use metrics::counter;

use crate::content_weights::{
    BatchContext, FilterResult, FilterStage, PipelineStage, StageDropStats, StagedRecord,
};
use crate::rate_limiter::{
    Clock, KeyRoot, KeyedRateLimitRequest, KeyedRateLimiter, KeyedRateLimiterConfig, ScriptVersion,
    TokenBucketRedis,
};
use crate::record::LogRecord;

use super::evaluate::{
    classify_sampling_record, evaluate_log_record, CompiledRuleSet, CostUnit, EvaluateResult,
    SamplingClassifyResult, SamplingDecision,
};

pub const SAMPLING_STAGE_NAME: &str = "sampling";
pub const SAMPLING_RATE_LIMITER_NAME: &str = "logs-sampling-rate";

fn record_bytes(record: &LogRecord) -> f64 {
    record.bytes_uncompressed().map_or(0.0, |n| n as f64)
}

fn count_eval_error(team_id: i64, phase: &'static str, error: &dyn std::fmt::Display) {
    let team: Arc<str> = Arc::from(team_id.to_string());
    counter!("logs_ingestion_sampling_eval_error_total", "team_id" => team, "phase" => phase)
        .increment(1);
    tracing::warn!(team_id, %error, "[logs-sampling] rule evaluation threw, keeping record");
}

/// An evaluation error keeps the record.
fn safe_classify(
    rule_set: &CompiledRuleSet,
    record: &LogRecord,
    team_id: i64,
) -> SamplingClassifyResult {
    classify_sampling_record(Some(rule_set), record).unwrap_or_else(|e| {
        count_eval_error(team_id, "classify", &e);
        SamplingClassifyResult::Resolved {
            decision: SamplingDecision::Keep,
            rule_id: None,
        }
    })
}

fn safe_evaluate(rule_set: &CompiledRuleSet, record: &LogRecord, team_id: i64) -> EvaluateResult {
    evaluate_log_record(Some(rule_set), record).unwrap_or_else(|e| {
        count_eval_error(team_id, "evaluate", &e);
        EvaluateResult {
            decision: SamplingDecision::Keep,
            rule_id: None,
        }
    })
}

pub struct LogsSamplingService {
    rate_limiter: KeyedRateLimiter,
}

impl LogsSamplingService {
    pub fn new(
        redis: Arc<dyn TokenBucketRedis>,
        ttl_seconds: f64,
        key_root: KeyRoot,
        clock: Clock,
    ) -> Self {
        let mut config = KeyedRateLimiterConfig::new(SAMPLING_RATE_LIMITER_NAME, key_root);
        config.ttl_seconds = Some(ttl_seconds);
        config.script_version = ScriptVersion::V3;
        LogsSamplingService {
            rate_limiter: KeyedRateLimiter::with_clock(config, redis, clock),
        }
    }

    pub async fn sample_records(
        &self,
        records: Vec<StagedRecord>,
        rule_set: &CompiledRuleSet,
        batch: &BatchContext,
        team_id: Option<i64>,
        header_bytes_uncompressed: f64,
    ) -> anyhow::Result<FilterResult> {
        let mut kept = Vec::with_capacity(records.len());
        let mut stats = StageDropStats::default();
        let drop = |stats: &mut StageDropStats, staged: &StagedRecord, rule_id: Option<&str>| {
            let rb = record_bytes(&staged.record);
            stats.records_dropped += 1.0;
            stats.bytes_dropped += rb;
            stats.content_bytes_dropped += batch.content_bytes_of(staged);
            if let Some(id) = rule_id {
                stats.records_dropped_by_rule_id.add(id, 1.0);
                stats.bytes_dropped_by_rule_id.add(id, rb);
            }
        };

        match team_id.filter(|_| rule_set.has_rate_limit_rules) {
            Some(team_id) => {
                let classifications: Vec<SamplingClassifyResult> = records
                    .iter()
                    .map(|r| safe_classify(rule_set, &r.record, team_id))
                    .collect();
                let mut pending: Vec<(String, Vec<usize>)> = Vec::new();
                for (i, c) in classifications.iter().enumerate() {
                    if let SamplingClassifyResult::RateLimit { rule_id } = c {
                        match pending.iter_mut().find(|(id, _)| id == rule_id) {
                            Some((_, list)) => list.push(i),
                            None => pending.push((rule_id.clone(), vec![i])),
                        }
                    }
                }
                let rate_keep = self
                    .apply_rate_limits(
                        team_id,
                        rule_set,
                        &pending,
                        &records,
                        header_bytes_uncompressed,
                        batch,
                    )
                    .await?;
                for (i, (staged, c)) in records.into_iter().zip(classifications).enumerate() {
                    match c {
                        SamplingClassifyResult::RateLimit { rule_id } => {
                            if rate_keep[i] == Some(false) {
                                drop(&mut stats, &staged, Some(&rule_id));
                                continue;
                            }
                            kept.push(staged);
                        }
                        SamplingClassifyResult::Resolved { decision, rule_id } => {
                            if decision.drops() {
                                drop(&mut stats, &staged, rule_id.as_deref());
                                continue;
                            }
                            kept.push(staged);
                        }
                    }
                }
            }
            None => {
                for staged in records {
                    let result = safe_evaluate(rule_set, &staged.record, team_id.unwrap_or(0));
                    if result.decision.drops() {
                        drop(&mut stats, &staged, result.rule_id.as_deref());
                        continue;
                    }
                    kept.push(staged);
                }
            }
        }
        Ok(FilterResult { kept, stats })
    }

    /// `None` means no decision, and the record is kept.
    async fn apply_rate_limits(
        &self,
        team_id: i64,
        rule_set: &CompiledRuleSet,
        pending: &[(String, Vec<usize>)],
        records: &[StagedRecord],
        header_bytes_uncompressed: f64,
        batch: &BatchContext,
    ) -> anyhow::Result<Vec<Option<bool>>> {
        let mut keep: Vec<Option<bool>> = vec![None; records.len()];
        if pending.is_empty() {
            return Ok(keep);
        }
        let pro_rata_scale = if header_bytes_uncompressed > 0.0 && batch.content_bytes_total > 0.0 {
            header_bytes_uncompressed / batch.content_bytes_total
        } else {
            0.0
        };
        let byte_cost = |idx: usize| -> f64 {
            if pro_rata_scale > 0.0 {
                batch.content_bytes_of(&records[idx]) * pro_rata_scale
            } else {
                record_bytes(&records[idx].record)
            }
        };
        // `new Map(rules.map(r => [r.id, r]))`: the last rule with an id wins.
        let rule_by_id = |id: &str| rule_set.rules.iter().rev().find(|r| r.id == id);

        let mut entries: Vec<(&[usize], Vec<f64>)> = Vec::new();
        let mut requests: Vec<KeyedRateLimitRequest> = Vec::new();
        for (rule_id, indices) in pending {
            let Some(rl) = rule_by_id(rule_id).and_then(|r| r.rate_limit) else {
                continue;
            };
            if indices.is_empty() {
                continue;
            }
            let costs: Vec<f64> = match rl.cost_unit {
                CostUnit::Bytes => indices.iter().map(|&i| byte_cost(i)).collect(),
                CostUnit::Records => vec![1.0; indices.len()],
            };
            let total_cost = costs.iter().fold(0.0, |a, b| a + b);
            requests.push(KeyedRateLimitRequest {
                id: format!("{team_id}/{rule_id}"),
                cost: total_cost,
                bucket_size: Some(rl.pool_max),
                refill_rate: Some(rl.refill_per_second),
                ttl_seconds: None,
                now: None,
            });
            entries.push((indices, costs));
        }

        if entries.is_empty() {
            for (_, indices) in pending {
                for &i in indices {
                    keep[i] = Some(true);
                }
            }
            return Ok(keep);
        }

        let results = self.rate_limiter.rate_limit_many(&requests).await?;
        for (i, (indices, costs)) in entries.iter().enumerate() {
            let tokens_before = results.get(i).map_or(0.0, |(_, r)| r.tokens_before);
            let budget = tokens_before.floor().max(0.0);
            let mut spent = 0.0;
            for (j, &idx) in indices.iter().enumerate() {
                let next = spent + costs[j];
                let admit = next <= budget;
                keep[idx] = Some(admit);
                if admit {
                    spent = next;
                }
            }
        }
        Ok(keep)
    }

    pub fn make_sampling_stage(
        self: &Arc<Self>,
        rule_set: Arc<CompiledRuleSet>,
        team_id: Option<i64>,
        header_bytes_uncompressed: f64,
    ) -> PipelineStage {
        PipelineStage::Filter(Box::new(SamplingStage {
            service: self.clone(),
            rule_set,
            team_id,
            header_bytes_uncompressed,
        }))
    }
}

struct SamplingStage {
    service: Arc<LogsSamplingService>,
    rule_set: Arc<CompiledRuleSet>,
    team_id: Option<i64>,
    header_bytes_uncompressed: f64,
}

#[async_trait]
impl FilterStage for SamplingStage {
    fn name(&self) -> &str {
        SAMPLING_STAGE_NAME
    }

    async fn run(
        &self,
        records: Vec<StagedRecord>,
        batch: &BatchContext,
    ) -> anyhow::Result<FilterResult> {
        self.service
            .sample_records(
                records,
                &self.rule_set,
                batch,
                self.team_id,
                self.header_bytes_uncompressed,
            )
            .await
    }
}