Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.9 KB
Raw Blame
//! Shared processor utilities for deduplication pipelines.
//!
//! This module provides common functionality used across different pipeline
//! processors to reduce code duplication.

use std::time::Instant;

use anyhow::{Context, Result};
use tracing::warn;

use crate::metrics::MetricsHelper;
use crate::metrics_const::{
    DEDUPLICATION_RESULT_COUNTER, MESSAGES_DROPPED_NO_STORE, ROCKSDB_MULTI_GET_DURATION_MS,
    ROCKSDB_PUT_BATCH_DURATION_MS,
};
use crate::pipelines::EventSimilarity;
use crate::store::deduplication_store::{DeduplicationStore, TimestampBatchEntry};
use crate::store_manager::{StoreError, StoreManager};

/// Reason why an event was identified as a duplicate or potential duplicate.
#[derive(strum_macros::Display, Debug, Copy, Clone, PartialEq)]
pub enum DuplicateReason {
    /// Exact same event (all fields match including UUID)
    SameEvent,
    /// Same dedup key with same UUID (retry of original)
    SameUuid,
    /// Same dedup key, only UUID differs (confirmed duplicate)
    OnlyUuidDifferent,
    /// Same dedup key but content differs beyond just UUID (potential duplicate)
    ContentDiffers,
}

/// Metadata about a duplicate event detection.
#[derive(Debug)]
pub struct DuplicateInfo<E> {
    /// Why this is considered a duplicate
    pub reason: DuplicateReason,
    /// Similarity between the new event and the original
    pub similarity: EventSimilarity,
    /// The original event that was stored
    pub original_event: E,
    /// Number of unique UUIDs seen for this dedup key (after update)
    pub unique_uuids_count: usize,
}

/// Deduplication result shared across all pipelines.
///
/// This type captures the outcome of deduplication checks:
/// - `New`: First occurrence of this event
/// - `ConfirmedDuplicate`: Definitely a duplicate (same UUID or identical event)
/// - `PotentialDuplicate`: Same dedup key but different content
/// - `Skipped`: Processing was skipped due to an error
#[derive(Debug)]
pub enum DeduplicationResult<E> {
    /// First time seeing this event
    New,
    /// Confirmed duplicate with full info
    ConfirmedDuplicate(DuplicateInfo<E>),
    /// Same dedup key but different content (not confirmed as duplicate)
    PotentialDuplicate(DuplicateInfo<E>),
    /// Processing was skipped (error)
    Skipped,
}

impl<E> DeduplicationResult<E> {
    /// Returns true only for confirmed duplicates.
    /// Potential duplicates are NOT treated as duplicates for filtering purposes.
    pub fn is_duplicate(&self) -> bool {
        matches!(self, DeduplicationResult::ConfirmedDuplicate(_))
    }

    /// Get the similarity info if this is a duplicate.
    pub fn get_similarity(&self) -> Option<&EventSimilarity> {
        match self {
            DeduplicationResult::ConfirmedDuplicate(info) => Some(&info.similarity),
            DeduplicationResult::PotentialDuplicate(info) => Some(&info.similarity),
            _ => None,
        }
    }

    /// Get the original event if this is a duplicate.
    pub fn get_original_event(&self) -> Option<&E> {
        match self {
            DeduplicationResult::ConfirmedDuplicate(info) => Some(&info.original_event),
            DeduplicationResult::PotentialDuplicate(info) => Some(&info.original_event),
            _ => None,
        }
    }

    /// Get the duplicate reason if this is a confirmed duplicate.
    pub fn get_reason(&self) -> Option<DuplicateReason> {
        match self {
            DeduplicationResult::ConfirmedDuplicate(info) => Some(info.reason),
            _ => None,
        }
    }
}

/// Labels for deduplication result metrics.
pub struct DeduplicationResultLabels<'a> {
    pub result_type: &'static str,
    pub reason: Option<&'static str>,
    pub lib: Option<&'a str>,
}

/// Get metric labels for a deduplication result with optional library info.
pub fn get_result_labels<'a, E>(
    result: &DeduplicationResult<E>,
    lib: Option<&'a str>,
) -> DeduplicationResultLabels<'a> {
    let (result_type, reason) = match result {
        DeduplicationResult::New => ("new", None),
        DeduplicationResult::ConfirmedDuplicate(info) => (
            "confirmed_duplicate",
            Some(match info.reason {
                DuplicateReason::SameEvent => "same_event",
                DuplicateReason::SameUuid => "same_uuid",
                DuplicateReason::OnlyUuidDifferent => "only_uuid_different",
                DuplicateReason::ContentDiffers => "content_differs",
            }),
        ),
        DeduplicationResult::PotentialDuplicate(info) => (
            "potential_duplicate",
            Some(match info.reason {
                DuplicateReason::SameEvent => "same_event",
                DuplicateReason::SameUuid => "same_uuid",
                DuplicateReason::OnlyUuidDifferent => "only_uuid_different",
                DuplicateReason::ContentDiffers => "content_differs",
            }),
        ),
        DeduplicationResult::Skipped => ("skipped", None),
    };

    DeduplicationResultLabels {
        result_type,
        reason,
        lib,
    }
}

/// Emit deduplication result metrics.
///
/// This is the shared implementation used by all pipeline processors.
pub fn emit_deduplication_result_metrics(
    topic: &str,
    partition: i32,
    pipeline: &str,
    labels: DeduplicationResultLabels<'_>,
) {
    let mut counter = MetricsHelper::with_partition(topic, partition)
        .with_label("pipeline", pipeline)
        .counter(DEDUPLICATION_RESULT_COUNTER)
        .with_label("result_type", labels.result_type);

    if let Some(reason) = labels.reason {
        counter = counter.with_label("reason", reason);
    }

    if let Some(lib) = labels.lib {
        counter = counter.with_label("lib", lib);
    }

    counter.increment(1);
}

/// Result of attempting to get a store for a partition.
pub enum StoreResult {
    /// Store was found and is ready to use
    Found(DeduplicationStore),
    /// Store was not found (partition likely revoked) - messages should be dropped
    NotFound,
}

/// Get a store for a partition, handling the "not found" case gracefully.
///
/// When a partition is revoked during rebalance, messages may still arrive due to
/// rdkafka buffering. This function handles that case by logging a warning and
/// emitting a metric, returning `StoreResult::NotFound` to indicate messages
/// should be dropped.
pub fn get_store_or_drop(
    store_manager: &StoreManager,
    topic: &str,
    partition: i32,
    message_count: usize,
) -> Result<StoreResult> {
    match store_manager.get_store(topic, partition) {
        Ok(store) => Ok(StoreResult::Found(store)),
        Err(StoreError::NotFound {
            topic: t,
            partition: p,
        }) => {
            warn!(
                topic = %t,
                partition = p,
                message_count = message_count,
                "No store for partition - dropping messages (expected during rebalance)"
            );
            metrics::counter!(
                MESSAGES_DROPPED_NO_STORE,
                "topic" => t.clone(),
                "partition" => p.to_string(),
            )
            .increment(message_count as u64);
            Ok(StoreResult::NotFound)
        }
        Err(StoreError::Other(e)) => Err(e),
    }
}

/// Batch read from RocksDB with metrics.
///
/// Reads multiple keys from the timestamp column family and records the duration.
/// Runs on a blocking thread to avoid starving the tokio async runtime.
pub async fn batch_read_timestamp_records(
    store: &DeduplicationStore,
    keys: Vec<&[u8]>,
) -> Result<Vec<Option<Vec<u8>>>> {
    let store = store.clone();
    let owned_keys: Vec<Vec<u8>> = keys.into_iter().map(|k| k.to_vec()).collect();
    tokio::task::spawn_blocking(move || {
        let key_refs: Vec<&[u8]> = owned_keys.iter().map(|k| k.as_slice()).collect();
        let start = Instant::now();
        let results = store.multi_get_timestamp_records(key_refs)?;
        let duration = start.elapsed();
        metrics::histogram!(ROCKSDB_MULTI_GET_DURATION_MS, "cf" => "timestamp")
            .record(duration.as_millis() as f64);
        Ok(results)
    })
    .await
    .context("RocksDB batch read task panicked")?
}

/// Batch write to RocksDB with metrics.
///
/// Writes multiple key-value pairs to the timestamp column family and records the duration.
/// Runs on a blocking thread to avoid starving the tokio async runtime.
pub async fn batch_write_timestamp_records(
    store: &DeduplicationStore,
    writes: Vec<(Vec<u8>, Vec<u8>)>,
) -> Result<()> {
    if writes.is_empty() {
        return Ok(());
    }

    let store = store.clone();
    tokio::task::spawn_blocking(move || {
        let entries: Vec<TimestampBatchEntry> = writes
            .iter()
            .map(|(key, value)| TimestampBatchEntry {
                key: key.as_slice(),
                value: value.as_slice(),
            })
            .collect();

        let start = Instant::now();
        store.put_timestamp_records_batch(entries)?;
        let duration = start.elapsed();
        metrics::histogram!(ROCKSDB_PUT_BATCH_DURATION_MS, "cf" => "timestamp")
            .record(duration.as_millis() as f64);

        Ok(())
    })
    .await
    .context("RocksDB batch write task panicked")?
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::rocksdb::store::RocksDbConfig;
    use crate::store::DeduplicationStoreConfig;
    use crate::test_utils::create_test_tracker;
    use std::sync::Arc;
    use tempfile::TempDir;

    fn create_test_store_manager() -> (Arc<StoreManager>, TempDir) {
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));
        (manager, temp_dir)
    }

    #[tokio::test]
    async fn test_get_store_or_drop_found() {
        let (manager, _temp_dir) = create_test_store_manager();

        // Create store first
        manager
            .get_or_create_for_rebalance("test-topic", 0)
            .await
            .unwrap();

        let result = get_store_or_drop(&manager, "test-topic", 0, 10).unwrap();
        assert!(matches!(result, StoreResult::Found(_)));
    }

    #[tokio::test]
    async fn test_get_store_or_drop_not_found() {
        let (manager, _temp_dir) = create_test_store_manager();

        // Don't create store - simulates revoked partition
        let result = get_store_or_drop(&manager, "test-topic", 0, 10).unwrap();
        assert!(matches!(result, StoreResult::NotFound));
    }

    #[tokio::test]
    async fn test_batch_read_write_roundtrip() {
        let (manager, _temp_dir) = create_test_store_manager();

        manager
            .get_or_create_for_rebalance("test-topic", 0)
            .await
            .unwrap();

        let store = manager.get_store("test-topic", 0).unwrap();

        // Write some records
        let writes = vec![
            (b"key1".to_vec(), b"value1".to_vec()),
            (b"key2".to_vec(), b"value2".to_vec()),
        ];
        batch_write_timestamp_records(&store, writes).await.unwrap();

        // Read them back
        let keys: Vec<&[u8]> = vec![b"key1", b"key2", b"key3"];
        let results = batch_read_timestamp_records(&store, keys).await.unwrap();

        assert_eq!(results.len(), 3);
        assert_eq!(results[0], Some(b"value1".to_vec()));
        assert_eq!(results[1], Some(b"value2".to_vec()));
        assert_eq!(results[2], None); // key3 doesn't exist
    }

    #[tokio::test]
    async fn test_batch_write_empty() {
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let store = DeduplicationStore::new(config, "test-topic".to_string(), 0).unwrap();

        let result = batch_write_timestamp_records(&store, vec![]).await;
        assert!(result.is_ok());
    }
}