:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Routing Processor - Routes messages to partition workers via channels
//!
//! This processor groups incoming messages by partition and routes them
//! to dedicated partition workers, enabling true parallel processing with
//! pipelining between consumption and processing.
use std::collections::HashMap;
use std::sync::Arc;
use anyhow::Result;
use async_trait::async_trait;
use futures::future::join_all;
use tracing::warn;
use crate::kafka::batch_consumer::BatchConsumerProcessor;
use crate::kafka::batch_message::KafkaMessage;
use crate::kafka::offset_tracker::OffsetTracker;
use crate::kafka::partition_router::PartitionRouter;
use crate::kafka::types::Partition;
/// A processor that routes messages to partition-specific workers
///
/// This implements `BatchConsumerProcessor` so it can be used with the existing
/// `BatchConsumer`. Instead of processing messages directly, it groups them by
/// partition and sends them to dedicated workers via channels.
pub struct RoutingProcessor<T, P>
where
T: Send + 'static,
P: BatchConsumerProcessor<T> + 'static,
{
router: Arc<PartitionRouter<T, P>>,
offset_tracker: Arc<OffsetTracker>,
}
impl<T, P> RoutingProcessor<T, P>
where
T: Send + 'static,
P: BatchConsumerProcessor<T> + 'static,
{
pub fn new(router: Arc<PartitionRouter<T, P>>, offset_tracker: Arc<OffsetTracker>) -> Self {
Self {
router,
offset_tracker,
}
}
/// Get a reference to the underlying router
pub fn router(&self) -> &Arc<PartitionRouter<T, P>> {
&self.router
}
/// Get a reference to the offset tracker
pub fn offset_tracker(&self) -> &Arc<OffsetTracker> {
&self.offset_tracker
}
}
#[async_trait]
impl<T, P> BatchConsumerProcessor<T> for RoutingProcessor<T, P>
where
T: Send + Sync + 'static,
P: BatchConsumerProcessor<T> + 'static,
{
async fn process_batch(&self, messages: Vec<KafkaMessage<T>>) -> Result<()> {
if messages.is_empty() {
return Ok(());
}
// Group messages by partition
let mut messages_by_partition: HashMap<Partition, Vec<KafkaMessage<T>>> = HashMap::new();
for message in messages {
let partition = message.get_topic_partition();
messages_by_partition
.entry(partition)
.or_default()
.push(message);
}
// Route all partitions concurrently to avoid head-of-line blocking
// If one partition's channel is full (backpressure), it won't block other partitions
let route_futures: Vec<_> = messages_by_partition
.into_iter()
.map(|(partition, partition_messages)| {
let router = self.router.clone();
// Assign a batch ID for ordering verification
let batch_id = self.offset_tracker.assign_batch_id();
async move {
let result = router
.route_batch(partition.clone(), partition_messages, batch_id)
.await;
(partition, result)
}
})
.collect();
let results = join_all(route_futures).await;
// Log any failures but don't fail the batch - workers may have been removed during rebalance
for (partition, result) in results {
if let Err(e) = result {
warn!(
"Failed to route batch to partition {}:{}: {e:#}",
partition.topic(),
partition.partition_number()
);
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::kafka::partition_router::shutdown_workers;
use crate::kafka::partition_router::PartitionRouterConfig;
use crate::test_utils::create_test_tracker;
use std::sync::atomic::{AtomicUsize, Ordering};
use tokio::time::{sleep, Duration};
struct CountingProcessor {
count: AtomicUsize,
}
impl CountingProcessor {
fn new() -> Self {
Self {
count: AtomicUsize::new(0),
}
}
fn get_count(&self) -> usize {
self.count.load(Ordering::SeqCst)
}
}
#[async_trait]
impl BatchConsumerProcessor<String> for CountingProcessor {
async fn process_batch(&self, messages: Vec<KafkaMessage<String>>) -> Result<()> {
self.count.fetch_add(messages.len(), Ordering::SeqCst);
Ok(())
}
}
#[tokio::test]
async fn test_routing_processor_groups_by_partition() {
let processor = Arc::new(CountingProcessor::new());
let coordinator = create_test_tracker();
let offset_tracker = Arc::new(OffsetTracker::new(coordinator));
let config = PartitionRouterConfig::default();
let router = Arc::new(PartitionRouter::new(
processor.clone(),
offset_tracker.clone(),
config,
));
// Add workers for two partitions
let p0 = Partition::new("test-topic".to_string(), 0);
let p1 = Partition::new("test-topic".to_string(), 1);
router.add_partition(p0.clone());
router.add_partition(p1.clone());
let routing_processor = RoutingProcessor::new(router.clone(), offset_tracker);
// Create messages for different partitions
let messages = vec![
KafkaMessage::new_for_test(p0.clone(), 0, "msg1".to_string()),
KafkaMessage::new_for_test(p0.clone(), 1, "msg2".to_string()),
KafkaMessage::new_for_test(p1.clone(), 0, "msg3".to_string()),
];
// Route the batch
routing_processor.process_batch(messages).await.unwrap();
// Give workers time to process
sleep(Duration::from_millis(50)).await;
// All 3 messages should have been processed
assert_eq!(processor.get_count(), 3);
// Cleanup
let workers = router.shutdown_all();
shutdown_workers(workers).await;
}
#[tokio::test]
async fn test_routing_processor_handles_missing_worker() {
let processor = Arc::new(CountingProcessor::new());
let coordinator = create_test_tracker();
let offset_tracker = Arc::new(OffsetTracker::new(coordinator));
let config = PartitionRouterConfig::default();
let router = Arc::new(PartitionRouter::new(
processor.clone(),
offset_tracker.clone(),
config,
));
// Only add worker for partition 0
let p0 = Partition::new("test-topic".to_string(), 0);
router.add_partition(p0.clone());
let routing_processor = RoutingProcessor::new(router.clone(), offset_tracker);
// Create messages including one for a partition without a worker
let p1 = Partition::new("test-topic".to_string(), 1);
let messages = vec![
KafkaMessage::new_for_test(p0.clone(), 0, "msg1".to_string()),
KafkaMessage::new_for_test(p1, 0, "msg2".to_string()), // No worker for this
];
// Should not fail, just warn
let result = routing_processor.process_batch(messages).await;
assert!(result.is_ok());
// Give worker time to process
sleep(Duration::from_millis(50)).await;
// Only the message for partition 0 should be processed
assert_eq!(processor.get_count(), 1);
// Cleanup
let workers = router.shutdown_all();
shutdown_workers(workers).await;
}
}