:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use crate::kafka::types::Partition;
use anyhow::{anyhow, Context, Result};
use rdkafka::message::{BorrowedMessage, Message, OwnedHeaders};
use serde::Deserialize;
pub struct Batch<T> {
messages: Vec<KafkaMessage<T>>,
errors: Vec<BatchError>,
}
impl<T> Batch<T> {
pub fn new() -> Self {
Self {
messages: vec![],
errors: vec![],
}
}
pub fn new_with_size_hint(hint: usize) -> Self {
Self {
messages: Vec::with_capacity(hint),
errors: vec![],
}
}
pub fn message_count(&self) -> usize {
self.messages.len()
}
pub fn error_count(&self) -> usize {
self.errors.len()
}
pub fn is_empty(&self) -> bool {
self.messages.is_empty() && self.errors.is_empty()
}
// consume this Batch and return it's contents
pub fn unpack(self) -> (Vec<KafkaMessage<T>>, Vec<BatchError>) {
(self.messages, self.errors)
}
pub fn get_messages(&self) -> &Vec<KafkaMessage<T>> {
&self.messages
}
pub fn push_message(&mut self, km: KafkaMessage<T>) {
self.messages.push(km);
}
pub fn get_errors(&self) -> &Vec<BatchError> {
&self.errors
}
pub fn push_error(&mut self, be: BatchError) {
self.errors.push(be);
}
}
impl<T> Default for Batch<T> {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug)]
pub struct BatchError {
// core error we're returning. These aren't
// global consumer errors but per-message serialization
// or receive problems that might be associated with a
// specific message
error: anyhow::Error,
// coordinates of the original message if available,
// in case we want to commit it's offset anyway
topic_partition: Option<Partition>,
offset: Option<i64>,
}
impl BatchError {
pub fn new(
error: anyhow::Error,
topic_partition: Option<Partition>,
offset: Option<i64>,
) -> Self {
Self {
error,
topic_partition,
offset,
}
}
}
impl BatchError {
pub fn get_error(&self) -> &anyhow::Error {
&self.error
}
pub fn get_offset(&self) -> Option<i64> {
self.offset
}
pub fn get_topic_partition(&self) -> Option<&Partition> {
self.topic_partition.as_ref()
}
}
/// A lightweight, owned representation of a Kafka message optimized for batch processing
#[derive(Debug, Clone)]
pub struct KafkaMessage<T> {
/// Topic name
topic_partition: Partition,
/// Message offset
offset: i64,
/// Optional message key as raw bytes
pub key: Option<Vec<u8>>,
/// hydrated message, if successfully deserialized from payload
pub message: Option<T>,
/// Message timestamp
pub timestamp: SystemTime,
/// Original headers from inbound message (for republishing)
pub original_headers: Option<OwnedHeaders>,
pub original_payload: Option<Vec<u8>>,
}
impl<T> KafkaMessage<T> {
pub fn new(
topic_partition: Partition,
offset: i64,
key: Option<Vec<u8>>,
message: Option<T>,
timestamp: SystemTime,
original_headers: Option<OwnedHeaders>,
original_payload: Option<Vec<u8>>,
) -> Self {
Self {
topic_partition,
offset,
key,
message,
timestamp,
original_headers,
original_payload,
}
}
/// Extract message from borrowed Kafka message efficiently
pub fn from_borrowed_message<'a>(msg: &'a BorrowedMessage<'a>) -> Result<Self>
where
T: for<'de> Deserialize<'de>,
{
// convert from rdkafka's timestamp to common Rust form
let resolved_timestamp = msg
.timestamp()
.to_millis()
.map(|ms| Duration::from_millis(ms as u64))
.map(|duration| UNIX_EPOCH + duration)
.ok_or_else(|| anyhow!("Failed to resolve timestamp"))?;
let mut out = Self {
topic_partition: Partition::new(msg.topic().to_owned(), msg.partition()),
offset: msg.offset(),
key: msg.key().map(|k| k.to_vec()),
message: None,
timestamp: resolved_timestamp,
original_headers: Some(msg.headers().map(|h| h.detach()).unwrap_or_default()),
original_payload: msg.payload().map(|p| p.to_vec()),
};
match &out.original_payload {
Some(payload) => {
out.message = Some(
serde_json::from_slice::<T>(payload)
.with_context(|| "Failed to deserialize message")?,
);
}
None => {
return Err(anyhow!("No payload in message"));
}
}
Ok(out)
}
// Convenience method for republishing the original event to downstream topic
pub fn to_original_contents(&self) -> (&[u8], Option<&OwnedHeaders>) {
(
self.original_payload.as_deref().unwrap_or(&[]),
self.original_headers.as_ref(),
)
}
// Convenience method to get the Partition associated with the message
pub fn get_topic_partition(&self) -> Partition {
self.topic_partition.clone()
}
/// Get the message key as a UTF-8 string if possible
pub fn key_as_str(&self) -> Option<Result<&str, std::str::Utf8Error>> {
self.key.as_ref().map(|k| std::str::from_utf8(k))
}
pub fn get_offset(&self) -> i64 {
self.offset
}
pub fn get_timestamp(&self) -> std::time::SystemTime {
self.timestamp
}
pub fn get_message(&self) -> Option<&T> {
self.message.as_ref()
}
pub fn take_message(&mut self) -> Option<T> {
self.message.take()
}
/// Create a simple KafkaMessage for testing purposes
#[cfg(test)]
pub fn new_for_test(partition: Partition, offset: i64, message: T) -> Self {
Self {
topic_partition: partition,
offset,
key: None,
message: Some(message),
timestamp: SystemTime::now(),
original_headers: None,
original_payload: None,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use common_types::CapturedEvent;
use time::OffsetDateTime;
use uuid::Uuid;
use rdkafka::message::{Header, Headers};
#[tokio::test]
async fn test_kafka_message_utilities() {
let now = std::time::SystemTime::now();
let now_offset_datetime = OffsetDateTime::from(now);
let now_rfc3339 = chrono::DateTime::<chrono::Utc>::from(now).to_rfc3339();
let distinct_id = Uuid::now_v7().to_string();
let token = Uuid::now_v7().to_string();
let event_name = "$pageview";
let ip = "127.0.0.1".to_string();
let event_uuid = Uuid::now_v7();
let topic = "test-topic";
let topic_partition = Partition::new(topic.to_string(), 0);
let offset = 123;
let key_str = format!("{token}:{distinct_id}");
let key = key_str.as_bytes().to_vec();
let payload = format!(
r#"{{"uuid": "{event_uuid}", "event": "{event_name}", "distinct_id": "{distinct_id}", "token": "{token}","properties": {{}}}}"#
);
let headers = OwnedHeaders::new()
.insert(Header {
key: "header1",
value: Some(b"value1"),
})
.insert(Header {
key: "header2",
value: Some(b""),
})
.insert(Header {
key: "some_number",
value: Some(b"123456789"),
})
.insert(Header {
key: "another_number",
value: Some(b"123"),
});
// Create a mock KafkaMessage for testing
let message = KafkaMessage::<CapturedEvent> {
topic_partition,
offset,
key: Some(key.clone()),
message: Some(CapturedEvent {
uuid: event_uuid,
distinct_id: distinct_id.to_string(),
session_id: None,
ip: ip.clone(),
now: now_rfc3339.clone(),
token: token.clone(),
// serialized RawEvent
data: payload.to_string(),
sent_at: Some(now_offset_datetime),
event: "test_event".to_string(),
timestamp: chrono::Utc::now(),
is_cookieless_mode: false,
historical_migration: false,
}),
timestamp: std::time::SystemTime::now(),
original_headers: Some(headers), // stub orig headers for testing
original_payload: Some(Vec::new()), // stub orig payload for testing
};
// Test string conversion methods
assert_eq!(message.key, Some(key.clone()));
assert_eq!(message.key_as_str().unwrap().unwrap(), &key_str);
// Test header access
assert!(message.original_headers.is_some());
let headers = message.original_headers.as_ref().unwrap();
assert!(headers.get(0).key == "header1");
assert!(headers.get(1).key == "header2");
assert!(headers.get(2).key == "some_number");
assert!(headers.get(3).key == "another_number");
// Test message access
assert!(message.get_message().is_some());
assert_eq!(message.get_message().unwrap().distinct_id, distinct_id);
assert_eq!(message.get_message().unwrap().token, token);
assert_eq!(message.get_message().unwrap().ip, ip);
assert_eq!(message.get_message().unwrap().now, now_rfc3339);
assert_eq!(
message.get_message().unwrap().sent_at,
Some(now_offset_datetime)
);
assert!(!message.get_message().unwrap().is_cookieless_mode);
assert_eq!(&message.get_message().unwrap().key(), &key_str);
assert_eq!(message.get_message().unwrap().uuid, event_uuid);
assert_eq!(message.get_message().unwrap().distinct_id, distinct_id);
assert_eq!(message.get_message().unwrap().token, token);
assert_eq!(message.get_message().unwrap().ip, ip);
assert_eq!(message.get_message().unwrap().now, now_rfc3339);
}
}