:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::{collections::HashMap, time::Duration};
use rdkafka::{
client::DefaultClientContext,
consumer::ConsumerGroupMetadata,
error::KafkaError,
producer::{FutureProducer, Producer},
ClientConfig, ClientContext, TopicPartitionList,
};
use serde::Serialize;
use tracing::{debug, error, info};
use crate::{
config::KafkaConfig,
kafka_consumer::Offset,
kafka_producer::{
send_keyed_iter_to_kafka, send_keyed_iter_to_kafka_with_headers, KafkaProduceError,
},
};
/// Startup metadata-ping bound for producers built without a broker
/// transaction bound — the value this call used before the bound
/// existed, kept so unrelated services' startup behavior is unchanged.
const UNBOUNDED_PING_TIMEOUT: Duration = Duration::from_secs(15);
/// The `ClientConfig` for a transactional producer: the shared
/// producer's tuning, the transactional id, and — when the caller
/// controls the broker-side bound — `transaction.timeout.ms`.
///
/// A pure function so what the fenced producer inherits is assertable.
/// It previously carried only a fixed subset of the shared tuning, so a
/// deployment that set `message.max.bytes` or batch tuning got two
/// producers behaving differently depending on which flag arm built
/// them.
///
/// Four knobs are deliberately not copied even when configured, because
/// the transactional contract pins them: transactions require
/// idempotence, and idempotence requires `acks=all`, retries above
/// zero, and at most five in-flight requests. librdkafka refuses to
/// build a producer configured against any of those, so a shared
/// override of `enable.idempotence`, `acks`, `retries`, or
/// `max.in.flight.requests.per.connection` must not reach this one.
fn transactional_client_config(
config: &KafkaConfig,
transactional_id: &str,
broker_txn_timeout: Option<Duration>,
) -> Result<ClientConfig, KafkaError> {
let mut client_config = ClientConfig::new();
client_config
.set("bootstrap.servers", &config.kafka_hosts)
.set("statistics.interval.ms", "10000")
.set("linger.ms", config.kafka_producer_linger_ms.to_string())
.set(
"message.timeout.ms",
config.kafka_message_timeout_ms.to_string(),
)
.set(
"compression.codec",
config.kafka_compression_codec.to_owned(),
)
.set(
"queue.buffering.max.kbytes",
(config.kafka_producer_queue_mib * 1024).to_string(),
)
.set(
"queue.buffering.max.messages",
config.kafka_producer_queue_messages.to_string(),
)
.set("transactional.id", transactional_id);
if !config.kafka_client_id.is_empty() {
client_config.set("client.id", &config.kafka_client_id);
}
if let Some(v) = config.kafka_producer_batch_size {
client_config.set("batch.size", v.to_string());
}
if let Some(v) = config.kafka_producer_batch_num_messages {
client_config.set("batch.num.messages", v.to_string());
}
if let Some(v) = config.kafka_producer_topic_metadata_refresh_interval_ms {
client_config.set("topic.metadata.refresh.interval.ms", v.to_string());
}
if let Some(v) = config.kafka_producer_message_max_bytes {
client_config.set("message.max.bytes", v.to_string());
}
if let Some(v) = config.kafka_producer_sticky_partitioning_linger_ms {
client_config.set("sticky.partitioning.linger.ms", v.to_string());
}
if let Some(ref v) = config.kafka_producer_partitioner {
client_config.set("partitioner", v);
}
if let Some(broker_txn_timeout) = broker_txn_timeout {
let message_timeout_ms = u128::from(config.kafka_message_timeout_ms);
if message_timeout_ms > broker_txn_timeout.as_millis() {
// librdkafka reports this as a bare string from
// `rd_kafka_new`; say which two knobs disagree instead.
return Err(KafkaError::ClientCreation(format!(
"message.timeout.ms ({message_timeout_ms}) exceeds the requested \
transaction.timeout.ms ({}) for transactional id {transactional_id}",
broker_txn_timeout.as_millis(),
)));
}
client_config.set(
"transaction.timeout.ms",
broker_txn_timeout.as_millis().to_string(),
);
}
if config.kafka_tls {
client_config
.set("security.protocol", "ssl")
.set("enable.ssl.certificate.verification", "false");
};
Ok(client_config)
}
// TODO - it's kinda gross to leak the underlying producer context type here, makes for a really gross API. We should
// probably figure out some trait to abstract over it
pub struct TransactionalProducer<C = DefaultClientContext>
where
C: ClientContext + 'static,
{
inner: FutureProducer<C>,
timeout: Duration,
}
/// A transactional producer that is connected but not yet initialized:
/// the client exists with its connections and metadata warm, and no
/// broker-side transactional state has been touched. `init_transactions`
/// is the fencing action — it bumps the transactional id's epoch and
/// cuts off every previous owner — so it belongs to the moment authority
/// is taken, not to construction. Splitting the phases lets a caller pay
/// the connection cost ahead of that moment.
pub struct ConnectedTransactionalProducer<C = DefaultClientContext>
where
C: ClientContext + 'static,
{
inner: FutureProducer<C>,
}
impl ConnectedTransactionalProducer<DefaultClientContext> {
/// Create the client and ping the brokers, bounded by `timeout`,
/// without initializing transactions — a producer whose open
/// transactions the broker abandons after `broker_txn_timeout`,
/// rather than after librdkafka's default minute.
///
/// The broker bound is a different quantity from `timeout`, which
/// bounds how long *this process* waits on a blocking call. It
/// matters to everyone else: until an abandoned transaction expires,
/// the partition's last-stable-offset does not advance and every
/// `read_committed` consumer stalls behind it. Only a caller that
/// also controls `message.timeout.ms` can set it, because librdkafka
/// requires `message.timeout.ms <= transaction.timeout.ms` and
/// refuses to build the producer at all otherwise.
pub fn connect_bounded(
config: &KafkaConfig,
transactional_id: &str,
timeout: Duration,
broker_txn_timeout: Duration,
) -> Result<Self, KafkaError> {
Self::connect_bounded_with_context(
config,
transactional_id,
timeout,
broker_txn_timeout,
DefaultClientContext,
)
}
}
impl<C: ClientContext + 'static> ConnectedTransactionalProducer<C> {
/// [`Self::connect_bounded`] with a caller-supplied client context, for
/// a caller that consumes librdkafka's statistics or logs.
pub fn connect_bounded_with_context(
config: &KafkaConfig,
transactional_id: &str,
timeout: Duration,
broker_txn_timeout: Duration,
context: C,
) -> Result<Self, KafkaError> {
let inner = connect(
config,
transactional_id,
timeout,
Some(broker_txn_timeout),
context,
)?;
Ok(ConnectedTransactionalProducer { inner })
}
}
impl<C: ClientContext> ConnectedTransactionalProducer<C> {
/// Claim the transactional id: one `init_transactions` round trip,
/// which fences every previous owner of the id.
pub fn init(
self,
timeout: Duration,
) -> Result<TransactionalProducer<C>, (KafkaError, ConnectedTransactionalProducer<C>)> {
match self.inner.init_transactions(timeout) {
Ok(()) => Ok(TransactionalProducer {
inner: self.inner,
timeout,
}),
// Hand the connection back with the error: a timed-out init
// does not invalidate the client, and the caller decides
// whether to retry on it or discard it.
Err(e) => Err((e, self)),
}
}
}
/// Create the client and ping the brokers: everything construction does
/// short of `init_transactions`.
fn connect<C: ClientContext>(
config: &KafkaConfig,
transactional_id: &str,
timeout: Duration,
broker_txn_timeout: Option<Duration>,
context: C,
) -> Result<FutureProducer<C>, KafkaError> {
let client_config = transactional_client_config(config, transactional_id, broker_txn_timeout)?;
debug!("rdkafka configuration: {:?}", client_config);
let api: FutureProducer<C> = client_config.create_with_context(context)?;
// "Ping" the Kafka brokers by requesting metadata. On the
// broker-bounded (partition-acquisition) path the ping is bounded
// by the caller's timeout — an unbounded stall there holds a warm
// slot and delays a handoff. Everywhere else the historical fixed
// bound is kept, so services that never opted into broker bounds
// keep their startup behavior.
let ping_timeout = if broker_txn_timeout.is_some() {
timeout
} else {
UNBOUNDED_PING_TIMEOUT
};
match api.client().fetch_metadata(None, ping_timeout) {
Ok(metadata) => {
info!(
"Successfully connected to Kafka brokers. Found {} topics.",
metadata.topics().len()
);
}
Err(error) => {
error!("Failed to fetch metadata from Kafka brokers: {:?}", error);
return Err(error);
}
}
Ok(api)
}
impl TransactionalProducer<DefaultClientContext> {
// Create a transactional producer, with a default context
pub fn from_config(
config: &KafkaConfig,
transactional_id: &str,
timeout: Duration,
) -> Result<Self, KafkaError> {
Self::with_context(config, transactional_id, timeout, DefaultClientContext)
}
}
impl<C: ClientContext> TransactionalProducer<C> {
pub fn with_context(
config: &KafkaConfig,
transactional_id: &str,
timeout: Duration,
context: C,
) -> Result<Self, KafkaError> {
Self::build(config, transactional_id, timeout, None, context)
}
fn build(
config: &KafkaConfig,
transactional_id: &str,
timeout: Duration,
broker_txn_timeout: Option<Duration>,
context: C,
) -> Result<Self, KafkaError> {
let api = connect(
config,
transactional_id,
timeout,
broker_txn_timeout,
context,
)?;
api.init_transactions(timeout)?;
Ok(TransactionalProducer {
inner: api,
timeout,
})
}
pub fn begin(&mut self) -> Result<KafkaTransaction<'_, C>, KafkaError> {
self.inner.begin_transaction()?;
Ok(KafkaTransaction { producer: self })
}
pub fn set_timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
self
}
// Expose the inner at the producer level, but not at the transaction level -
// during a transaction, we want strong control over the operations done, but outside
// of the transaction, we want to be able to do things like fetch metadata
pub fn inner(&self) -> &FutureProducer<C> {
&self.inner
}
}
pub struct KafkaTransaction<'a, C = DefaultClientContext>
where
C: ClientContext + 'static,
{
// NOTE: kafka requires any producer have only a single transaction running at any time. We
// enforce this by having transactions mutably borrow the initiating producer, although this
// is not strictly necessary by the rdkafka interface itself
producer: &'a mut TransactionalProducer<C>,
}
// TODO - most of these are blocking, and we should wrap them in spawn_blocking and expose
// a purely async interface
impl<'a, C: ClientContext> KafkaTransaction<'a, C> {
pub async fn send_keyed_iter_to_kafka<D>(
&self,
topic: &str,
key_extractor: impl Fn(&D) -> Option<String>,
iter: impl IntoIterator<Item = D>,
) -> Vec<Result<(), KafkaProduceError>>
where
D: Serialize,
{
send_keyed_iter_to_kafka(&self.producer.inner, topic, key_extractor, iter).await
}
pub async fn send_keyed_iter_to_kafka_with_headers<D>(
&self,
topic: &str,
key_extractor: impl Fn(&D) -> Option<String>,
headers_extractor: impl Fn(&D) -> Option<rdkafka::message::OwnedHeaders>,
iter: impl IntoIterator<Item = D>,
) -> Vec<Result<(), KafkaProduceError>>
where
D: Serialize,
{
send_keyed_iter_to_kafka_with_headers(
&self.producer.inner,
topic,
key_extractor,
headers_extractor,
iter,
)
.await
}
pub async fn send_iter_to_kafka<D>(
&self,
topic: &str,
iter: impl IntoIterator<Item = D>,
) -> Vec<Result<(), KafkaProduceError>>
where
D: Serialize,
{
send_keyed_iter_to_kafka(&self.producer.inner, topic, |_| None, iter).await
}
pub fn associate_offsets(
&self,
offsets: Vec<Offset>,
metadata: &ConsumerGroupMetadata,
) -> Result<(), KafkaError> {
let tpl = to_topic_partition_list(offsets)?;
self.producer
.inner
.send_offsets_to_transaction(&tpl, metadata, self.producer.timeout)
}
pub fn commit(self) -> Result<(), KafkaError> {
self.producer
.inner
.commit_transaction(self.producer.timeout)?;
Ok(())
}
pub fn abort(self) -> Result<(), KafkaError> {
self.producer
.inner
.abort_transaction(self.producer.timeout)?;
Ok(())
}
}
fn to_topic_partition_list(offsets: Vec<Offset>) -> Result<TopicPartitionList, KafkaError> {
let mut topic_map = HashMap::new();
for offset in offsets.into_iter() {
let key = (offset.topic, offset.partition);
let stored = topic_map.entry(key).or_insert(offset.offset);
if *stored < offset.offset {
*stored = offset.offset
}
}
let topic_map = topic_map
.into_iter()
// Docs say: "The offsets should be the next message your application will consume,
// i.e., one greater than the the last processed message’s offset for each partition."
// Link: https://docs.rs/rdkafka/latest/rdkafka/producer/trait.Producer.html#tymethod.send_offsets_to_transaction.
// Since this is only used for associating offsets with a transaction, we know that each
// offset should be the next message to be consumed, i.e. the high watermark + 1.
.map(|(k, v)| (k, rdkafka::Offset::from_raw(v + 1)))
.collect();
TopicPartitionList::from_topic_map(&topic_map)
}
#[cfg(test)]
mod tests {
use super::*;
/// librdkafka refuses to build a transactional producer whose
/// `message.timeout.ms` exceeds its `transaction.timeout.ms`, and the
/// two are set from different places: the message timeout comes from
/// the shared `KafkaConfig`, the transaction timeout from whoever asks
/// for a bounded producer. A caller that sets only the latter gets a
/// producer that cannot be constructed at all — which is a startup
/// crash loop, not a degraded mode.
///
/// `Duration::from_secs(10)` against the default 20s message timeout
/// is the combination that shape belongs to; the bounded constructor
/// has to reject it by name rather than let librdkafka reject it by
/// string.
#[test]
fn a_broker_bound_under_the_message_timeout_is_refused_by_name() {
let config = KafkaConfig {
kafka_message_timeout_ms: 20_000,
..KafkaConfig::default()
};
let Err(err) = ConnectedTransactionalProducer::connect_bounded(
&config,
"test-txn-id",
Duration::from_secs(10),
Duration::from_secs(10),
) else {
panic!("a broker bound below the message timeout must not build a producer");
};
// Deliberately asserting on the values, not the knob names:
// librdkafka's own rejection string names both knobs too, so a
// name-only assertion would pass just as well with this check
// removed and the failure left to the C library.
let message = err.to_string();
assert!(
message.contains("20000") && message.contains("10000"),
"the error should report the two timeouts that disagree, got: {message}"
);
}
/// The unbounded constructors must not set `transaction.timeout.ms` at
/// all, so a caller whose message timeout exceeds its operation
/// timeout keeps librdkafka's default rather than inheriting a bound
/// it never asked for.
#[test]
fn an_unbounded_producer_does_not_inherit_the_operation_timeout() {
let config = KafkaConfig {
kafka_message_timeout_ms: 20_000,
..KafkaConfig::default()
};
let client_config = transactional_client_config(&config, "test-txn-id", None)
.expect("an unbounded config always builds");
assert_eq!(
client_config.get("transaction.timeout.ms"),
None,
"no broker bound was requested, so none may be set"
);
assert_eq!(client_config.get("message.timeout.ms"), Some("20000"));
}
/// The fenced producer must carry the shared producer's tuning, not a
/// fixed subset of it: a deployment that raises `message.max.bytes`
/// or sets batch or metadata-refresh tuning would otherwise get two
/// producers behaving differently depending on which flag arm built
/// them — most consequentially the message ceiling, since the leader
/// admits person properties within a whisker of librdkafka's default.
#[test]
fn the_fenced_producer_inherits_the_shared_tuning() {
let config = KafkaConfig {
kafka_client_id: "leader-changelog".to_string(),
kafka_producer_batch_size: Some(524_288),
kafka_producer_batch_num_messages: Some(1_000),
kafka_producer_topic_metadata_refresh_interval_ms: Some(30_000),
kafka_producer_message_max_bytes: Some(2_097_152),
kafka_producer_sticky_partitioning_linger_ms: Some(25),
kafka_producer_partitioner: Some("murmur2_random".to_string()),
..KafkaConfig::default()
};
let client_config = transactional_client_config(&config, "test-txn-id", None)
.expect("a compatible config builds");
for (key, value) in [
("client.id", "leader-changelog"),
("batch.size", "524288"),
("batch.num.messages", "1000"),
("topic.metadata.refresh.interval.ms", "30000"),
("message.max.bytes", "2097152"),
("sticky.partitioning.linger.ms", "25"),
("partitioner", "murmur2_random"),
] {
assert_eq!(
client_config.get(key),
Some(value),
"the fenced producer dropped the shared `{key}` tuning"
);
}
}
/// The four knobs the transactional contract pins must not be copied
/// even when the shared config sets them: transactions require
/// idempotence, and idempotence requires acks=all, retries above
/// zero, and at most five in-flight requests — librdkafka refuses to
/// build a producer configured against any of those.
#[test]
fn the_transactional_contract_pins_its_knobs() {
let config = KafkaConfig {
kafka_producer_enable_idempotence: Some(false),
kafka_producer_acks: Some("1".to_string()),
kafka_producer_retries: Some(0),
kafka_producer_max_in_flight_requests_per_connection: Some(50),
..KafkaConfig::default()
};
let client_config = transactional_client_config(&config, "test-txn-id", None)
.expect("pinned knobs are dropped, not refused");
for key in [
"enable.idempotence",
"acks",
"retries",
"max.in.flight.requests.per.connection",
] {
assert_eq!(
client_config.get(key),
None,
"`{key}` is pinned by the transactional contract and must not be copied"
);
}
}
}