:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
use dashmap::DashMap;
use kafka_assigner_proto::kafka_assigner::v1 as proto;
use kafka_assigner_proto::kafka_assigner::v1::kafka_assigner_server::KafkaAssigner;
use tokio::sync::{mpsc, Mutex};
use tokio_stream::wrappers::ReceiverStream;
use tonic::{Request, Response, Status};
use k8s_awareness::K8sAwareness;
use crate::consumer_registry::{ConsumerConnection, ConsumerRegistry};
use crate::grpc::convert;
use crate::store::KafkaAssignerStore;
use crate::types::{
AssignmentEvent, ConsumerStatus, HandoffPhase, RegisteredConsumer, TopicConfig,
};
/// Kafka connection settings for admin metadata lookups.
#[derive(Clone)]
pub struct KafkaConfig {
pub hosts: String,
pub tls: bool,
pub metadata_timeout: Duration,
}
pub struct KafkaAssignerService {
registry: Arc<ConsumerRegistry>,
store: Arc<KafkaAssignerStore>,
kafka_config: KafkaConfig,
/// Per-topic lock to serialize concurrent `ensure_topic_config` calls,
/// avoiding duplicate Kafka metadata fetches for the same new topic.
topic_init_locks: DashMap<String, Arc<Mutex<()>>>,
stream_channel_size: usize,
consumer_lease_ttl: i64,
consumer_keepalive_interval: Duration,
/// Optional K8s awareness for controller discovery and departure classification.
k8s_awareness: Option<Arc<K8sAwareness>>,
}
impl KafkaAssignerService {
pub fn new(store: Arc<KafkaAssignerStore>, registry: Arc<ConsumerRegistry>) -> Self {
let kafka_config = KafkaConfig {
hosts: "localhost:9092".to_string(),
tls: false,
metadata_timeout: Duration::from_secs(15),
};
Self::with_config(
store,
registry,
kafka_config,
1024,
30,
Duration::from_secs(10),
None,
)
}
pub fn from_config(
store: Arc<KafkaAssignerStore>,
registry: Arc<ConsumerRegistry>,
config: &crate::config::Config,
k8s_awareness: Option<Arc<K8sAwareness>>,
) -> Self {
let kafka_config = KafkaConfig {
hosts: config.kafka_hosts.clone(),
tls: config.kafka_tls,
metadata_timeout: config.kafka_metadata_timeout(),
};
Self::with_config(
store,
registry,
kafka_config,
config.stream_channel_size,
config.consumer_lease_ttl_secs,
config.consumer_keepalive_interval(),
k8s_awareness,
)
}
fn with_config(
store: Arc<KafkaAssignerStore>,
registry: Arc<ConsumerRegistry>,
kafka_config: KafkaConfig,
stream_channel_size: usize,
consumer_lease_ttl: i64,
consumer_keepalive_interval: Duration,
k8s_awareness: Option<Arc<K8sAwareness>>,
) -> Self {
Self {
registry,
store,
kafka_config,
topic_init_locks: DashMap::new(),
stream_channel_size,
consumer_lease_ttl,
consumer_keepalive_interval,
k8s_awareness,
}
}
/// Ensure a `TopicConfig` exists in etcd for the given topic.
///
/// If one already exists, this is a no-op. Otherwise, fetch the partition
/// count from Kafka broker metadata and store the config in etcd.
async fn ensure_topic_config(&self, topic: &str) -> Result<(), Status> {
let store = &self.store;
let kafka_config = &self.kafka_config;
run_once_per_key(
topic,
&self.topic_init_locks,
|| async {
store
.get_topic_config(topic)
.await
.map(|opt| opt.is_some())
.map_err(|e| Status::internal(format!("failed to check topic config: {e}")))
},
|| async {
tracing::info!(topic, "topic config not found in etcd, fetching from Kafka");
let kc = kafka_config.clone();
let topic_owned = topic.to_string();
let partition_count = tokio::task::spawn_blocking(move || {
crate::kafka_admin::fetch_partition_count(
&kc.hosts,
kc.tls,
&topic_owned,
kc.metadata_timeout,
)
})
.await
.map_err(|e| {
tracing::error!(topic, error = %e, "metadata fetch task panicked");
Status::internal(format!("metadata fetch task panicked: {e}"))
})?
.map_err(|e| {
tracing::error!(topic, error = %e, "failed to fetch partition count");
Status::internal("failed to fetch Kafka metadata")
})?;
let config = TopicConfig {
topic: topic.to_string(),
partition_count,
};
store
.set_topic_config(&config)
.await
.map_err(|e| Status::internal(format!("failed to store topic config: {e}")))?;
tracing::info!(topic, partition_count, "stored topic config in etcd");
Ok(())
},
)
.await
}
}
#[tonic::async_trait]
impl KafkaAssigner for KafkaAssignerService {
type RegisterStream = ReceiverStream<Result<proto::AssignmentCommand, Status>>;
async fn register(
&self,
request: Request<proto::RegisterRequest>,
) -> Result<Response<Self::RegisterStream>, Status> {
let req = request.into_inner();
let consumer_name = req.consumer_name;
let topic = req.topic;
assignment_coordination::util::validate_identifier(&consumer_name)
.map_err(|e| Status::invalid_argument(format!("invalid consumer_name: {e}")))?;
if !topic.is_empty() {
assignment_coordination::util::validate_identifier(&topic)
.map_err(|e| Status::invalid_argument(format!("invalid topic: {e}")))?;
self.ensure_topic_config(&topic).await?;
}
// Grant an etcd lease for this consumer's registration.
let lease_id = self
.store
.grant_lease(self.consumer_lease_ttl)
.await
.map_err(|e| Status::internal(format!("failed to grant lease: {e}")))?;
// Register the consumer in etcd (keyed to the lease).
let now = assignment_coordination::util::now_seconds();
// If K8s awareness is enabled, discover the consumer's controller
// and generation hash. This is a one-time lookup per consumer.
let (generation, controller) = if let Some(k8s) = &self.k8s_awareness {
match k8s.discover_controller(&consumer_name).await {
Ok(pod_info) => {
tracing::info!(
consumer = %consumer_name,
controller = %pod_info.controller,
generation = %pod_info.generation,
"discovered K8s controller for consumer"
);
(pod_info.generation, Some(pod_info.controller))
}
Err(e) => {
tracing::warn!(
consumer = %consumer_name,
error = %e,
"failed to discover K8s controller, proceeding without"
);
(String::new(), None)
}
}
} else {
(String::new(), None)
};
let consumer = RegisteredConsumer {
consumer_name: consumer_name.clone(),
status: ConsumerStatus::Ready,
registered_at: now,
generation,
controller,
};
self.store
.register_consumer(&consumer, lease_id)
.await
.map_err(|e| Status::internal(format!("failed to register consumer: {e}")))?;
// Create the channel that bridges the registry to the gRPC stream.
let (event_tx, event_rx) = mpsc::channel::<AssignmentEvent>(self.stream_channel_size);
// Always send an Assignment snapshot as the first message to satisfy
// the stream contract (service.proto). Consumers rely on receiving a
// snapshot before any Warm/Release commands.
let assignments = self
.store
.list_assignments()
.await
.map_err(|e| Status::internal(format!("failed to list assignments: {e}")))?;
let owned = convert::consumer_partitions(&consumer_name, &assignments);
let initial = AssignmentEvent::Assignment {
assigned: owned,
unassigned: vec![],
};
event_tx
.send(initial)
.await
.map_err(|_| Status::internal("failed to send initial assignment"))?;
// Register in the local consumer registry BEFORE replaying
// handoffs. This ensures there is no window where a live handoff
// PUT from the relay is dropped (no sender) AND is also absent
// from the replay snapshot. Duplicates are safe since warm/release
// handling is idempotent.
self.registry.register(ConsumerConnection {
consumer_name: consumer_name.clone(),
command_tx: event_tx.clone(),
lease_id,
});
// Replay pending handoffs that involve this consumer.
// The relay only forwards etcd watch events, so if a handoff was
// created or advanced while this consumer was disconnected (e.g.
// during a rolling restart), the command was never delivered.
let handoffs = self
.store
.list_handoffs()
.await
.map_err(|e| Status::internal(format!("failed to list handoffs: {e}")))?;
let pending_warms: Vec<_> = handoffs
.iter()
.filter(|h| h.phase == HandoffPhase::Warming && h.new_owner == consumer_name)
.cloned()
.collect();
let pending_releases: Vec<_> = handoffs
.iter()
.filter(|h| h.phase == HandoffPhase::Complete && h.old_owner == consumer_name)
.cloned()
.collect();
if !pending_warms.is_empty() {
tracing::info!(
consumer = %consumer_name,
count = pending_warms.len(),
"replaying pending Warming handoffs on reconnect"
);
event_tx
.send(AssignmentEvent::Warm(pending_warms))
.await
.map_err(|_| Status::internal("failed to send pending warms"))?;
}
if !pending_releases.is_empty() {
tracing::info!(
consumer = %consumer_name,
count = pending_releases.len(),
"replaying pending Complete handoffs (releases) on reconnect"
);
event_tx
.send(AssignmentEvent::Release(pending_releases))
.await
.map_err(|_| Status::internal("failed to send pending releases"))?;
}
tracing::info!(consumer = %consumer_name, "consumer registered via gRPC");
// Spawn a background task for lease keepalive and cleanup on disconnect.
let store = Arc::clone(&self.store);
let registry = Arc::clone(&self.registry);
let name = consumer_name.clone();
let keepalive_interval = self.consumer_keepalive_interval;
// Create a proto stream that converts AssignmentEvent -> proto::AssignmentCommand.
let (proto_tx, proto_rx) =
mpsc::channel::<Result<proto::AssignmentCommand, Status>>(self.stream_channel_size);
// Bridge: read domain events, convert to proto, forward to the gRPC stream.
// When the event channel closes (consumer unregistered), this task exits,
// which drops proto_tx, closing the gRPC stream.
tokio::spawn({
let proto_tx = proto_tx.clone();
async move {
let mut event_rx = event_rx;
while let Some(event) = event_rx.recv().await {
let mut failed = false;
for cmd in convert::to_proto_commands(&event) {
if proto_tx.send(Ok(cmd)).await.is_err() {
failed = true;
break;
}
}
if failed {
break;
}
}
}
});
// Keepalive + cleanup task. Keeps the etcd lease alive while the gRPC stream
// is open. When the stream drops (proto_tx closed from consumer side), this
// task detects it via the closed channel and cleans up.
tokio::spawn(async move {
let result =
run_consumer_keepalive(&store, lease_id, keepalive_interval, &proto_tx).await;
if let Err(e) = &result {
tracing::warn!(consumer = %name, error = %e, "consumer keepalive ended");
// Notify the consumer that the stream is being closed so it
// knows to reconnect rather than waiting for a timeout.
drop(
proto_tx
.send(Err(Status::unavailable(format!("keepalive lost: {e}"))))
.await,
);
}
// Unregister from the local in-memory registry so the server stops
// trying to send commands on the dead channel.
//
// We intentionally do NOT revoke the etcd lease here. Letting it
// expire naturally (after consumer_lease_ttl) gives the consumer a
// grace period to reconnect — e.g. during a rolling deployment —
// without triggering a rebalance. If the consumer reconnects before
// the TTL elapses, it overwrites the key with a fresh lease and no
// partition movement occurs. If it doesn't come back, the lease
// expires and the assigner reassigns its partitions.
registry.unregister(&name);
tracing::info!(consumer = %name, lease_id, "consumer disconnected, lease will expire via TTL");
});
Ok(Response::new(ReceiverStream::new(proto_rx)))
}
async fn partition_ready(
&self,
request: Request<proto::PartitionReadyRequest>,
) -> Result<Response<proto::PartitionReadyResponse>, Status> {
let req = request.into_inner();
let tp = convert::topic_partition_from_ready(&req)
.ok_or_else(|| Status::invalid_argument("partition is required"))?;
assignment_coordination::util::validate_identifier(&req.consumer_name)
.map_err(|e| Status::invalid_argument(format!("invalid consumer_name: {e}")))?;
assignment_coordination::util::validate_identifier(&tp.topic)
.map_err(|e| Status::invalid_argument(format!("invalid topic: {e}")))?;
// Write handoff phase to Ready in etcd. The leader's watch_handoffs_loop
// will see this and call complete_handoff.
let mut handoff = self
.store
.get_handoff(&tp)
.await
.map_err(|e| Status::internal(format!("failed to get handoff: {e}")))?
.ok_or_else(|| {
Status::not_found(format!("no handoff for {}/{}", tp.topic, tp.partition))
})?;
handoff.phase = crate::types::HandoffPhase::Ready;
self.store
.put_handoff(&handoff)
.await
.map_err(|e| Status::internal(format!("failed to update handoff: {e}")))?;
tracing::info!(
consumer = %req.consumer_name,
topic = %tp.topic,
partition = tp.partition,
"partition ready signaled via gRPC"
);
Ok(Response::new(proto::PartitionReadyResponse {}))
}
async fn partition_released(
&self,
request: Request<proto::PartitionReleasedRequest>,
) -> Result<Response<proto::PartitionReleasedResponse>, Status> {
let req = request.into_inner();
let tp = convert::topic_partition_from_released(&req)
.ok_or_else(|| Status::invalid_argument("partition is required"))?;
assignment_coordination::util::validate_identifier(&req.consumer_name)
.map_err(|e| Status::invalid_argument(format!("invalid consumer_name: {e}")))?;
assignment_coordination::util::validate_identifier(&tp.topic)
.map_err(|e| Status::invalid_argument(format!("invalid topic: {e}")))?;
// Delete the handoff from etcd. This signals to the leader that the
// old consumer has released and the handoff is fully complete.
self.store
.delete_handoff(&tp)
.await
.map_err(|e| Status::internal(format!("failed to delete handoff: {e}")))?;
tracing::info!(
consumer = %req.consumer_name,
topic = %tp.topic,
partition = tp.partition,
"partition released signaled via gRPC"
);
Ok(Response::new(proto::PartitionReleasedResponse {}))
}
async fn deregister(
&self,
request: Request<proto::DeregisterRequest>,
) -> Result<Response<proto::DeregisterResponse>, Status> {
let req = request.into_inner();
let consumer_name = &req.consumer_name;
assignment_coordination::util::validate_identifier(consumer_name)
.map_err(|e| Status::invalid_argument(format!("invalid consumer_name: {e}")))?;
// Look up the consumer's lease ID from the local registry.
let lease_id = self
.registry
.get_lease_id(consumer_name)
.ok_or_else(|| Status::not_found(format!("consumer {consumer_name} not connected")))?;
// Transition the consumer to Draining in etcd. The assigner's
// consumer watch loop will see this change and exclude the consumer
// from new partition assignments. On the next rebalance (triggered
// when all in-flight handoffs complete), the consumer's partitions
// will be handed off to other Ready consumers.
self.store
.update_consumer_status(consumer_name, ConsumerStatus::Draining, lease_id)
.await
.map_err(|e| {
Status::internal(format!("failed to update consumer status to Draining: {e}"))
})?;
// Determine the appropriate action based on K8s awareness.
let action = if let Some(k8s) = &self.k8s_awareness {
// Look up this consumer's controller and generation from etcd.
let consumer = self
.store
.get_consumer(consumer_name)
.await
.map_err(|e| Status::internal(format!("failed to get consumer: {e}")))?
.ok_or_else(|| {
Status::not_found(format!("consumer {consumer_name} not found in etcd"))
})?;
match &consumer.controller {
Some(controller) => {
let reason = k8s
.classify_departure(controller, &consumer.generation)
.await;
tracing::info!(
consumer = %consumer_name,
controller = %controller,
generation = %consumer.generation,
reason = %reason,
"classified departure reason via K8s awareness"
);
match reason {
// StatefulSet rollout: same pod name comes back, shut down immediately
k8s_awareness::DepartureReason::Rollout
if controller.kind
== k8s_awareness::types::ControllerKind::StatefulSet =>
{
proto::DeregisterAction::ShutdownNow
}
// Deployment rollout or downscale: drain partitions first
k8s_awareness::DepartureReason::Rollout
| k8s_awareness::DepartureReason::Downscale => {
proto::DeregisterAction::WaitForDrain
}
// Crash or unknown: shut down immediately
_ => proto::DeregisterAction::ShutdownNow,
}
}
// No controller info: fall back to default
None => proto::DeregisterAction::ShutdownNow,
}
} else {
proto::DeregisterAction::ShutdownNow
};
tracing::info!(
consumer = %consumer_name,
action = ?action,
"consumer deregistered via gRPC, status set to Draining"
);
Ok(Response::new(proto::DeregisterResponse {
action: action.into(),
}))
}
}
/// Double-checked locking: run `init_fn` at most once per key.
///
/// `is_initialized` is called first without a lock (fast path). If it returns
/// false, a per-key mutex is acquired and `is_initialized` is re-checked. Only
/// if still false, `init_fn` runs.
async fn run_once_per_key<C, CF, I, IF, E>(
key: &str,
locks: &DashMap<String, Arc<Mutex<()>>>,
is_initialized: C,
init_fn: I,
) -> Result<(), E>
where
C: Fn() -> CF,
CF: Future<Output = Result<bool, E>>,
I: FnOnce() -> IF,
IF: Future<Output = Result<(), E>>,
{
if is_initialized().await? {
return Ok(());
}
let lock = locks
.entry(key.to_string())
.or_insert_with(|| Arc::new(Mutex::new(())))
.clone();
let _guard = lock.lock().await;
if is_initialized().await? {
return Ok(());
}
init_fn().await
}
/// Keep the etcd lease alive while the gRPC stream is still open.
///
/// The `proto_tx` sender is used to detect when the consumer disconnects:
/// if the consumer drops the stream, the receiver side closes, and
/// `proto_tx.is_closed()` becomes true.
async fn run_consumer_keepalive(
store: &KafkaAssignerStore,
lease_id: i64,
interval: Duration,
proto_tx: &mpsc::Sender<Result<proto::AssignmentCommand, Status>>,
) -> crate::error::Result<()> {
let (mut keeper, mut stream) = store.keep_alive(lease_id).await?;
loop {
tokio::select! {
_ = proto_tx.closed() => {
// Consumer disconnected — the gRPC stream was dropped.
return Ok(());
}
_ = tokio::time::sleep(interval) => {
keeper.keep_alive().await?;
if stream.message().await?.is_none() {
return Err(crate::error::Error::leadership_lost());
}
}
}
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use super::*;
#[tokio::test]
async fn run_once_skips_init_when_already_initialized() {
let locks = DashMap::new();
let init_count = AtomicU32::new(0);
run_once_per_key(
"topic-a",
&locks,
|| async { Ok::<_, String>(true) },
|| async {
init_count.fetch_add(1, Ordering::SeqCst);
Ok(())
},
)
.await
.unwrap();
assert_eq!(init_count.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn run_once_calls_init_when_not_initialized() {
let locks = DashMap::new();
let init_count = AtomicU32::new(0);
run_once_per_key(
"topic-a",
&locks,
|| async { Ok::<_, String>(false) },
|| async {
init_count.fetch_add(1, Ordering::SeqCst);
Ok(())
},
)
.await
.unwrap();
assert_eq!(init_count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn run_once_concurrent_callers_only_init_once() {
let locks = Arc::new(DashMap::new());
let init_count = Arc::new(AtomicU32::new(0));
let initialized = Arc::new(AtomicBool::new(false));
let mut handles = Vec::new();
for _ in 0..10 {
let locks = Arc::clone(&locks);
let count = Arc::clone(&init_count);
let flag = Arc::clone(&initialized);
handles.push(tokio::spawn(async move {
run_once_per_key(
"topic-a",
&locks,
|| {
let flag = Arc::clone(&flag);
async move { Ok::<_, String>(flag.load(Ordering::SeqCst)) }
},
|| {
let count = Arc::clone(&count);
let flag = Arc::clone(&flag);
async move {
count.fetch_add(1, Ordering::SeqCst);
// Simulate slow work so other callers queue up on the lock.
tokio::time::sleep(Duration::from_millis(50)).await;
flag.store(true, Ordering::SeqCst);
Ok(())
}
},
)
.await
}));
}
for handle in handles {
handle.await.unwrap().unwrap();
}
assert_eq!(init_count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn run_once_different_keys_init_independently() {
let locks = Arc::new(DashMap::new());
let count_a = Arc::new(AtomicU32::new(0));
let count_b = Arc::new(AtomicU32::new(0));
let locks_a = Arc::clone(&locks);
let ca = Arc::clone(&count_a);
let handle_a = tokio::spawn(async move {
run_once_per_key(
"topic-a",
&locks_a,
|| async { Ok::<_, String>(false) },
|| async {
ca.fetch_add(1, Ordering::SeqCst);
Ok(())
},
)
.await
});
let locks_b = Arc::clone(&locks);
let cb = Arc::clone(&count_b);
let handle_b = tokio::spawn(async move {
run_once_per_key(
"topic-b",
&locks_b,
|| async { Ok::<_, String>(false) },
|| async {
cb.fetch_add(1, Ordering::SeqCst);
Ok(())
},
)
.await
});
handle_a.await.unwrap().unwrap();
handle_b.await.unwrap().unwrap();
assert_eq!(count_a.load(Ordering::SeqCst), 1);
assert_eq!(count_b.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn run_once_propagates_check_error() {
let locks = DashMap::new();
let result = run_once_per_key(
"topic-a",
&locks,
|| async { Err::<bool, _>("check failed") },
|| async { Ok(()) },
)
.await;
assert_eq!(result.unwrap_err(), "check failed");
}
#[tokio::test]
async fn run_once_propagates_init_error() {
let locks = DashMap::new();
let result = run_once_per_key(
"topic-a",
&locks,
|| async { Ok::<_, String>(false) },
|| async { Err("init failed".to_string()) },
)
.await;
assert_eq!(result.unwrap_err(), "init failed");
}
}