Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.7 KB
Raw Blame
use k8s_awareness::types::ControllerRef;
use serde::{Deserialize, Serialize};

/// A Kafka topic-partition pair — the fundamental unit of work assignment.
///
/// In Kafka, a partition is only meaningful in the context of a topic.
/// This type is used throughout the assigner to identify what's being assigned,
/// handed off, or watched.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct TopicPartition {
    pub topic: String,
    pub partition: u32,
}

/// Configuration for a topic being coordinated within a consumer group.
///
/// Stored under `{prefix}config/topics/{topic}`. The coordinator uses this
/// to know how many partitions to distribute across consumers.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TopicConfig {
    pub topic: String,
    pub partition_count: u32,
}

/// A consumer registered in etcd under `{prefix}consumers/{consumer_name}`.
///
/// Each consumer creates this on startup with an etcd lease attached.
/// When the lease expires (consumer crashes), the key is automatically deleted,
/// triggering the coordinator to reassign that consumer's partitions.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RegisteredConsumer {
    pub consumer_name: String,
    pub status: ConsumerStatus,
    pub registered_at: i64,
    /// Pod-template-hash (Deployment) or controller-revision-hash (StatefulSet).
    /// Populated server-side via K8s discovery on registration. Empty when
    /// K8s awareness is disabled.
    #[serde(default)]
    pub generation: String,
    /// The K8s controller that owns this consumer's pod.
    /// Populated server-side via K8s discovery on registration.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub controller: Option<ControllerRef>,
}

/// Lifecycle status of a consumer.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum ConsumerStatus {
    /// Consumer is registered and eligible for partition assignments.
    Ready,
    /// Consumer is shutting down gracefully; excluded from new partition assignments.
    Draining,
}

/// Ownership of a single topic-partition, stored under
/// `{prefix}assignments/{topic}/{partition}`.
///
/// The consumer group is implicit in the prefix:
/// `prefix = /kafka-assigner/{consumer_group}/`
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PartitionAssignment {
    pub topic: String,
    pub partition: u32,
    /// The `consumer_name` that currently owns this topic-partition.
    pub owner: String,
    pub status: AssignmentStatus,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum AssignmentStatus {
    Active,
}

/// Tracks the progress of moving a topic-partition from one consumer to another.
/// Stored under `{prefix}handoffs/{topic}/{partition}`.
///
/// The coordinator creates the handoff (`Warming`), the new consumer advances it
/// to `Ready` after warming up, then the coordinator transitions to `Complete`
/// and updates the assignment atomically.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HandoffState {
    pub topic: String,
    pub partition: u32,
    pub old_owner: String,
    pub new_owner: String,
    pub phase: HandoffPhase,
    pub started_at: i64,
}

/// State machine for partition handoffs:
///
/// ```text
/// Warming → Ready → Complete → (deleted)
///   ^new consumer   ^coordinator    ^old consumer
///    warms           completes       releases
/// ```
///
/// Unlike personhog-coordination, there is no router ack step.
/// The coordinator transitions directly from `Ready` to `Complete`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum HandoffPhase {
    /// New owner is warming up (e.g., downloading checkpoints, replaying logs).
    Warming,
    /// New owner finished warming. Coordinator will complete the handoff.
    Ready,
    /// Handoff complete. Assignment updated. Old owner should release resources.
    Complete,
}

/// A command the assigner sends to a consumer via gRPC.
///
/// This is the domain-level representation. It gets converted to one or more
/// proto `AssignmentCommand` messages at the gRPC boundary.
///
/// `Warm` and `Release` carry batched handoffs per-consumer so the relay can
/// send a single event per consumer per watch response, avoiding channel
/// overflow when many partitions are handed off simultaneously.
#[derive(Debug, Clone)]
pub enum AssignmentEvent {
    /// Batch assignment update: partitions added and/or removed.
    /// Also used for the initial snapshot on connect (assigned = current, unassigned = []).
    Assignment {
        assigned: Vec<TopicPartition>,
        unassigned: Vec<TopicPartition>,
    },
    /// Start warming one or more partitions for handoff.
    Warm(Vec<HandoffState>),
    /// Release one or more partitions after handoff completion.
    Release(Vec<HandoffState>),
}

impl PartitionAssignment {
    pub fn topic_partition(&self) -> TopicPartition {
        TopicPartition {
            topic: self.topic.clone(),
            partition: self.partition,
        }
    }
}

impl HandoffState {
    pub fn topic_partition(&self) -> TopicPartition {
        TopicPartition {
            topic: self.topic.clone(),
            partition: self.partition,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn topic_partition_roundtrip() {
        let tp = TopicPartition {
            topic: "events_json".to_string(),
            partition: 42,
        };
        let json = serde_json::to_string(&tp).unwrap();
        let deserialized: TopicPartition = serde_json::from_str(&json).unwrap();
        assert_eq!(tp, deserialized);
    }

    #[test]
    fn topic_config_roundtrip() {
        let config = TopicConfig {
            topic: "events_json".to_string(),
            partition_count: 1024,
        };
        let json = serde_json::to_string(&config).unwrap();
        let deserialized: TopicConfig = serde_json::from_str(&json).unwrap();
        assert_eq!(config, deserialized);
    }

    #[test]
    fn registered_consumer_roundtrip() {
        let consumer = RegisteredConsumer {
            consumer_name: "consumer-0".to_string(),
            status: ConsumerStatus::Ready,
            registered_at: 1700000000,
            generation: String::new(),
            controller: None,
        };
        let json = serde_json::to_string(&consumer).unwrap();
        let deserialized: RegisteredConsumer = serde_json::from_str(&json).unwrap();
        assert_eq!(consumer, deserialized);
    }

    #[test]
    fn partition_assignment_roundtrip() {
        let assignment = PartitionAssignment {
            topic: "events_json".to_string(),
            partition: 42,
            owner: "consumer-1".to_string(),
            status: AssignmentStatus::Active,
        };
        let json = serde_json::to_string(&assignment).unwrap();
        let deserialized: PartitionAssignment = serde_json::from_str(&json).unwrap();
        assert_eq!(assignment, deserialized);
    }

    #[test]
    fn partition_assignment_topic_partition() {
        let assignment = PartitionAssignment {
            topic: "events_json".to_string(),
            partition: 42,
            owner: "consumer-1".to_string(),
            status: AssignmentStatus::Active,
        };
        assert_eq!(
            assignment.topic_partition(),
            TopicPartition {
                topic: "events_json".to_string(),
                partition: 42,
            }
        );
    }

    #[test]
    fn handoff_state_roundtrip() {
        let handoff = HandoffState {
            topic: "events_json".to_string(),
            partition: 42,
            old_owner: "consumer-1".to_string(),
            new_owner: "consumer-3".to_string(),
            phase: HandoffPhase::Warming,
            started_at: 1700000000,
        };
        let json = serde_json::to_string(&handoff).unwrap();
        let deserialized: HandoffState = serde_json::from_str(&json).unwrap();
        assert_eq!(handoff, deserialized);
    }

    #[test]
    fn handoff_state_topic_partition() {
        let handoff = HandoffState {
            topic: "events_json".to_string(),
            partition: 42,
            old_owner: "consumer-1".to_string(),
            new_owner: "consumer-3".to_string(),
            phase: HandoffPhase::Warming,
            started_at: 1700000000,
        };
        assert_eq!(
            handoff.topic_partition(),
            TopicPartition {
                topic: "events_json".to_string(),
                partition: 42,
            }
        );
    }

    #[test]
    fn consumer_status_variants_serialize() {
        for (status, expected) in [
            (ConsumerStatus::Ready, "\"Ready\""),
            (ConsumerStatus::Draining, "\"Draining\""),
        ] {
            assert_eq!(serde_json::to_string(&status).unwrap(), expected);
        }
    }

    #[test]
    fn handoff_phase_variants_serialize() {
        for (phase, expected) in [
            (HandoffPhase::Warming, "\"Warming\""),
            (HandoffPhase::Ready, "\"Ready\""),
            (HandoffPhase::Complete, "\"Complete\""),
        ] {
            assert_eq!(serde_json::to_string(&phase).unwrap(), expected);
        }
    }

    #[test]
    fn topic_partition_eq_and_hash() {
        use std::collections::HashSet;

        let tp1 = TopicPartition {
            topic: "events".to_string(),
            partition: 0,
        };
        let tp2 = TopicPartition {
            topic: "events".to_string(),
            partition: 0,
        };
        let tp3 = TopicPartition {
            topic: "clicks".to_string(),
            partition: 0,
        };

        assert_eq!(tp1, tp2);
        assert_ne!(tp1, tp3);

        let mut set = HashSet::new();
        set.insert(tp1.clone());
        assert!(set.contains(&tp2));
        assert!(!set.contains(&tp3));
    }
}