Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

20.5 KB
Raw Blame
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;

use futures::StreamExt;
use k8s_openapi::api::apps::v1::{Deployment, ReplicaSet, StatefulSet};
use kube::api::{Api, ListParams};
use kube::runtime::watcher::{self, Config as WatcherConfig, Event};
use kube::runtime::WatchStreamExt;
use kube::Client;
use tokio::sync::RwLock;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, warn};

use crate::detection;
use crate::discovery::{self, DiscoveryError};
use crate::types::{ClusterIntent, ControllerKind, ControllerRef, DepartureReason, PodInfo};

/// Timeout for `stream.next()` in watcher loops. Prevents the watcher task from
/// blocking indefinitely when the K8s API server is unreachable, which would
/// otherwise create a deadlock path (the assigner cannot make progress).
const STREAM_NEXT_TIMEOUT: Duration = Duration::from_secs(30);

#[derive(Debug, thiserror::Error)]
pub enum K8sAwarenessError {
    #[error("discovery failed: {0}")]
    Discovery(#[from] DiscoveryError),
    #[error("kubernetes API error: {0}")]
    Kube(#[from] kube::Error),
}

/// Manages K8s watchers for multiple controllers.
///
/// One assigner can serve consumers from N different Deployments/StatefulSets.
/// `K8sAwareness` auto-discovers each consumer's controller via ownerReferences
/// and starts a watcher for new controllers as they're discovered.
pub struct K8sAwareness {
    client: Client,
    namespace: String,
    controllers: Arc<RwLock<HashMap<ControllerRef, ClusterIntent>>>,
    watching: Arc<RwLock<HashMap<ControllerRef, CancellationToken>>>,
    cancel: CancellationToken,
}

impl K8sAwareness {
    pub fn new(client: Client, namespace: String, cancel: CancellationToken) -> Self {
        Self {
            client,
            namespace,
            controllers: Arc::new(RwLock::new(HashMap::new())),
            watching: Arc::new(RwLock::new(HashMap::new())),
            cancel,
        }
    }

    /// Discover which controller owns a pod and start watching it.
    ///
    /// Walks ownerReferences: pod → ReplicaSet → Deployment, or pod → StatefulSet.
    /// Starts a watcher for the controller if not already watching.
    pub async fn discover_controller(&self, pod_name: &str) -> Result<PodInfo, K8sAwarenessError> {
        let pod_info =
            discovery::discover_controller(&self.client, &self.namespace, pod_name).await?;

        self.ensure_watching(&pod_info.controller).await?;

        Ok(pod_info)
    }

    /// Start watching a controller known by reference — the entry point
    /// for consumers that learn controllers from member registrations
    /// rather than pod-name discovery (the coordinator sees each pod's
    /// self-reported controller ref and has no pod of its own to
    /// discover from). Idempotent: an already-watched controller is a
    /// no-op. Returns whether this call started a new watch, so callers
    /// that need intent can bound-wait for the first report instead of
    /// planning without it.
    pub async fn watch_controller(
        &self,
        controller: &ControllerRef,
    ) -> Result<bool, K8sAwarenessError> {
        self.ensure_watching(controller).await
    }

    /// Like [`Self::cluster_intent`], but polls up to `deadline` for a
    /// just-started watcher's first report. A freshly elected
    /// coordinator's first evaluation otherwise plans deterministically
    /// without intent — and mid-rollout, planning without placement
    /// policies means a balanced plan that moves partitions the rollout
    /// has already placed. `None` after the deadline (API server down,
    /// or controller genuinely unwatched) degrades exactly like
    /// [`Self::cluster_intent`] returning `None`.
    pub async fn cluster_intent_within(
        &self,
        controller: &ControllerRef,
        deadline: Duration,
    ) -> Option<ClusterIntent> {
        let start = tokio::time::Instant::now();
        loop {
            if let Some(intent) = self.cluster_intent(controller).await {
                return Some(intent);
            }
            if start.elapsed() >= deadline {
                return None;
            }
            tokio::time::sleep(Duration::from_millis(50)).await;
        }
    }

    /// Classify why a member is departing based on its controller's current intent.
    ///
    /// Returns `DepartureReason::Unknown` if the controller isn't being watched
    /// or if the lock is contended (avoids blocking the assigner).
    pub async fn classify_departure(
        &self,
        controller: &ControllerRef,
        generation: &str,
    ) -> DepartureReason {
        // Use try_read to avoid blocking if the watcher is updating controllers.
        // If the lock is contended, return Unknown — the assigner will treat the
        // consumer as active, which is the safe default.
        match self.controllers.try_read() {
            Ok(controllers) => match controllers.get(controller) {
                Some(intent) => detection::classify_departure(intent, generation),
                None => {
                    warn!(
                        controller = %controller,
                        "no cluster intent available, returning Unknown"
                    );
                    DepartureReason::Unknown
                }
            },
            Err(_) => {
                debug!(controller = %controller, "controllers lock contended, returning Unknown");
                DepartureReason::Unknown
            }
        }
    }

    /// The watcher's current view of a controller's intent, if watched.
    ///
    /// Returns `None` when the controller isn't being watched or hasn't
    /// reported yet — callers must treat that as "no rollout signal" and
    /// fall back to policy-free behavior. Waits for the lock (writers
    /// hold it only for map updates): a `try_read` here would blank the
    /// intent under contention, and a blanked intent flips a whole
    /// rollout's placement policies to plain balancing for that
    /// evaluation, churning partitions in both directions.
    pub async fn cluster_intent(&self, controller: &ControllerRef) -> Option<ClusterIntent> {
        self.controllers.read().await.get(controller).cloned()
    }

    /// Start watching a controller if not already watching. Returns
    /// whether a new watch was started.
    async fn ensure_watching(&self, controller: &ControllerRef) -> Result<bool, K8sAwarenessError> {
        let mut watching = self.watching.write().await;
        if watching.contains_key(controller) {
            return Ok(false);
        }

        let child_cancel = self.cancel.child_token();
        watching.insert(controller.clone(), child_cancel.clone());
        drop(watching);

        let controllers = Arc::clone(&self.controllers);
        let watching = Arc::clone(&self.watching);
        let client = self.client.clone();
        let namespace = self.namespace.clone();
        let controller_ref = controller.clone();

        info!(controller = %controller_ref, "starting K8s watcher");

        tokio::spawn(async move {
            match controller_ref.kind {
                ControllerKind::Deployment => {
                    run_deployment_watcher(
                        &client,
                        &namespace,
                        &controller_ref,
                        &controllers,
                        &watching,
                        child_cancel,
                    )
                    .await;
                }
                ControllerKind::StatefulSet => {
                    run_statefulset_watcher(
                        &client,
                        &namespace,
                        &controller_ref,
                        &controllers,
                        &watching,
                        child_cancel,
                    )
                    .await;
                }
            }
        });

        Ok(true)
    }
}

/// Remove a controller from the `watching` map so `ensure_watching` can restart it.
async fn unregister_watcher(
    controller: &ControllerRef,
    watching: &Arc<RwLock<HashMap<ControllerRef, CancellationToken>>>,
) {
    let mut map = watching.write().await;
    map.remove(controller);
    info!(controller = %controller, "unregistered watcher");
}

/// Watch a Deployment and update its ClusterIntent on changes.
async fn run_deployment_watcher(
    client: &Client,
    namespace: &str,
    controller: &ControllerRef,
    controllers: &Arc<RwLock<HashMap<ControllerRef, ClusterIntent>>>,
    watching: &Arc<RwLock<HashMap<ControllerRef, CancellationToken>>>,
    cancel: CancellationToken,
) {
    let api: Api<Deployment> = Api::namespaced(client.clone(), namespace);
    let config = WatcherConfig::default().fields(&format!("metadata.name={}", controller.name));

    // Without backoff, an unreachable API server makes the watcher retry
    // its connection in a hot loop, flooding logs at sub-millisecond rate.
    let stream = watcher::watcher(api, config).default_backoff();
    tokio::pin!(stream);

    loop {
        tokio::select! {
            _ = cancel.cancelled() => break,
            item = tokio::time::timeout(STREAM_NEXT_TIMEOUT, stream.next()) => {
                match item {
                    Ok(Some(Ok(event))) => {
                        handle_deployment_event(
                            client,
                            namespace,
                            controller,
                            controllers,
                            watching,
                            event,
                        ).await;
                    }
                    Ok(Some(Err(e))) => {
                        warn!(
                            controller = %controller,
                            error = %e,
                            "deployment watcher error, stream will retry"
                        );
                    }
                    Ok(None) => {
                        info!(controller = %controller, "deployment watcher stream ended");
                        break;
                    }
                    Err(_elapsed) => {
                        debug!(
                            controller = %controller,
                            "deployment watcher stream.next() timed out, retrying"
                        );
                    }
                }
            }
        }
    }

    unregister_watcher(controller, watching).await;
}

async fn handle_deployment_event(
    client: &Client,
    namespace: &str,
    controller: &ControllerRef,
    controllers: &Arc<RwLock<HashMap<ControllerRef, ClusterIntent>>>,
    watching: &Arc<RwLock<HashMap<ControllerRef, CancellationToken>>>,
    event: Event<Deployment>,
) {
    let deploy = match event {
        Event::Apply(d) | Event::InitApply(d) => d,
        Event::Delete(_) => {
            info!(controller = %controller, "deployment deleted");
            let mut map = controllers.write().await;
            map.remove(controller);
            unregister_watcher(controller, watching).await;
            return;
        }
        Event::Init | Event::InitDone => return,
    };

    let spec = match &deploy.spec {
        Some(s) => s,
        None => return,
    };
    let status = deploy.status.as_ref();

    let desired_replicas = spec.replicas.unwrap_or(1) as u32;
    let generation = deploy.metadata.generation.unwrap_or(0);
    let observed_generation = status.and_then(|s| s.observed_generation).unwrap_or(0);
    // observedGeneration catches up seconds after a spec change, long
    // before pods actually roll — on its own it only covers the window
    // until the new ReplicaSet exists. The rollout itself is in progress
    // for as long as more than one generation has desired replicas, which
    // is judged below once the ReplicaSets are listed.
    let spec_pending = generation != observed_generation;

    // Build a label selector from the Deployment's matchLabels to scope RS queries
    let label_selector = spec
        .selector
        .match_labels
        .as_ref()
        .map(|labels| {
            labels
                .iter()
                .map(|(k, v)| format!("{k}={v}"))
                .collect::<Vec<_>>()
                .join(",")
        })
        .unwrap_or_default();

    // Determine generations by inspecting owned ReplicaSets.
    // If this fails, skip the entire intent update to avoid mixing fresh
    // deployment fields (like rollout_in_progress) with stale generation data,
    // which would create an inconsistent ClusterIntent.
    let owned_rses =
        list_owned_replicasets(client, namespace, &controller.name, &label_selector).await;
    let rses = match owned_rses {
        Ok(rses) => rses,
        Err(e) => {
            if spec_pending {
                warn!(
                    controller = %controller,
                    error = %e,
                    "failed to list ReplicaSets during rollout, skipping intent update"
                );
            } else {
                debug!(
                    controller = %controller,
                    error = %e,
                    "failed to list ReplicaSets, skipping intent update"
                );
            }
            return;
        }
    };

    let live_generations = rses.iter().filter(|(_, _, replicas)| *replicas > 0).count();
    let rollout_in_progress = spec_pending || live_generations > 1;

    let (current_gen, target_gen) = if rollout_in_progress {
        let target = rses
            .first()
            .map(|(_, hash, _)| hash.clone())
            .unwrap_or_default();
        let current = rses
            .iter()
            .skip(1)
            .find(|(_, _, replicas)| *replicas > 0)
            .map(|(_, hash, _)| hash.clone())
            .unwrap_or_default();
        (current, Some(target))
    } else {
        let gen = rses
            .first()
            .map(|(_, hash, _)| hash.clone())
            .unwrap_or_default();
        (gen, None)
    };

    let mut map = controllers.write().await;
    // Only update previous_replicas when desired_replicas actually changes,
    // otherwise status-only events would overwrite it with the current value.
    let previous_replicas = map.get(controller).and_then(|prev| {
        if prev.desired_replicas != desired_replicas {
            Some(prev.desired_replicas)
        } else {
            prev.previous_replicas
        }
    });

    let intent = ClusterIntent {
        desired_replicas,
        previous_replicas,
        rollout_in_progress,
        current_generation: current_gen,
        target_generation: target_gen,
    };

    debug!(
        controller = %controller,
        desired_replicas = intent.desired_replicas,
        previous_replicas = ?intent.previous_replicas,
        rollout_in_progress = intent.rollout_in_progress,
        current_generation = %intent.current_generation,
        target_generation = ?intent.target_generation,
        "updated deployment cluster intent"
    );

    map.insert(controller.clone(), intent);
}

/// List ReplicaSets owned by a Deployment, sorted by revision descending.
///
/// Returns `(revision, pod-template-hash, replica_count)` tuples.
/// Uses the Deployment's pod-selector labels to scope the query instead of
/// listing all RSes in the namespace.
async fn list_owned_replicasets(
    client: &Client,
    namespace: &str,
    deployment_name: &str,
    label_selector: &str,
) -> Result<Vec<(u64, String, i32)>, kube::Error> {
    let rs_api: Api<ReplicaSet> = Api::namespaced(client.clone(), namespace);
    let params = if label_selector.is_empty() {
        ListParams::default()
    } else {
        ListParams::default().labels(label_selector)
    };
    let rs_list = rs_api.list(&params).await?;

    let mut owned: Vec<(u64, String, i32)> = Vec::new();

    for rs in &rs_list.items {
        let owners = rs.metadata.owner_references.as_deref().unwrap_or_default();
        if !owners
            .iter()
            .any(|o| o.kind == "Deployment" && o.name == deployment_name)
        {
            continue;
        }

        let revision = rs
            .metadata
            .annotations
            .as_ref()
            .and_then(|a| a.get("deployment.kubernetes.io/revision"))
            .and_then(|v| v.parse::<u64>().ok())
            .unwrap_or(0);

        let hash = rs
            .metadata
            .labels
            .as_ref()
            .and_then(|l| l.get("pod-template-hash"))
            .cloned()
            .unwrap_or_default();

        let replicas = rs.spec.as_ref().and_then(|s| s.replicas).unwrap_or(0);

        owned.push((revision, hash, replicas));
    }

    // Sort by revision descending: newest first
    owned.sort_by(|a, b| b.0.cmp(&a.0));

    Ok(owned)
}

/// Watch a StatefulSet and update its ClusterIntent on changes.
async fn run_statefulset_watcher(
    client: &Client,
    namespace: &str,
    controller: &ControllerRef,
    controllers: &Arc<RwLock<HashMap<ControllerRef, ClusterIntent>>>,
    watching: &Arc<RwLock<HashMap<ControllerRef, CancellationToken>>>,
    cancel: CancellationToken,
) {
    let api: Api<StatefulSet> = Api::namespaced(client.clone(), namespace);
    let config = WatcherConfig::default().fields(&format!("metadata.name={}", controller.name));

    // Same backoff rationale as the deployment watcher above.
    let stream = watcher::watcher(api, config).default_backoff();
    tokio::pin!(stream);

    loop {
        tokio::select! {
            _ = cancel.cancelled() => break,
            item = tokio::time::timeout(STREAM_NEXT_TIMEOUT, stream.next()) => {
                match item {
                    Ok(Some(Ok(event))) => {
                        handle_statefulset_event(controller, controllers, watching, event).await;
                    }
                    Ok(Some(Err(e))) => {
                        warn!(
                            controller = %controller,
                            error = %e,
                            "statefulset watcher error, stream will retry"
                        );
                    }
                    Ok(None) => {
                        info!(controller = %controller, "statefulset watcher stream ended");
                        break;
                    }
                    Err(_elapsed) => {
                        debug!(
                            controller = %controller,
                            "statefulset watcher stream.next() timed out, retrying"
                        );
                    }
                }
            }
        }
    }

    unregister_watcher(controller, watching).await;
}

async fn handle_statefulset_event(
    controller: &ControllerRef,
    controllers: &Arc<RwLock<HashMap<ControllerRef, ClusterIntent>>>,
    watching: &Arc<RwLock<HashMap<ControllerRef, CancellationToken>>>,
    event: Event<StatefulSet>,
) {
    let ss = match event {
        Event::Apply(s) | Event::InitApply(s) => s,
        Event::Delete(_) => {
            info!(controller = %controller, "statefulset deleted");
            let mut map = controllers.write().await;
            map.remove(controller);
            unregister_watcher(controller, watching).await;
            return;
        }
        Event::Init | Event::InitDone => return,
    };

    let spec = match &ss.spec {
        Some(s) => s,
        None => return,
    };
    let status = ss.status.as_ref();

    let desired_replicas = spec.replicas.unwrap_or(1) as u32;

    // StatefulSet rollout: currentRevision != updateRevision
    let current_revision = status
        .and_then(|s| s.current_revision.clone())
        .unwrap_or_default();
    let update_revision = status
        .and_then(|s| s.update_revision.clone())
        .unwrap_or_default();

    let rollout_in_progress = !current_revision.is_empty()
        && !update_revision.is_empty()
        && current_revision != update_revision;

    let target_generation = if rollout_in_progress {
        Some(update_revision.clone())
    } else {
        None
    };

    let mut map = controllers.write().await;
    let previous_replicas = map.get(controller).and_then(|prev| {
        if prev.desired_replicas != desired_replicas {
            Some(prev.desired_replicas)
        } else {
            prev.previous_replicas
        }
    });

    let intent = ClusterIntent {
        desired_replicas,
        previous_replicas,
        rollout_in_progress,
        current_generation: current_revision,
        target_generation,
    };

    debug!(
        controller = %controller,
        desired_replicas = intent.desired_replicas,
        previous_replicas = ?intent.previous_replicas,
        rollout_in_progress = intent.rollout_in_progress,
        current_generation = %intent.current_generation,
        target_generation = ?intent.target_generation,
        "updated statefulset cluster intent"
    );

    map.insert(controller.clone(), intent);
}