Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

23.1 KB
Raw Blame
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;

use anyhow::Context;
use axum::extract::{Path, Query, State};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use axum::routing::{delete, get, post};
use axum::{Json, Router};
use personhog_coordination::types::{ChangelogConfig, MigrationRequest, MigrationState};
use serde::{Deserialize, Serialize};
use serde_json::json;
use uuid::Uuid;

use k8s_awareness::DiscoveredPod;

use crate::changelog::{self, RestoreError};
use crate::etcd;
use crate::jobs::{AnalysisRequest, JobView};
use crate::kafka::browse::{self, BrowseParams, BrowseStop, MessageFilter, MessageRecord};
use crate::kafka::client;
use crate::kafka::lag::{self, ConsumerTarget, GroupLag, LagOverview};
use crate::parity;
use crate::personhog;
use crate::proxy;
use crate::state::AppState;
use crate::ui;

pub struct ApiError {
    status: StatusCode,
    message: String,
}

impl ApiError {
    pub fn bad_request(message: impl Into<String>) -> Self {
        Self {
            status: StatusCode::BAD_REQUEST,
            message: message.into(),
        }
    }

    pub fn not_found(message: impl Into<String>) -> Self {
        Self {
            status: StatusCode::NOT_FOUND,
            message: message.into(),
        }
    }

    pub fn upstream(message: impl Into<String>) -> Self {
        Self {
            status: StatusCode::BAD_GATEWAY,
            message: message.into(),
        }
    }

    pub fn unavailable(message: impl Into<String>) -> Self {
        Self {
            status: StatusCode::SERVICE_UNAVAILABLE,
            message: message.into(),
        }
    }

    pub fn too_many_requests(message: impl Into<String>) -> Self {
        Self {
            status: StatusCode::TOO_MANY_REQUESTS,
            message: message.into(),
        }
    }
}

impl IntoResponse for ApiError {
    fn into_response(self) -> Response {
        (self.status, Json(json!({ "error": self.message }))).into_response()
    }
}

/// The API only accepts exact group/topic pairs present in the (cached)
/// discovered targets — prefix checks alone would let a fabricated group
/// read any prefixed topic, and the tool must not become a generic Kafka
/// browser. Prefix mismatches fail fast with a clearer message.
async fn validated_target(
    state: &AppState,
    group: &str,
    topic: &str,
) -> Result<ConsumerTarget, ApiError> {
    if !group.starts_with(&state.config.group_prefix) {
        return Err(ApiError::bad_request(format!(
            "group '{group}' is outside the '{}' prefix",
            state.config.group_prefix
        )));
    }
    if !topic.starts_with(&state.config.topic_prefix) {
        return Err(ApiError::bad_request(format!(
            "topic '{topic}' is outside the '{}' prefix",
            state.config.topic_prefix
        )));
    }

    let targets = state
        .discovery
        .get_or_refresh(|| lag::discover_targets(&state.config))
        .await
        .map_err(|e| ApiError::upstream(format!("target discovery failed: {e:#}")))?;
    let target = ConsumerTarget {
        group: group.to_string(),
        topic: topic.to_string(),
    };
    if !targets.contains(&target) {
        return Err(ApiError::bad_request(format!(
            "'{group}' / '{topic}' is not a discovered consumer target \
             (discovery refreshes every {}s)",
            state.config.discovery_cache_ttl_secs
        )));
    }
    Ok(target)
}

async fn get_lag_overview(State(state): State<AppState>) -> Result<Json<LagOverview>, ApiError> {
    let overview = state
        .overview
        .get_or_refresh(|| async {
            let targets = state
                .discovery
                .get_or_refresh(|| lag::discover_targets(&state.config))
                .await?;
            Ok(lag::scan_targets(&state.config, targets.as_ref().clone()).await)
        })
        .await
        .map_err(|e| ApiError::upstream(format!("lag overview failed: {e:#}")))?;
    Ok(Json(overview.as_ref().clone()))
}

#[derive(Deserialize)]
struct LagQuery {
    group: String,
    topic: String,
}

async fn get_lag(
    State(state): State<AppState>,
    Query(query): Query<LagQuery>,
) -> Result<Json<GroupLag>, ApiError> {
    let target = validated_target(&state, &query.group, &query.topic).await?;

    let group_lag = lag::scan_group_lag(&state.config, &target)
        .await
        .map_err(|e| ApiError::upstream(format!("lag scan failed: {e:#}")))?;

    Ok(Json(group_lag))
}

async fn create_analysis(
    State(state): State<AppState>,
    Json(request): Json<AnalysisRequest>,
) -> Result<(StatusCode, Json<serde_json::Value>), ApiError> {
    let target = validated_target(&state, &request.group, &request.topic).await?;
    if request.partition < 0 {
        return Err(ApiError::bad_request("partition must be non-negative"));
    }
    let Some(slot) = state.jobs.try_reserve_slot() else {
        return Err(ApiError::too_many_requests(
            "too many pending analyses; wait for current jobs to finish or cancel one",
        ));
    };

    let job_id = state
        .jobs
        .start(
            Arc::clone(&state.config),
            Arc::clone(&state.teams),
            target,
            request,
            slot,
        )
        .await
        .map_err(|e| ApiError::upstream(format!("failed to start analysis: {e:#}")))?;

    Ok((StatusCode::ACCEPTED, Json(json!({ "job_id": job_id }))))
}

async fn list_analyses(State(state): State<AppState>) -> Json<Vec<JobView>> {
    Json(state.jobs.list())
}

async fn get_analysis(
    State(state): State<AppState>,
    Path(id): Path<Uuid>,
) -> Result<Json<JobView>, ApiError> {
    let job = state
        .jobs
        .get(&id)
        .ok_or_else(|| ApiError::not_found(format!("no analysis job '{id}'")))?;
    Ok(Json(job.view()))
}

async fn cancel_analysis(
    State(state): State<AppState>,
    Path(id): Path<Uuid>,
) -> Result<StatusCode, ApiError> {
    if state.jobs.cancel(&id) {
        Ok(StatusCode::NO_CONTENT)
    } else {
        Err(ApiError::not_found(format!("no analysis job '{id}'")))
    }
}

#[derive(Deserialize)]
struct MessagesQuery {
    group: String,
    topic: String,
    partition: i32,
    /// Cursor; defaults to the group's committed offset (or the low
    /// watermark when the group never committed).
    start_offset: Option<i64>,
    limit: Option<usize>,
    token: Option<String>,
    event: Option<String>,
    distinct_id: Option<String>,
}

#[derive(Serialize)]
struct MessagesResponse {
    partition: i32,
    start_offset: i64,
    /// Cursor for the next page.
    next_offset: i64,
    low_watermark: i64,
    high_watermark: i64,
    scanned: u64,
    stop: BrowseStop,
    reached_end: bool,
    duration_ms: u64,
    messages: Vec<MessageRecord>,
    /// team_id per distinct token in `messages` (null when unresolvable).
    token_teams: HashMap<String, Option<i32>>,
}

/// Synchronous filtered scan of one partition, returning matched message
/// headers (never payloads) plus a cursor. Bounded per request by the page
/// limit, a scan budget, a deadline, and a byte budget, whichever hits first.
async fn get_messages(
    State(state): State<AppState>,
    Query(query): Query<MessagesQuery>,
) -> Result<Json<MessagesResponse>, ApiError> {
    let target = validated_target(&state, &query.group, &query.topic).await?;
    if query.partition < 0 {
        return Err(ApiError::bad_request("partition must be non-negative"));
    }
    let limit = query.limit.unwrap_or(100).clamp(1, 500);
    // An empty query-param value means "not filtered", so it can never pin a
    // filter to the empty string.
    let non_empty = |s: Option<String>| s.filter(|s| !s.is_empty());
    let filter = MessageFilter {
        token: non_empty(query.token),
        event: non_empty(query.event),
        distinct_id: non_empty(query.distinct_id),
    };

    let Ok(_permit) = Arc::clone(&state.browse_permits).try_acquire_owned() else {
        return Err(ApiError::too_many_requests(
            "too many concurrent message scans; wait for one to finish",
        ));
    };

    let config = Arc::clone(&state.config);
    let partition = query.partition;
    let requested_start = query.start_offset;
    let (bounds, start_offset, outcome) = tokio::task::spawn_blocking(move || {
        let timeout = Duration::from_millis(config.kafka_metadata_timeout_ms);
        let bounds = lag::fetch_partition_bounds_blocking(&config, &target, partition, timeout)?;
        let start_offset = requested_start
            .unwrap_or_else(|| bounds.committed_offset.unwrap_or(bounds.low_watermark))
            .clamp(bounds.low_watermark, bounds.high_watermark);
        let params = BrowseParams {
            topic: target.topic.clone(),
            partition,
            start_offset,
            end_offset_exclusive: bounds.high_watermark,
            limit,
            scan_limit: config.browse_scan_message_count,
            deadline: Duration::from_secs(config.browse_deadline_secs),
            max_bytes: config.browse_max_fetch_bytes,
            poll_timeout: Duration::from_millis(config.kafka_fetch_poll_timeout_ms),
        };
        let consumer = client::fetch_consumer(&config).context("create fetch client")?;
        let outcome = browse::run_browse(&consumer, &params, &filter)?;
        anyhow::Ok((bounds, start_offset, outcome))
    })
    .await
    .context("browse task panicked")
    .and_then(|res| res)
    .map_err(|e| ApiError::upstream(format!("message scan failed: {e:#}")))?;

    let mut tokens: Vec<String> = outcome
        .records
        .iter()
        .filter_map(|record| record.token.clone())
        .collect();
    tokens.sort();
    tokens.dedup();
    let token_teams = state.teams.resolve(&tokens).await;

    Ok(Json(MessagesResponse {
        partition: query.partition,
        start_offset,
        next_offset: outcome.next_offset,
        low_watermark: bounds.low_watermark,
        high_watermark: bounds.high_watermark,
        scanned: outcome.scanned,
        stop: outcome.stop,
        reached_end: outcome.next_offset >= bounds.high_watermark,
        duration_ms: outcome.duration_ms,
        messages: outcome.records,
        token_teams,
    }))
}

/// The configured etcd handle, or 503 when `ETCD_ENDPOINTS` is unset.
fn etcd_handle(state: &AppState) -> Result<Arc<crate::etcd::EtcdHandle>, ApiError> {
    state
        .etcd
        .clone()
        .ok_or_else(|| ApiError::unavailable("etcd tools are disabled (ETCD_ENDPOINTS not set)"))
}

async fn etcd_client(state: &AppState) -> Result<etcd_client::Client, ApiError> {
    etcd_handle(state)?
        .client()
        .await
        .map_err(|e| ApiError::unavailable(format!("etcd unreachable: {e:#}")))
}

/// Every explorer operation must stay under one of the configured allowed
/// prefixes, so a deployment can scope what this unauthenticated-but-
/// internal tool may touch.
fn validated_etcd_key<'a>(state: &AppState, key: &'a str) -> Result<&'a str, ApiError> {
    if key.is_empty() {
        return Err(ApiError::bad_request("key must not be empty"));
    }
    let allowed = state.config.etcd_allowed_prefix_list();
    if !allowed.iter().any(|prefix| key.starts_with(prefix)) {
        return Err(ApiError::bad_request(format!(
            "key '{key}' is outside the allowed prefixes {allowed:?}"
        )));
    }
    Ok(key)
}

#[derive(Deserialize)]
struct EtcdKeysQuery {
    prefix: String,
    from_key: Option<String>,
    limit: Option<i64>,
}

async fn etcd_list_keys(
    State(state): State<AppState>,
    Query(query): Query<EtcdKeysQuery>,
) -> Result<Json<etcd::KeyList>, ApiError> {
    validated_etcd_key(&state, &query.prefix)?;
    let limit = query.limit.unwrap_or(500).clamp(1, 1000);
    let client = etcd_client(&state).await?;
    let list = etcd::list_keys(&client, &query.prefix, query.from_key.as_deref(), limit)
        .await
        .map_err(|e| ApiError::upstream(format!("etcd list failed: {e:#}")))?;
    Ok(Json(list))
}

#[derive(Deserialize)]
struct EtcdKeyQuery {
    key: String,
}

async fn etcd_get_key(
    State(state): State<AppState>,
    Query(query): Query<EtcdKeyQuery>,
) -> Result<Json<etcd::KeyDetail>, ApiError> {
    validated_etcd_key(&state, &query.key)?;
    let client = etcd_client(&state).await?;
    let detail = etcd::get_key(&client, &query.key)
        .await
        .map_err(|e| ApiError::upstream(format!("etcd get failed: {e:#}")))?
        .ok_or_else(|| ApiError::not_found(format!("no key '{}'", query.key)))?;
    Ok(Json(detail))
}

#[derive(Deserialize)]
struct EtcdPutRequest {
    key: String,
    value: String,
}

async fn etcd_put_key(
    State(state): State<AppState>,
    Json(request): Json<EtcdPutRequest>,
) -> Result<Json<serde_json::Value>, ApiError> {
    validated_etcd_key(&state, &request.key)?;
    let client = etcd_client(&state).await?;
    let revision = etcd::put_key(&client, &request.key, &request.value)
        .await
        .map_err(|e| ApiError::upstream(format!("etcd put failed: {e:#}")))?;
    Ok(Json(json!({ "revision": revision })))
}

async fn etcd_delete_key(
    State(state): State<AppState>,
    Query(query): Query<EtcdKeyQuery>,
) -> Result<StatusCode, ApiError> {
    validated_etcd_key(&state, &query.key)?;
    let client = etcd_client(&state).await?;
    let deleted = etcd::delete_key(&client, &query.key)
        .await
        .map_err(|e| ApiError::upstream(format!("etcd delete failed: {e:#}")))?;
    if deleted {
        Ok(StatusCode::NO_CONTENT)
    } else {
        Err(ApiError::not_found(format!("no key '{}'", query.key)))
    }
}

async fn personhog_topology(
    State(state): State<AppState>,
) -> Result<Json<personhog::TopologyView>, ApiError> {
    let client = etcd_client(&state).await?;
    let deadlines = personhog::Deadlines {
        handoff: Duration::from_secs(state.config.personhog_handoff_deadline_secs),
        warming: Duration::from_secs(state.config.personhog_warming_deadline_secs),
    };
    let view = personhog::fetch_topology(&client, &state.config.personhog_etcd_prefix, &deadlines)
        .await
        .map_err(|e| ApiError::upstream(format!("personhog topology failed: {e:#}")))?;
    Ok(Json(view))
}

async fn personhog_changelog(
    State(state): State<AppState>,
) -> Result<Json<changelog::ChangelogView>, ApiError> {
    let client = etcd_client(&state).await?;
    let view = changelog::view(&state.config, &client, &state.config.personhog_etcd_prefix)
        .await
        .map_err(|e| ApiError::upstream(format!("changelog view failed: {e:#}")))?;
    Ok(Json(view))
}

#[derive(Deserialize)]
struct SeedChangelogRequest {
    topic: String,
    /// The writer's consumer group, as its `KAFKA_CONSUMER_GROUP` names it.
    writer_group: String,
    #[serde(default)]
    cluster: Option<String>,
}

async fn personhog_seed_changelog(
    State(state): State<AppState>,
    Json(request): Json<SeedChangelogRequest>,
) -> Result<Json<ChangelogConfig>, ApiError> {
    let prefix = state.config.personhog_etcd_prefix.clone();
    validated_etcd_key(&state, &prefix)?;
    let topic = request.topic.trim();
    if topic.is_empty() {
        return Err(ApiError::bad_request("topic must not be empty"));
    }
    let writer_group = request.writer_group.trim();
    if writer_group.is_empty() {
        return Err(ApiError::bad_request("writer_group must not be empty"));
    }
    let client = etcd_client(&state).await?;
    let cluster = trimmed(request.cluster.as_deref());
    let seeded = changelog::seed(
        &state.config,
        &client,
        &prefix,
        topic,
        writer_group,
        cluster,
    )
    .await
    .map_err(|e| ApiError::bad_request(format!("seed failed: {e:#}")))?;
    Ok(Json(seeded))
}

fn trimmed(value: Option<&str>) -> Option<&str> {
    value.map(str::trim).filter(|v| !v.is_empty())
}

fn backup_store(state: &AppState) -> Result<Arc<dyn crate::backup::BackupStore>, ApiError> {
    state.backups.clone().ok_or_else(|| {
        ApiError::unavailable("changelog backups are disabled (CHANGELOG_BACKUP_BUCKET not set)")
    })
}

async fn personhog_changelog_backups(
    State(state): State<AppState>,
) -> Result<Json<Vec<crate::backup::BackupVersion>>, ApiError> {
    let backups = backup_store(&state)?;
    let versions = changelog::backups(backups.as_ref(), &state.config.personhog_etcd_prefix)
        .await
        .map_err(|e| ApiError::upstream(format!("listing backups failed: {e:#}")))?;
    Ok(Json(versions))
}

#[derive(Deserialize)]
struct RestoreChangelogRequest {
    version_id: Option<String>,
}

async fn personhog_restore_changelog(
    State(state): State<AppState>,
    Json(request): Json<RestoreChangelogRequest>,
) -> Result<Json<ChangelogConfig>, ApiError> {
    let prefix = state.config.personhog_etcd_prefix.clone();
    validated_etcd_key(&state, &prefix)?;
    let backups = backup_store(&state)?;
    let client = etcd_client(&state).await?;
    let version_id = request
        .version_id
        .as_deref()
        .map(str::trim)
        .filter(|v| !v.is_empty());
    let restored = changelog::restore(
        &state.config,
        &client,
        &prefix,
        backups.as_ref(),
        version_id,
    )
    .await
    .map_err(|e| match e {
        RestoreError::NotFound(message) => ApiError::not_found(message),
        RestoreError::Refused(message) => {
            ApiError::bad_request(format!("restore refused: {message}"))
        }
        RestoreError::Upstream(e) => ApiError::upstream(format!("restore failed: {e:#}")),
    })?;
    Ok(Json(restored))
}

#[derive(Deserialize)]
struct StartMigrationRequest {
    topic: String,
    #[serde(default)]
    cluster: Option<String>,
}

async fn personhog_start_migration(
    State(state): State<AppState>,
    Json(request): Json<StartMigrationRequest>,
) -> Result<Json<MigrationRequest>, ApiError> {
    let prefix = state.config.personhog_etcd_prefix.clone();
    validated_etcd_key(&state, &prefix)?;
    let topic = request.topic.trim();
    if topic.is_empty() {
        return Err(ApiError::bad_request("topic must not be empty"));
    }
    let client = etcd_client(&state).await?;
    let cluster = trimmed(request.cluster.as_deref());
    let migration = changelog::start_migration(&state.config, &client, &prefix, topic, cluster)
        .await
        .map_err(|e| ApiError::bad_request(format!("migration refused: {e:#}")))?;
    Ok(Json(migration))
}

#[derive(Deserialize)]
struct RetireTopicRequest {
    topic: String,
}

async fn personhog_retire_topic(
    State(state): State<AppState>,
    Json(request): Json<RetireTopicRequest>,
) -> Result<Json<serde_json::Value>, ApiError> {
    let prefix = state.config.personhog_etcd_prefix.clone();
    validated_etcd_key(&state, &prefix)?;
    let client = etcd_client(&state).await?;
    changelog::retire(&state.config, &client, &prefix, request.topic.trim())
        .await
        .map_err(|e| ApiError::bad_request(format!("retire refused: {e:#}")))?;
    Ok(Json(json!({ "retired": request.topic.trim() })))
}

#[derive(Deserialize)]
struct RetireGroupRequest {
    group: String,
}

async fn personhog_add_retire_group(
    State(state): State<AppState>,
    Json(request): Json<RetireGroupRequest>,
) -> Result<Json<serde_json::Value>, ApiError> {
    let prefix = state.config.personhog_etcd_prefix.clone();
    validated_etcd_key(&state, &prefix)?;
    let group = request.group.trim();
    if group.is_empty() {
        return Err(ApiError::bad_request("group must not be empty"));
    }
    let client = etcd_client(&state).await?;
    changelog::add_retire_group(&client, &prefix, group)
        .await
        .map_err(|e| ApiError::bad_request(format!("add retire group refused: {e:#}")))?;
    Ok(Json(json!({ "group": group })))
}

async fn personhog_remove_retire_group(
    State(state): State<AppState>,
    Json(request): Json<RetireGroupRequest>,
) -> Result<Json<serde_json::Value>, ApiError> {
    let prefix = state.config.personhog_etcd_prefix.clone();
    validated_etcd_key(&state, &prefix)?;
    let group = request.group.trim();
    let client = etcd_client(&state).await?;
    changelog::remove_retire_group(&client, &prefix, group)
        .await
        .map_err(|e| ApiError::bad_request(format!("remove retire group refused: {e:#}")))?;
    Ok(Json(json!({ "removed": group })))
}

#[derive(Deserialize)]
struct StopMigrationRequest {
    /// The request or saga the operator saw; anything else is left alone.
    id: String,
}

async fn personhog_cancel_migration(
    State(state): State<AppState>,
    Json(request): Json<StopMigrationRequest>,
) -> Result<Json<MigrationState>, ApiError> {
    let prefix = state.config.personhog_etcd_prefix.clone();
    validated_etcd_key(&state, &prefix)?;
    let client = etcd_client(&state).await?;
    let migration = changelog::cancel_migration(&client, &prefix, &request.id)
        .await
        .map_err(|e| ApiError::bad_request(format!("cancel refused: {e:#}")))?;
    Ok(Json(migration))
}

#[derive(Serialize)]
struct PodsResponse {
    pods: Vec<DiscoveredPod>,
}

async fn list_pods(State(state): State<AppState>) -> Result<Json<PodsResponse>, ApiError> {
    let pods = state
        .pods
        .list_pods(&state.config)
        .await
        .map_err(|e| ApiError::unavailable(format!("pod discovery unavailable: {e:#}")))?;
    Ok(Json(PodsResponse { pods }))
}

pub fn router(state: AppState) -> Router {
    Router::new()
        .route("/", get(ui::index))
        .route("/api/lag", get(get_lag))
        .route("/api/lag/overview", get(get_lag_overview))
        .route("/api/messages", get(get_messages))
        .route("/api/analyses", get(list_analyses).post(create_analysis))
        .route(
            "/api/analyses/:id",
            get(get_analysis).delete(cancel_analysis),
        )
        .route("/api/pods", get(list_pods))
        .route("/api/etcd/keys", get(etcd_list_keys))
        .route(
            "/api/etcd/key",
            get(etcd_get_key).put(etcd_put_key).delete(etcd_delete_key),
        )
        .route("/api/personhog/topology", get(personhog_topology))
        .route("/api/personhog/parity/mismatches", get(parity::mismatches))
        .route("/api/personhog/changelog", get(personhog_changelog))
        .route(
            "/api/personhog/changelog/seed",
            post(personhog_seed_changelog),
        )
        .route(
            "/api/personhog/changelog/backups",
            get(personhog_changelog_backups),
        )
        .route(
            "/api/personhog/changelog/restore",
            post(personhog_restore_changelog),
        )
        .route("/api/personhog/migrations", post(personhog_start_migration))
        .route(
            "/api/personhog/migrations/current",
            delete(personhog_cancel_migration),
        )
        .route(
            "/api/personhog/changelog/retire",
            post(personhog_retire_topic),
        )
        .route(
            "/api/personhog/changelog/retire-groups",
            post(personhog_add_retire_group).delete(personhog_remove_retire_group),
        )
        // The consumer debug UI is served per pod; its relative `debug/...`
        // fetches resolve to the proxy route below. Pods are addressed by
        // namespace + name so identical names across lanes cannot collide.
        .route("/pods/:namespace/:name/", get(ui::consumer_debug))
        .route(
            "/pods/:namespace/:name/debug/*rest",
            get(proxy::proxy_debug),
        )
        .with_state(state)
}