Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

20.1 KB
Raw Blame
use std::collections::HashMap;

use anyhow::{anyhow, Result};
use axum::extract::{Path, Query, State};
use axum::http::{HeaderMap, StatusCode};
use axum::Json;
use bytes::Bytes;
use chrono::{TimeZone, Utc};
use metrics::counter;
use prometheus_rw_proto::prometheus::v1::{
    metric_metadata::MetricType, MetricMetadata, TimeSeries, WriteRequest,
};
use prost::Message;
use serde::Deserialize;
use serde_json::json;
use tracing::{debug, error, instrument, warn};
use uuid::Uuid;

use crate::authorizer::Signal;
use crate::log_record::override_timestamp;
use crate::metric_record::{compute_series_fingerprint, KafkaMetricRow};
use crate::service::Service;

const METRIC_NAME_LABEL: &str = "__name__";
const JOB_LABEL: &str = "job";
const INSTANCE_LABEL: &str = "instance";

/// Approximate in-memory/Avro cost of one row beyond its label data (uuid,
/// timestamps, type/temporality strings, map overhead). Used by
/// [`estimate_expanded_bytes`] so a request full of near-empty samples still
/// registers a per-row cost.
const ROW_OVERHEAD_BYTES: u64 = 256;

/// How much larger than the (already decompression-capped) request body the
/// expanded row set may be. Row building clones the series label map per
/// sample and the whole batch becomes a single Kafka message, so a 2 wire-byte
/// sample expands to hundreds of bytes — legitimate senders (Prometheus
/// defaults to 2k samples per send, vmagent to 10k rows per block) stay far
/// below this; only expansion-bomb payloads exceed it.
const MAX_EXPANSION_FACTOR: u64 = 16;

/// Body compression of a remote-write request. Both protocols carry the same
/// `WriteRequest` protobuf; only the compression differs.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RemoteWriteEncoding {
    /// Prometheus remote-write v1: snappy block format.
    Snappy,
    /// VictoriaMetrics remote-write protocol, which vmagent tries first by
    /// default: zstd frame, sent with `Content-Encoding: zstd` and
    /// `X-VictoriaMetrics-Remote-Write-Version: 1`.
    Zstd,
}

impl RemoteWriteEncoding {
    /// `Content-Encoding: zstd` selects the VictoriaMetrics protocol. Every other
    /// value, and a missing header, keeps the snappy default, because Prometheus
    /// always sends snappy and some senders omit the header.
    pub fn from_headers(headers: &HeaderMap) -> Self {
        let is_zstd = headers
            .get("content-encoding")
            .and_then(|v| v.to_str().ok())
            .is_some_and(|v| v.trim().eq_ignore_ascii_case("zstd"));
        if is_zstd {
            Self::Zstd
        } else {
            Self::Snappy
        }
    }

    pub fn as_str(self) -> &'static str {
        match self {
            Self::Snappy => "snappy",
            Self::Zstd => "zstd",
        }
    }
}

/// Decode a compressed Prometheus remote-write v1 payload, returning the
/// request and its decompressed size in bytes.
///
/// Prometheus sends `Content-Encoding: snappy` using the snappy *block* format
/// (not the framed format), and vmagent sends `Content-Encoding: zstd`. The
/// global `RequestDecompressionLayer` understands neither, so we decode the
/// raw body ourselves.
///
/// Both formats let the sender claim the decompressed size (the snappy length
/// header, the optional zstd frame content size), so the claim is checked
/// against `max_decompressed_bytes` before any allocation happens. The zstd
/// decoder is also capped while it streams, for frames without a claim.
///
/// The size is returned because this route bypasses `RequestDecompressionLayer`,
/// so the request body is still compressed when the handler sees it. The OTLP and
/// logs paths report a body that layer already decompressed, and all three feed
/// the same quota and rate-limiting counters, so the measure has to match.
pub fn decode_write_request(
    body: &[u8],
    encoding: RemoteWriteEncoding,
    max_decompressed_bytes: usize,
) -> Result<(WriteRequest, u64)> {
    let decompressed = match encoding {
        RemoteWriteEncoding::Snappy => {
            let claimed = snap::raw::decompress_len(body)
                .map_err(|e| anyhow!("snappy decode failed: {e}"))?;
            if claimed > max_decompressed_bytes {
                return Err(anyhow!(
                    "decompressed request body exceeds limit ({claimed} > {max_decompressed_bytes} bytes)"
                ));
            }
            snap::raw::Decoder::new()
                .decompress_vec(body)
                .map_err(|e| anyhow!("snappy decode failed: {e}"))?
        }
        RemoteWriteEncoding::Zstd => {
            common_compression::decompress_zstd_capped(body, max_decompressed_bytes)
                .map_err(|e| anyhow!("zstd decode failed: {e}"))?
        }
    };
    let uncompressed_bytes = decompressed.len() as u64;
    let request = WriteRequest::decode(decompressed.as_slice())
        .map_err(|e| anyhow!("remote-write protobuf decode failed: {e}"))?;
    Ok((request, uncompressed_bytes))
}

/// Translate a decoded remote-write request into `KafkaMetricRow` records — the
/// same Avro shape the OTLP path produces, so the rest of the pipeline
/// (`ingestion-metrics` consumer → `clickhouse_metrics` → `metrics1`) is
/// unchanged. One row is emitted per sample. Returns the rows and the number of
/// samples whose timestamp was clamped by `override_timestamp`.
pub fn write_request_to_kafka_rows(req: WriteRequest) -> (Vec<KafkaMetricRow>, u64) {
    write_timeseries_to_kafka_rows(req.timeseries, &req.metadata)
}

pub fn write_timeseries_to_kafka_rows(
    timeseries: Vec<TimeSeries>,
    metadata: &[MetricMetadata],
) -> (Vec<KafkaMetricRow>, u64) {
    // Metric-family name -> declared (type, unit). Only populated when the
    // sender includes the optional metadata block (vmagent and many agents
    // omit it, so the name-suffix heuristic in `classify` is the primary path).
    let metadata = build_metadata_map(metadata);

    let mut rows = Vec::new();
    let mut timestamps_overridden = 0u64;
    let mut dropped_series = 0u64;
    let mut dropped_samples = 0u64;

    for series in timeseries {
        let mut metric_name = String::new();
        let mut service_name = String::new();
        let mut resource_attributes: HashMap<String, String> = HashMap::new();
        let mut attributes: HashMap<String, String> = HashMap::new();

        for label in series.labels {
            match label.name.as_str() {
                METRIC_NAME_LABEL => metric_name = label.value,
                // Map Prometheus topology labels onto OTel resource semantics so
                // `service_name` populates the same way it does for OTLP. Map
                // values are JSON-encoded to match the OTLP/Datadog paths — the
                // ClickHouse MV applies JSONExtractString to them.
                JOB_LABEL => {
                    service_name = label.value.clone();
                    resource_attributes
                        .insert("service.name".to_string(), json!(label.value).to_string());
                }
                // `instance` is copied, not moved: Prometheus queries filter on
                // `instance`, and OTel queries on `service.instance.id`.
                INSTANCE_LABEL => {
                    let value = json!(label.value).to_string();
                    resource_attributes.insert("service.instance.id".to_string(), value.clone());
                    attributes.insert(label.name, value);
                }
                _ => {
                    attributes.insert(label.name, json!(label.value).to_string());
                }
            }
        }

        // A series with no metric name is unusable; skip it.
        if metric_name.is_empty() {
            continue;
        }

        // The vendored v1 proto deliberately skips native-histogram fields, so a
        // native-histogram series decodes to labels with no float samples and
        // yields no rows. Count it rather than let the whole request 204 as if
        // nothing were lost — the sender would otherwise drop the batch silently.
        if series.samples.is_empty() {
            dropped_series += 1;
            continue;
        }

        let declared = lookup_metadata(&metadata, &metric_name);
        let unit = declared.map(|(_, u)| u.clone()).unwrap_or_default();
        let (metric_type, is_monotonic, temporality) =
            classify(&metric_name, declared.map(|(t, _)| *t));

        let series_fingerprint = compute_series_fingerprint(
            &metric_name,
            metric_type,
            &service_name,
            &resource_attributes,
            &attributes,
        );

        for sample in series.samples {
            // Prometheus marks a series stale with a NaN payload and remote-write
            // forwards it (vmagent emits these by default on scrape failure or
            // series churn). The metrics pipeline has no stale-marker concept, and
            // a non-finite value survives the storage/query path uncaught —
            // `ifNull` never fires on NaN, and it corrupts rate/aggregate results.
            // Drop it here, as staleness-unaware receivers do.
            if !sample.value.is_finite() {
                dropped_samples += 1;
                continue;
            }

            // Remote-write sample timestamps are milliseconds since the epoch.
            let raw_timestamp = Utc
                .timestamp_millis_opt(sample.timestamp)
                .single()
                .unwrap_or_else(Utc::now);
            let (timestamp, original_timestamp) = override_timestamp(raw_timestamp);

            let mut sample_attributes = attributes.clone();
            if let Some(original) = original_timestamp {
                timestamps_overridden += 1;
                sample_attributes.insert("$originalTimestamp".to_string(), original.to_rfc3339());
            }

            rows.push(KafkaMetricRow {
                uuid: Uuid::now_v7().to_string(),
                trace_id: String::new(),
                span_id: String::new(),
                trace_flags: 0,
                timestamp,
                observed_timestamp: Utc::now(),
                service_name: service_name.clone(),
                metric_name: metric_name.clone(),
                metric_type: metric_type.to_string(),
                value: sample.value,
                count: 1,
                histogram_bounds: Vec::new(),
                histogram_counts: Vec::new(),
                unit: unit.clone(),
                aggregation_temporality: temporality.to_string(),
                is_monotonic,
                resource_attributes: resource_attributes.clone(),
                instrumentation_scope: String::new(),
                attributes: sample_attributes,
                series_fingerprint,
                has_labels: true,
                retention_days: None,
            });
        }
    }

    if dropped_series > 0 {
        counter!("capture_metrics_remote_write_dropped_series").increment(dropped_series);
        warn!(
            "Dropped {dropped_series} remote-write series with no ingestable samples (native histograms are not yet supported)"
        );
    }

    // Stale markers are routine, so this is counted (for volume visibility) but
    // not warned about, unlike the dropped-series case above.
    if dropped_samples > 0 {
        counter!("capture_metrics_remote_write_dropped_samples").increment(dropped_samples);
    }

    (rows, timestamps_overridden)
}

/// Build the metric-family-name -> (type, unit) map from the optional metadata
/// block. Entries with an unrecognized type are dropped. Shared by the row
/// builder and the expansion estimate so both agree on exactly which units get
/// cloned into rows.
fn build_metadata_map(metadata: &[MetricMetadata]) -> HashMap<String, (MetricType, String)> {
    metadata
        .iter()
        .filter_map(|m| {
            MetricType::try_from(m.r#type)
                .ok()
                .map(|t| (m.metric_family_name.clone(), (t, m.unit.clone())))
        })
        .collect()
}

/// Look up declared metadata, falling back to the base family name —
/// histogram/summary/counter families register under a base name while the
/// exposed series append `_bucket`/`_sum`/`_count`/`_total`.
fn lookup_metadata<'a>(
    metadata: &'a HashMap<String, (MetricType, String)>,
    name: &str,
) -> Option<&'a (MetricType, String)> {
    if let Some(m) = metadata.get(name) {
        return Some(m);
    }
    for suffix in ["_bucket", "_sum", "_count", "_total"] {
        if let Some(base) = name.strip_suffix(suffix) {
            if let Some(m) = metadata.get(base) {
                return Some(m);
            }
        }
    }
    None
}

/// Approximate the bytes the request expands to as `KafkaMetricRow`s: every
/// sample becomes a row carrying a clone of its series' label data, its declared
/// metadata unit, plus fixed per-row overhead. Wire size bounds none of these —
/// a zero-valued `Sample` is 2 wire bytes, and one fat label set or a fat
/// `MetricMetadata.unit` is encoded once but cloned per sample — so this is
/// checked against a multiple of the body limit before any row is built. Also
/// keeps the single Kafka message the batch becomes bounded.
pub fn estimate_expanded_bytes(req: &WriteRequest) -> u64 {
    let metadata = build_metadata_map(&req.metadata);
    req.timeseries
        .iter()
        .map(|series| {
            let label_bytes: u64 = series
                .labels
                .iter()
                .map(|l| (l.name.len() + l.value.len()) as u64)
                .sum();
            // The declared unit is cloned into every row (see `write_request_to_
            // kafka_rows`), so it must be part of the per-row cost — otherwise a
            // fat unit plus many compact samples slips under the cap and only
            // explodes at row-build time. Take the largest candidate rather than
            // one specific `__name__` label: nothing rejects a series carrying
            // several, the builder's loop keeps the last, and an estimate that
            // resolved a different one would under-count by the whole unit.
            let unit_bytes = series
                .labels
                .iter()
                .filter(|l| l.name == METRIC_NAME_LABEL)
                .filter_map(|l| lookup_metadata(&metadata, &l.value))
                .map(|(_, unit)| unit.len() as u64)
                .max()
                .unwrap_or(0);
            (series.samples.len() as u64) * (ROW_OVERHEAD_BYTES + label_bytes + unit_bytes)
        })
        .sum()
}

/// Infer (metric_type, is_monotonic, aggregation_temporality) for a series.
///
/// Prometheus counters, histograms and summaries are cumulative. RW v1 carries
/// no per-sample type, so declared metadata (when present) takes precedence and
/// the `_total`/`_bucket`/`_sum`/`_count` suffix heuristic is the fallback.
/// Classic histograms/summaries are stored as their decomposed scalar component
/// series (queryable via PromQL); native-histogram array reconstruction is
/// deferred.
fn classify(name: &str, declared: Option<MetricType>) -> (&'static str, bool, &'static str) {
    if let Some(t) = declared {
        match t {
            MetricType::Counter => return ("sum", true, "cumulative"),
            MetricType::Gauge => return ("gauge", false, ""),
            // Histogram/Summary expose decomposed cumulative component series;
            // type each component via the suffix heuristic below.
            _ => {}
        }
    }

    if name.ends_with("_total")
        || name.ends_with("_bucket")
        || name.ends_with("_count")
        || name.ends_with("_sum")
    {
        ("sum", true, "cumulative")
    } else {
        ("gauge", false, "")
    }
}

#[derive(Deserialize)]
pub struct RemoteWriteQueryParams {
    token: Option<String>,
}

/// Resolve the project token from (in order) the URL path, the Authorization
/// header (Bearer or bare), or the `token` query param — matching the
/// flexibility remote-write senders actually have.
fn extract_token(
    path_token: Option<String>,
    headers: &HeaderMap,
    query_token: Option<&str>,
) -> Option<String> {
    if let Some(token) = path_token {
        if !token.is_empty() {
            return Some(token);
        }
    }
    if let Some(auth) = headers.get("authorization").and_then(|v| v.to_str().ok()) {
        let token = auth
            .strip_prefix("Bearer ")
            .or_else(|| auth.strip_prefix("bearer "))
            .unwrap_or(auth)
            .trim();
        if !token.is_empty() {
            return Some(token.to_string());
        }
    }
    query_token.filter(|t| !t.is_empty()).map(str::to_string)
}

/// Prometheus remote-write v1 ingestion endpoint. Also accepts the
/// VictoriaMetrics remote-write protocol (same protobuf, zstd body).
///
/// Decompresses the protobuf body, maps it to `KafkaMetricRow` records, and
/// produces them through the shared `KafkaSink` so the rest of the pipeline is
/// identical to OTLP ingestion. Response codes follow remote-write semantics:
/// 204 on success, 400 on a permanent decode failure (sender drops the batch;
/// vmagent also falls back to snappy on 400), 5xx on a transient produce
/// failure (sender retries).
#[instrument(skip_all, fields(
    token = tracing::field::Empty,
    user_agent = %headers.get("user-agent")
        .and_then(|v| v.to_str().ok())
        .unwrap_or(""),
    content_encoding = %headers.get("content-encoding")
        .and_then(|v| v.to_str().ok())
        .unwrap_or(""),
    content_length = %headers.get("content-length")
        .and_then(|v| v.to_str().ok())
        .unwrap_or("")))
]
pub async fn export_prometheus_remote_write_http(
    State(service): State<Service>,
    path_token: Option<Path<String>>,
    Query(query_params): Query<RemoteWriteQueryParams>,
    headers: HeaderMap,
    body: Bytes,
) -> Result<StatusCode, (StatusCode, Json<serde_json::Value>)> {
    let token = match extract_token(
        path_token.map(|Path(t)| t),
        &headers,
        query_params.token.as_deref(),
    ) {
        Some(token) => token,
        None => {
            error!("No token provided");
            return Err((
                StatusCode::UNAUTHORIZED,
                Json(json!({"error": "No token provided"})),
            ));
        }
    };

    service
        .authorizer
        .authorize_token(&token, Signal::Metrics)?;

    tracing::Span::current().record("token", &token);

    let encoding = RemoteWriteEncoding::from_headers(&headers);
    counter!(
        "capture_metrics_remote_write_requests",
        "encoding" => encoding.as_str()
    )
    .increment(1);

    let (write_request, uncompressed_bytes) =
        match decode_write_request(&body, encoding, service.max_request_body_size_bytes) {
            Ok(decoded) => decoded,
            Err(e) => {
                error!("Failed to decode remote-write request: {e}");
                return Err((
                    StatusCode::BAD_REQUEST,
                    Json(json!({ "error": format!("{e}") })),
                ));
            }
        };

    // 400, not 5xx: a batch that expands past the limit will never fit, so the
    // sender must drop it rather than retry it forever.
    let expanded = estimate_expanded_bytes(&write_request);
    let max_expanded = MAX_EXPANSION_FACTOR * service.max_request_body_size_bytes as u64;
    if expanded > max_expanded {
        error!(
            "Rejecting remote-write request expanding to ~{expanded} bytes (limit {max_expanded})"
        );
        return Err((
            StatusCode::BAD_REQUEST,
            Json(json!({
                "error": format!("request expands to too many samples (~{expanded} > {max_expanded} bytes); send smaller batches")
            })),
        ));
    }

    let (rows, timestamps_overridden) = write_request_to_kafka_rows(write_request);
    let row_count = rows.len();

    if let Err(e) = service
        .sink
        .write_metrics(&token, rows, uncompressed_bytes, timestamps_overridden)
        .await
    {
        error!("Failed to send remote-write metrics to Kafka: {e}");
        return Err((
            StatusCode::INTERNAL_SERVER_ERROR,
            Json(json!({"error": "Internal server error"})),
        ));
    }

    counter!("capture_metrics_records_produced", "source" => "remote_write")
        .increment(row_count as u64);
    debug!("Sent {row_count} remote-write data points to Kafka");
    Ok(StatusCode::NO_CONTENT)
}