Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(capture-apm-metrics): copy k8s labels to OTel resource attributes for opted-in tokens (#130330)

Daniel Visca6cbdf222d9f4
diff --git a/rust/capture-apm-metrics/README.md b/rust/capture-apm-metrics/README.md
index 55c2298a768d..b3f799a32fd3 100644
--- a/rust/capture-apm-metrics/README.md
+++ b/rust/capture-apm-metrics/README.md
@@ -11,6 +11,7 @@ This binary is the metrics half of [`capture-logs`](../capture-logs/README.md).
| Variable | Default | Description |
|----------|---------|-------------|
| REMOTE_WRITE_MAX_EXPANDED_BYTES | 167772160 | Largest estimated row set (160 MiB) one remote-write request may expand to. A larger request gets a 400. Separate from `MAX_REQUEST_BODY_SIZE_BYTES` |
+| REMOTE_WRITE_K8S_RESOURCE_ATTRIBUTE_TOKENS | (empty) | Comma-separated project tokens. Their remote-write series get `k8s.namespace.name`, `k8s.pod.name`, `k8s.node.name`, `k8s.container.name`, `k8s.deployment.name` and `k8s.daemonset.name` resource attributes copied from the matching Prometheus labels, the keys the OTel collector sets. The labels stay. This changes series fingerprints, so it is opt-in per project |
| REDIS_URL | (none) | Redis that shares the set of recently labelled metric series between pods. Unset means local cache only |
| METRICS_SERIES_LABEL_GATE_ENABLED | false | Strip label maps from repeat metric rows (see below). When false the gate only reports metrics |
| METRICS_SERIES_LABEL_INTERVAL_SECS | 1800 | Longest gap between two labelled rows of one metric series. Each series uses a window up to a quarter shorter, derived from its key |
diff --git a/rust/capture-apm-metrics/src/config.rs b/rust/capture-apm-metrics/src/config.rs
index 07e8343cd9eb..f7237f3dd7a0 100644
--- a/rust/capture-apm-metrics/src/config.rs
+++ b/rust/capture-apm-metrics/src/config.rs
@@ -17,6 +17,13 @@ pub struct Config {
#[envconfig(from = "REMOTE_WRITE_MAX_EXPANDED_BYTES", default = "167772160")] // 160 MiB
pub remote_write_max_expanded_bytes: u64,
+ /// Comma-separated project tokens whose remote-write series get the OTel
+ /// `k8s.*` resource attributes copied from their `namespace`, `pod`, `node`,
+ /// `container`, `deployment` and `daemonset` labels. Empty turns it off.
+ /// The copy changes series fingerprints, so it is opt-in per project.
+ #[envconfig(from = "REMOTE_WRITE_K8S_RESOURCE_ATTRIBUTE_TOKENS", default = "")]
+ pub remote_write_k8s_resource_attribute_tokens: String,
+
/// When false the gate still tracks series and reports metrics but leaves
/// every row labelled, so the switch can flip once the metrics2 read path
/// takes labels from the series rollup.
diff --git a/rust/capture-apm-metrics/src/k8s_resource_attributes.rs b/rust/capture-apm-metrics/src/k8s_resource_attributes.rs
new file mode 100644
index 000000000000..e23e6828fbae
--- /dev/null
+++ b/rust/capture-apm-metrics/src/k8s_resource_attributes.rs
@@ -0,0 +1,311 @@
+//! Copies the Kubernetes topology labels of remote-write series onto the OTel
+//! resource attributes that the OTLP path sets.
+//!
+//! The OTel collector's Kubernetes discovery writes `k8s.namespace.name`,
+//! `k8s.pod.name` and similar resource attributes. vmagent writes the same facts
+//! as the Prometheus labels `namespace`, `pod` and so on. A project that moves
+//! from the collector to vmagent keeps its insights working only when both paths
+//! carry the same keys, so this step copies each label onto its OTel key. The
+//! label stays in `attributes`, as `instance` does for `service.instance.id`.
+//!
+//! The copy changes the series fingerprint of existing series, so it runs only
+//! for the tokens in `REMOTE_WRITE_K8S_RESOURCE_ATTRIBUTE_TOKENS`.
+
+use std::collections::{HashMap, HashSet};
+
+use capture_logs::metric_record::{compute_series_fingerprint, KafkaMetricRow};
+use metrics::counter;
+use prometheus_rw_proto::prometheus::v1::WriteRequest;
+
+/// Prometheus label -> OTel semantic-convention resource attribute.
+const LABEL_TO_RESOURCE_ATTRIBUTE: [(&str, &str); 6] = [
+ ("namespace", "k8s.namespace.name"),
+ ("pod", "k8s.pod.name"),
+ ("node", "k8s.node.name"),
+ ("container", "k8s.container.name"),
+ ("deployment", "k8s.deployment.name"),
+ ("daemonset", "k8s.daemonset.name"),
+];
+
+fn resource_key_for(label: &str) -> Option<&'static str> {
+ LABEL_TO_RESOURCE_ATTRIBUTE
+ .iter()
+ .find(|(name, _)| *name == label)
+ .map(|(_, key)| *key)
+}
+
+/// The row builder adds this attribute per sample after it computes the
+/// fingerprint, so the recomputed fingerprint must leave it out too.
+const ORIGINAL_TIMESTAMP_ATTRIBUTE: &str = "$originalTimestamp";
+
+/// Attribute values are JSON-encoded strings, so an empty label is `""`.
+const EMPTY_JSON_STRING: &str = "\"\"";
+
+/// Length of `value` as the row builder stores it: a JSON string, quotes and
+/// escapes included. A control character escapes to six bytes, so the raw
+/// length would undercount a crafted label by up to six times.
+fn json_string_len(value: &str) -> usize {
+ let escaped: usize = value
+ .chars()
+ .map(|c| match c {
+ '"' | '\\' | '\u{08}' | '\u{0c}' | '\n' | '\r' | '\t' => 2,
+ c if (c as u32) < 0x20 => 6,
+ c => c.len_utf8(),
+ })
+ .sum();
+ escaped + 2
+}
+
+pub struct K8sResourceAttributes {
+ tokens: HashSet<String>,
+}
+
+impl K8sResourceAttributes {
+ /// Parses a comma-separated token list. Blank entries are ignored, so an
+ /// unset or empty variable turns the step off.
+ pub fn from_token_list(tokens: &str) -> Self {
+ Self {
+ tokens: tokens
+ .split(',')
+ .map(str::trim)
+ .filter(|token| !token.is_empty())
+ .map(str::to_string)
+ .collect(),
+ }
+ }
+
+ pub fn is_enabled_for(&self, token: &str) -> bool {
+ self.tokens.contains(token)
+ }
+
+ /// Bytes that `apply` adds to the rows of this request. The handler adds it
+ /// to the expansion estimate, because `apply` clones each copied label into
+ /// every row after that check, and a large label would otherwise slip past
+ /// the memory guard.
+ pub fn estimate_added_bytes(&self, token: &str, request: &WriteRequest) -> u64 {
+ if !self.is_enabled_for(token) {
+ return 0;
+ }
+ request
+ .timeseries
+ .iter()
+ .map(|series| {
+ let per_row: u64 = series
+ .labels
+ .iter()
+ .filter(|label| !label.value.is_empty())
+ .filter_map(|label| resource_key_for(&label.name).map(|key| (key, label)))
+ .map(|(key, label)| (key.len() + json_string_len(&label.value)) as u64)
+ .sum();
+ per_row * series.samples.len() as u64
+ })
+ .sum()
+ }
+
+ pub fn apply(&self, token: &str, rows: &mut [KafkaMetricRow]) {
+ if !self.is_enabled_for(token) {
+ return;
+ }
+
+ // Rows of one series share labels, so each fingerprint maps to one new value.
+ let mut recomputed: HashMap<i64, i64> = HashMap::new();
+ let mut copied = 0u64;
+
+ for row in rows.iter_mut() {
+ let mut changed = false;
+ for (label, resource_key) in LABEL_TO_RESOURCE_ATTRIBUTE {
+ // An explicit resource attribute wins over the label.
+ if row.resource_attributes.contains_key(resource_key) {
+ continue;
+ }
+ let Some(value) = row.attributes.get(label) else {
+ continue;
+ };
+ if value == EMPTY_JSON_STRING {
+ continue;
+ }
+ let key = resource_key.to_string();
+ row.resource_attributes.insert(key, value.clone());
+ changed = true;
+ }
+ if !changed {
+ continue;
+ }
+ copied += 1;
+
+ let fingerprint = match recomputed.get(&row.series_fingerprint) {
+ Some(fingerprint) => *fingerprint,
+ None => {
+ let mut series_attributes = row.attributes.clone();
+ series_attributes.remove(ORIGINAL_TIMESTAMP_ATTRIBUTE);
+ let fingerprint = compute_series_fingerprint(
+ &row.metric_name,
+ &row.metric_type,
+ &row.service_name,
+ &row.resource_attributes,
+ &series_attributes,
+ );
+ recomputed.insert(row.series_fingerprint, fingerprint);
+ fingerprint
+ }
+ };
+ row.series_fingerprint = fingerprint;
+ }
+
+ if copied > 0 {
+ counter!("capture_metrics_remote_write_k8s_resource_attributes_copied")
+ .increment(copied);
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use capture_logs::endpoints::prometheus::write_timeseries_to_kafka_rows;
+ use chrono::Utc;
+ use prometheus_rw_proto::prometheus::v1::{Label, Sample, TimeSeries};
+
+ const TOKEN: &str = "phc_internal";
+
+ const POD_SERIES: [(&str, &str); 6] = [
+ ("__name__", "capture_kafka_producer_queue_depth"),
+ ("job", "kubernetes-pods"),
+ ("instance", "10.0.0.1:8000"),
+ ("namespace", "capture"),
+ ("pod", "capture-abc"),
+ ("node", ""),
+ ];
+
+ fn label(name: &str, value: &str) -> Label {
+ Label {
+ name: name.to_string(),
+ value: value.to_string(),
+ }
+ }
+
+ fn sample(value: f64) -> Sample {
+ Sample {
+ value,
+ timestamp: Utc::now().timestamp_millis(),
+ }
+ }
+
+ /// Two samples of one series, built by the real remote-write row builder.
+ fn pod_rows() -> Vec<KafkaMetricRow> {
+ let series = TimeSeries {
+ labels: POD_SERIES.iter().map(|(n, v)| label(n, v)).collect(),
+ samples: vec![sample(1.0), sample(2.0)],
+ };
+ write_timeseries_to_kafka_rows(vec![series], &[]).0
+ }
+
+ fn resource<'a>(row: &'a KafkaMetricRow, key: &str) -> Option<&'a str> {
+ row.resource_attributes.get(key).map(String::as_str)
+ }
+
+ fn step(tokens: &str) -> K8sResourceAttributes {
+ K8sResourceAttributes::from_token_list(tokens)
+ }
+
+ #[test]
+ fn copies_labels_for_listed_tokens() {
+ let mut rows = pod_rows();
+ step(&format!(" other ,{TOKEN}")).apply(TOKEN, &mut rows);
+
+ for row in &rows {
+ let namespace = resource(row, "k8s.namespace.name");
+ assert_eq!(namespace, Some("\"capture\""));
+ let pod = resource(row, "k8s.pod.name");
+ assert_eq!(pod, Some("\"capture-abc\""));
+ // An empty label does not become an attribute.
+ assert_eq!(resource(row, "k8s.node.name"), None);
+ // The label stays queryable under its Prometheus name.
+ assert!(row.attributes.contains_key("namespace"));
+ }
+ }
+
+ #[test]
+ fn recomputes_one_fingerprint_per_series() {
+ let original = pod_rows();
+ let mut rows = pod_rows();
+ step(TOKEN).apply(TOKEN, &mut rows);
+
+ let row = &rows[0];
+ let expected = compute_series_fingerprint(
+ &row.metric_name,
+ &row.metric_type,
+ &row.service_name,
+ &row.resource_attributes,
+ &row.attributes,
+ );
+ assert_ne!(row.series_fingerprint, original[0].series_fingerprint);
+ assert_eq!(row.series_fingerprint, expected);
+ assert_eq!(rows[1].series_fingerprint, expected);
+ }
+
+ #[test]
+ fn leaves_other_tokens_unchanged() {
+ let original = pod_rows();
+ let mut rows = pod_rows();
+ step(TOKEN).apply("phc_customer", &mut rows);
+ step("").apply(TOKEN, &mut rows);
+
+ let (row, before) = (&rows[0], &original[0]);
+ assert_eq!(row.resource_attributes, before.resource_attributes);
+ assert_eq!(row.series_fingerprint, before.series_fingerprint);
+ }
+
+ #[test]
+ fn estimates_the_bytes_the_copy_adds() {
+ let series = TimeSeries {
+ labels: POD_SERIES.iter().map(|(n, v)| label(n, v)).collect(),
+ samples: vec![sample(1.0), sample(2.0), sample(3.0)],
+ };
+ let request = WriteRequest {
+ timeseries: vec![series],
+ metadata: vec![],
+ };
+
+ // k8s.namespace.name + "capture" and k8s.pod.name + "capture-abc",
+ // each value with its JSON quotes. The empty node label adds nothing.
+ let per_row = (18 + 7 + 2) + (12 + 11 + 2);
+ let added = step(TOKEN).estimate_added_bytes(TOKEN, &request);
+ assert_eq!(added, 3 * per_row);
+ let other = step(TOKEN).estimate_added_bytes("phc_customer", &request);
+ assert_eq!(other, 0);
+ }
+
+ #[test]
+ fn json_string_len_matches_the_encoded_value() {
+ for value in [
+ "capture-abc",
+ "",
+ "a\"b\\c",
+ "\n\t\u{08}\u{0c}\r",
+ "\u{01}\u{1f}",
+ "pod-é-\u{7f}",
+ ] {
+ assert_eq!(
+ json_string_len(value),
+ serde_json::json!(value).to_string().len(),
+ "{value:?}"
+ );
+ }
+ }
+
+ #[test]
+ fn keeps_an_existing_resource_attribute() {
+ let mut rows = pod_rows();
+ for row in rows.iter_mut() {
+ let key = "k8s.namespace.name".to_string();
+ row.resource_attributes.insert(key, "\"explicit\"".into());
+ }
+ step(TOKEN).apply(TOKEN, &mut rows);
+
+ let namespace = resource(&rows[0], "k8s.namespace.name");
+ assert_eq!(namespace, Some("\"explicit\""));
+ let pod = resource(&rows[0], "k8s.pod.name");
+ assert_eq!(pod, Some("\"capture-abc\""));
+ }
+}
diff --git a/rust/capture-apm-metrics/src/lib.rs b/rust/capture-apm-metrics/src/lib.rs
index 870b814fd03a..7be4fcb7f8c2 100644
--- a/rust/capture-apm-metrics/src/lib.rs
+++ b/rust/capture-apm-metrics/src/lib.rs
@@ -1,5 +1,6 @@
pub mod config;
pub mod histogram_assembly;
+pub mod k8s_resource_attributes;
pub mod management;
pub mod prometheus;
pub mod series_label_gate;
diff --git a/rust/capture-apm-metrics/src/main.rs b/rust/capture-apm-metrics/src/main.rs
index 41607a6de32e..ee1e7d6858b0 100644
--- a/rust/capture-apm-metrics/src/main.rs
+++ b/rust/capture-apm-metrics/src/main.rs
@@ -9,6 +9,7 @@ use std::time::Duration;
use axum::{extract::DefaultBodyLimit, http::Method, routing::get, routing::post, Router};
use capture::metrics_middleware::track_metrics;
use capture_apm_metrics::config::Config;
+use capture_apm_metrics::k8s_resource_attributes::K8sResourceAttributes;
use capture_apm_metrics::management;
use capture_apm_metrics::prometheus;
use capture_apm_metrics::series_label_gate::{spawn_redis_writer, CacheLimits, SeriesLabelGate};
@@ -248,6 +249,9 @@ async fn run(
config.base.max_request_body_size_bytes,
config.remote_write_max_expanded_bytes,
series_label_gate,
+ Arc::new(K8sResourceAttributes::from_token_list(
+ &config.remote_write_k8s_resource_attribute_tokens,
+ )),
);
let http_bind = format!("{}:{}", config.base.host, config.base.port);
info!("Listening on {}", http_bind);
diff --git a/rust/capture-apm-metrics/src/prometheus.rs b/rust/capture-apm-metrics/src/prometheus.rs
index 4360cb0289a9..5c53c6476610 100644
--- a/rust/capture-apm-metrics/src/prometheus.rs
+++ b/rust/capture-apm-metrics/src/prometheus.rs
@@ -126,7 +126,11 @@ pub async fn export_prometheus_remote_write_http(
// expansion-bomb payloads and oversized blocks exceed it.
// 400, not 5xx: a batch that expands past the limit will never fit, so the
// sender must drop it rather than retry it forever.
- let expanded = estimate_expanded_bytes(&write_request);
+ // The k8s resource attribute copy clones labels into every row after this
+ // check, so its bytes count toward the same limit.
+ let k8s_resource_attributes = &service.k8s_resource_attributes;
+ let added = k8s_resource_attributes.estimate_added_bytes(&token, &write_request);
+ let expanded = estimate_expanded_bytes(&write_request).saturating_add(added);
let max_expanded = service.remote_write_max_expanded_bytes;
if expanded > max_expanded {
error!(
@@ -140,8 +144,11 @@ pub async fn export_prometheus_remote_write_http(
));
}
- let (rows, timestamps_overridden) =
+ let (mut rows, timestamps_overridden) =
write_timeseries_to_kafka_rows(write_request.timeseries, &write_request.metadata);
+ // Before the histogram fold, so folded histograms carry the attributes and
+ // fingerprint them.
+ service.k8s_resource_attributes.apply(&token, &mut rows);
let mut rows = fold_classic_histograms(rows, &write_request.metadata);
service.series_label_gate.apply(&token, &mut rows);
let row_count = rows.len();
diff --git a/rust/capture-apm-metrics/src/service.rs b/rust/capture-apm-metrics/src/service.rs
index 750facf9ec06..ac931f55ba2e 100644
--- a/rust/capture-apm-metrics/src/service.rs
+++ b/rust/capture-apm-metrics/src/service.rs
@@ -25,6 +25,7 @@ use serde::Deserialize;
use serde_json::json;
use tracing::{debug, error, instrument};
+use crate::k8s_resource_attributes::K8sResourceAttributes;
use crate::series_label_gate::SeriesLabelGate;
#[derive(Clone)]
@@ -34,6 +35,7 @@ pub struct MetricsService {
pub(crate) max_request_body_size_bytes: usize,
pub(crate) remote_write_max_expanded_bytes: u64,
pub(crate) series_label_gate: Arc<SeriesLabelGate>,
+ pub(crate) k8s_resource_attributes: Arc<K8sResourceAttributes>,
}
#[derive(Deserialize)]
@@ -48,6 +50,7 @@ impl MetricsService {
max_request_body_size_bytes: usize,
remote_write_max_expanded_bytes: u64,
series_label_gate: Arc<SeriesLabelGate>,
+ k8s_resource_attributes: Arc<K8sResourceAttributes>,
) -> Self {
Self {
sink: kafka_sink,
@@ -55,6 +58,7 @@ impl MetricsService {
max_request_body_size_bytes,
remote_write_max_expanded_bytes,
series_label_gate,
+ k8s_resource_attributes,
}
}
}