:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(personhog): count etcd client errors and align etcd histogram ladders (#125022)
Zach Brownf3aaa5ac1d3d
diff --git a/rust/common/assignment-coordination/src/failover.rs b/rust/common/assignment-coordination/src/failover.rsindex 33058c0b74ba..b1a0915024ab 100644--- a/rust/common/assignment-coordination/src/failover.rs+++ b/rust/common/assignment-coordination/src/failover.rs@@ -33,6 +33,26 @@ pub fn is_member_level(err: &etcd_client::Error) -> bool {}}+/// `connection` for a failure under the call, otherwise etcd's gRPC code.+pub(crate) fn error_kind(err: &etcd_client::Error) -> &'static str {+ match err {+ etcd_client::Error::GRpcStatus(status) if raised_by_connection(status) => "connection",+ etcd_client::Error::GRpcStatus(status) => match status.code() {+ Code::Unavailable => "unavailable",+ Code::DeadlineExceeded => "deadline_exceeded",+ Code::ResourceExhausted => "resource_exhausted",+ Code::OutOfRange => "out_of_range",+ Code::InvalidArgument => "invalid_argument",+ Code::FailedPrecondition => "failed_precondition",+ Code::NotFound => "not_found",+ Code::Cancelled => "cancelled",+ _ => "other_status",+ },+ etcd_client::Error::TransportError(_) | etcd_client::Error::IoError(_) => "connection",+ _ => "other",+ }+}+/// The Unavailable error etcd answers a require-leader request with when the/// member has no leader, for the same condition reported another way.pub fn no_leader_error(message: impl Into<String>) -> etcd_client::Error {@@ -352,6 +372,28 @@ mod tests {etcd_client::Error::GRpcStatus(tonic::Status::new(code, "test"))}+ #[test]+ fn error_kinds_separate_broken_connections_from_etcd_answers() {+ let broken_stream =+ tonic::Status::from_error(Box::new(h2::Error::from(h2::Reason::STREAM_CLOSED)));+ assert_eq!(+ error_kind(&etcd_client::Error::GRpcStatus(broken_stream)),+ "connection"+ );+ assert_eq!(+ error_kind(&etcd_client::Error::IoError(std::io::Error::from(+ std::io::ErrorKind::ConnectionReset+ ))),+ "connection"+ );+ assert_eq!(error_kind(&status(Code::Unavailable)), "unavailable");+ assert_eq!(+ error_kind(&status(Code::ResourceExhausted)),+ "resource_exhausted"+ );+ assert_eq!(error_kind(&status(Code::PermissionDenied)), "other_status");+ }+#[test]fn member_level_errors_are_the_ones_a_different_member_could_avoid() {assert!(is_member_level(&status(Code::Unavailable)));diff --git a/rust/common/assignment-coordination/src/store.rs b/rust/common/assignment-coordination/src/store.rsindex 87a110fd18cf..61da8369be68 100644--- a/rust/common/assignment-coordination/src/store.rs+++ b/rust/common/assignment-coordination/src/store.rs@@ -9,7 +9,7 @@ use serde::de::DeserializeOwned;use serde::Serialize;use crate::error::{Error, Result};-use crate::failover::{is_member_level, MemberFailover};+use crate::failover::{error_kind, is_member_level, MemberFailover};#[derive(Debug, Clone)]pub struct StoreConfig {@@ -155,8 +155,17 @@ impl EtcdStore {fn observe<T>(&self,+ op: &'static str,result: std::result::Result<T, etcd_client::Error>,) -> std::result::Result<T, etcd_client::Error> {+ if let Err(err) = &result {+ metrics::counter!(+ "assignment_coordination_etcd_errors_total",+ "op" => op,+ "kind" => error_kind(err),+ )+ .increment(1);+ }self.observe_member_outcome(result.as_ref().map(|_| ()));result}@@ -173,7 +182,7 @@ impl EtcdStore {pub async fn get_raw(&self, key: &str) -> Result<Option<Vec<u8>>> {let _t = OpTimer::new("get_raw");- let resp = self.observe(self.client.clone().get(key, None).await)?;+ let resp = self.observe("get_raw", self.client.clone().get(key, None).await)?;record_payload_bytes("get_raw", kvs_bytes(resp.kvs()));Ok(resp.kvs().first().map(|kv| kv.value().to_vec()))}@@ -182,7 +191,7 @@ impl EtcdStore {let _t = OpTimer::new("put_raw");let value = value.into();record_payload_bytes("put_raw", key.len() + value.len());- self.observe(self.client.clone().put(key, value, None).await)?;+ self.observe("put_raw", self.client.clone().put(key, value, None).await)?;Ok(())}@@ -190,7 +199,7 @@ impl EtcdStore {pub async fn get<T: DeserializeOwned>(&self, key: &str) -> Result<Option<T>> {let _t = OpTimer::new("get");- let resp = self.observe(self.client.clone().get(key, None).await)?;+ let resp = self.observe("get", self.client.clone().get(key, None).await)?;record_payload_bytes("get", kvs_bytes(resp.kvs()));match resp.kvs().first() {Some(kv) => Ok(Some(serde_json::from_slice(kv.value())?)),@@ -200,7 +209,7 @@ impl EtcdStore {pub async fn get_versioned<T: DeserializeOwned>(&self, key: &str) -> Result<Option<(T, i64)>> {let _t = OpTimer::new("get_versioned");- let resp = self.observe(self.client.clone().get(key, None).await)?;+ let resp = self.observe("get_versioned", self.client.clone().get(key, None).await)?;record_payload_bytes("get_versioned", kvs_bytes(resp.kvs()));match resp.kvs().first() {Some(kv) => {@@ -222,7 +231,10 @@ impl EtcdStore {key: &str,) -> Result<Option<(T, i64)>> {let _t = OpTimer::new("get_with_mod_revision");- let resp = self.observe(self.client.clone().get(key, None).await)?;+ let resp = self.observe(+ "get_with_mod_revision",+ self.client.clone().get(key, None).await,+ )?;record_payload_bytes("get_with_mod_revision", kvs_bytes(resp.kvs()));match resp.kvs().first() {Some(kv) => {@@ -243,7 +255,10 @@ impl EtcdStore {key: &str,) -> Result<(Option<T>, i64)> {let _t = OpTimer::new("get_with_revision");- let resp = self.observe(self.client.clone().get(key, None).await)?;+ let resp = self.observe(+ "get_with_revision",+ self.client.clone().get(key, None).await,+ )?;record_payload_bytes("get_with_revision", kvs_bytes(resp.kvs()));let revision = resp.header().map(|h| h.revision()).unwrap_or(0);match resp.kvs().first() {@@ -263,7 +278,10 @@ impl EtcdStore {pub async fn list_keys(&self, prefix: &str) -> Result<Vec<String>> {let _t = OpTimer::new("list_keys");let options = GetOptions::new().with_prefix().with_keys_only();- let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;+ let resp = self.observe(+ "list_keys",+ self.client.clone().get(prefix, Some(options)).await,+ )?;record_payload_bytes("list_keys", kvs_bytes(resp.kvs()));Ok(resp.kvs()@@ -277,7 +295,10 @@ impl EtcdStore {pub async fn list_keys_with_mod_revisions(&self, prefix: &str) -> Result<Vec<(String, i64)>> {let _t = OpTimer::new("list_keys_with_mod_revisions");let options = GetOptions::new().with_prefix().with_keys_only();- let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;+ let resp = self.observe(+ "list_keys_with_mod_revisions",+ self.client.clone().get(prefix, Some(options)).await,+ )?;record_payload_bytes("list_keys_with_mod_revisions", kvs_bytes(resp.kvs()));Ok(resp.kvs()@@ -301,7 +322,10 @@ impl EtcdStore {) -> Result<(Vec<T>, i64)> {let _t = OpTimer::new("list_with_revision");let options = GetOptions::new().with_prefix();- let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;+ let resp = self.observe(+ "list_with_revision",+ self.client.clone().get(prefix, Some(options)).await,+ )?;record_payload_bytes("list_with_revision", kvs_bytes(resp.kvs()));let revision = resp.header().map(|h| h.revision()).unwrap_or(0);let items = resp@@ -321,7 +345,10 @@ impl EtcdStore {) -> Result<Vec<(T, i64)>> {let _t = OpTimer::new("list_with_mod_revisions");let options = GetOptions::new().with_prefix();- let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;+ let resp = self.observe(+ "list_with_mod_revisions",+ self.client.clone().get(prefix, Some(options)).await,+ )?;record_payload_bytes("list_with_mod_revisions", kvs_bytes(resp.kvs()));resp.kvs().iter()@@ -352,27 +379,33 @@ impl EtcdStore {let value = serde_json::to_string(value)?;record_payload_bytes("put", key.len() + value.len());let options = lease_id.map(|id| PutOptions::new().with_lease(id));- self.observe(self.client.clone().put(key, value, options).await)?;+ self.observe("put", self.client.clone().put(key, value, options).await)?;Ok(())}pub async fn delete(&self, key: &str) -> Result<()> {let _t = OpTimer::new("delete");- self.observe(self.client.clone().delete(key, None).await)?;+ self.observe("delete", self.client.clone().delete(key, None).await)?;Ok(())}pub async fn delete_prefix(&self, prefix: &str) -> Result<()> {let _t = OpTimer::new("delete_prefix");let options = DeleteOptions::new().with_prefix();- self.observe(self.client.clone().delete(prefix, Some(options)).await)?;+ self.observe(+ "delete_prefix",+ self.client.clone().delete(prefix, Some(options)).await,+ )?;Ok(())}pub async fn watch(&self, prefix: &str) -> Result<WatchStream> {let _t = OpTimer::new("watch");let options = WatchOptions::new().with_prefix();- let stream = self.observe(self.client.clone().watch(prefix, Some(options)).await)?;+ let stream = self.observe(+ "watch",+ self.client.clone().watch(prefix, Some(options)).await,+ )?;Ok(stream)}@@ -388,7 +421,10 @@ impl EtcdStore {let options = WatchOptions::new().with_prefix().with_start_revision(start_revision);- let stream = self.observe(self.client.clone().watch(prefix, Some(options)).await)?;+ let stream = self.observe(+ "watch_from",+ self.client.clone().watch(prefix, Some(options)).await,+ )?;Ok(stream)}@@ -398,7 +434,10 @@ impl EtcdStore {pub async fn watch_key_from(&self, key: &str, start_revision: i64) -> Result<WatchStream> {let _t = OpTimer::new("watch_key_from");let options = WatchOptions::new().with_start_revision(start_revision);- let stream = self.observe(self.client.clone().watch(key, Some(options)).await)?;+ let stream = self.observe(+ "watch_key_from",+ self.client.clone().watch(key, Some(options)).await,+ )?;Ok(stream)}@@ -413,7 +452,7 @@ impl EtcdStore {/// request side directly instead.pub async fn txn(&self, txn: Txn) -> Result<TxnResponse> {let _t = OpTimer::new("txn");- Ok(self.observe(self.client.clone().txn(txn).await)?)+ Ok(self.observe("txn", self.client.clone().txn(txn).await)?)}/// Atomically create `key` bound to `lease_id`, only if it does not@@ -429,7 +468,7 @@ impl EtcdStore {value,Some(PutOptions::new().with_lease(lease_id)),)]);- let resp = self.observe(self.client.clone().txn(txn).await)?;+ let resp = self.observe("put_if_absent", self.client.clone().txn(txn).await)?;Ok(resp.succeeded())}@@ -437,7 +476,10 @@ impl EtcdStore {pub async fn grant_lease(&self, ttl: i64) -> Result<i64> {let _t = OpTimer::new("grant_lease");- let resp = self.observe(self.client.clone().lease_grant(ttl, None).await)?;+ let resp = self.observe(+ "grant_lease",+ self.client.clone().lease_grant(ttl, None).await,+ )?;Ok(resp.id())}@@ -446,14 +488,19 @@ impl EtcdStore {lease_id: i64,) -> Result<(etcd_client::LeaseKeeper, etcd_client::LeaseKeepAliveStream)> {let _t = OpTimer::new("keep_alive");- let (keeper, stream) =- self.observe(self.client.clone().lease_keep_alive(lease_id).await)?;+ let (keeper, stream) = self.observe(+ "keep_alive",+ self.client.clone().lease_keep_alive(lease_id).await,+ )?;Ok((keeper, stream))}pub async fn revoke_lease(&self, lease_id: i64) -> Result<()> {let _t = OpTimer::new("revoke_lease");- self.observe(self.client.clone().lease_revoke(lease_id).await)?;+ self.observe(+ "revoke_lease",+ self.client.clone().lease_revoke(lease_id).await,+ )?;Ok(())}diff --git a/rust/common/metrics/src/lib.rs b/rust/common/metrics/src/lib.rsindex d17043e0edf5..d629c1da9fa6 100644--- a/rust/common/metrics/src/lib.rs+++ b/rust/common/metrics/src/lib.rs@@ -112,6 +112,11 @@ pub const ETCD_PAYLOAD_SIZE_BUCKETS_BYTES: &[f64] = &[1024.0, 8192.0, 65536.0, 262144.0, 524288.0, 1048576.0, 1572864.0, 2097152.0, 4194304.0,];+/// Pinned like the payload ladder, since every `EtcdStore` user emits the metric.+pub const ETCD_OP_LATENCY_BUCKETS_MS: &[f64] = &[+ 1.0, 5.0, 10.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2000.0, 5000.0, 10000.0,+];+fn build_prometheus_builder(product: Option<String>) -> PrometheusBuilder {const BUCKETS: &[f64] = &[1.0, 5.0, 10.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2000.0, 5000.0, 10000.0,@@ -124,6 +129,11 @@ fn build_prometheus_builder(product: Option<String>) -> PrometheusBuilder {Matcher::Full("assignment_coordination_etcd_payload_bytes".into()),ETCD_PAYLOAD_SIZE_BUCKETS_BYTES,)+ .unwrap()+ .set_buckets_for_metric(+ Matcher::Full("assignment_coordination_etcd_op_ms".into()),+ ETCD_OP_LATENCY_BUCKETS_MS,+ ).unwrap();if let Some(product) = product {builder = builder.add_global_label("product", product);diff --git a/rust/personhog-identity/src/main.rs b/rust/personhog-identity/src/main.rsindex 23555f3d4bba..8cbe5c7fcec9 100644--- a/rust/personhog-identity/src/main.rs+++ b/rust/personhog-identity/src/main.rs@@ -4,6 +4,7 @@ use std::time::Duration;use assignment_coordination::store::{EtcdStore, StoreConfig};use axum::{routing::get, Router};use common_database::get_pool_with_config;+use common_metrics::{ETCD_OP_LATENCY_BUCKETS_MS, ETCD_PAYLOAD_SIZE_BUCKETS_BYTES};use envconfig::Envconfig;use lifecycle::{ComponentOptions, Manager};use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};@@ -232,6 +233,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.unwrap().set_buckets_for_metric(Matcher::Suffix("_per_call".into()), PER_CALL_BUCKETS).unwrap()+ .set_buckets_for_metric(+ Matcher::Full("assignment_coordination_etcd_op_ms".into()),+ ETCD_OP_LATENCY_BUCKETS_MS,+ )+ .unwrap()+ .set_buckets_for_metric(+ Matcher::Full("assignment_coordination_etcd_payload_bytes".into()),+ ETCD_PAYLOAD_SIZE_BUCKETS_BYTES,+ )+ .unwrap().install_recorder().expect("Failed to install metrics recorder");{