Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(personhog): count etcd client errors and align etcd histogram ladders (#125022)

Zach Brownf3aaa5ac1d3d
diff --git a/rust/common/assignment-coordination/src/failover.rs b/rust/common/assignment-coordination/src/failover.rs
index 33058c0b74ba..b1a0915024ab 100644
--- a/rust/common/assignment-coordination/src/failover.rs
+++ b/rust/common/assignment-coordination/src/failover.rs
@@ -33,6 +33,26 @@ pub fn is_member_level(err: &etcd_client::Error) -> bool {
}
}
+/// `connection` for a failure under the call, otherwise etcd's gRPC code.
+pub(crate) fn error_kind(err: &etcd_client::Error) -> &'static str {
+ match err {
+ etcd_client::Error::GRpcStatus(status) if raised_by_connection(status) => "connection",
+ etcd_client::Error::GRpcStatus(status) => match status.code() {
+ Code::Unavailable => "unavailable",
+ Code::DeadlineExceeded => "deadline_exceeded",
+ Code::ResourceExhausted => "resource_exhausted",
+ Code::OutOfRange => "out_of_range",
+ Code::InvalidArgument => "invalid_argument",
+ Code::FailedPrecondition => "failed_precondition",
+ Code::NotFound => "not_found",
+ Code::Cancelled => "cancelled",
+ _ => "other_status",
+ },
+ etcd_client::Error::TransportError(_) | etcd_client::Error::IoError(_) => "connection",
+ _ => "other",
+ }
+}
+
/// The Unavailable error etcd answers a require-leader request with when the
/// member has no leader, for the same condition reported another way.
pub fn no_leader_error(message: impl Into<String>) -> etcd_client::Error {
@@ -352,6 +372,28 @@ mod tests {
etcd_client::Error::GRpcStatus(tonic::Status::new(code, "test"))
}
+ #[test]
+ fn error_kinds_separate_broken_connections_from_etcd_answers() {
+ let broken_stream =
+ tonic::Status::from_error(Box::new(h2::Error::from(h2::Reason::STREAM_CLOSED)));
+ assert_eq!(
+ error_kind(&etcd_client::Error::GRpcStatus(broken_stream)),
+ "connection"
+ );
+ assert_eq!(
+ error_kind(&etcd_client::Error::IoError(std::io::Error::from(
+ std::io::ErrorKind::ConnectionReset
+ ))),
+ "connection"
+ );
+ assert_eq!(error_kind(&status(Code::Unavailable)), "unavailable");
+ assert_eq!(
+ error_kind(&status(Code::ResourceExhausted)),
+ "resource_exhausted"
+ );
+ assert_eq!(error_kind(&status(Code::PermissionDenied)), "other_status");
+ }
+
#[test]
fn member_level_errors_are_the_ones_a_different_member_could_avoid() {
assert!(is_member_level(&status(Code::Unavailable)));
diff --git a/rust/common/assignment-coordination/src/store.rs b/rust/common/assignment-coordination/src/store.rs
index 87a110fd18cf..61da8369be68 100644
--- a/rust/common/assignment-coordination/src/store.rs
+++ b/rust/common/assignment-coordination/src/store.rs
@@ -9,7 +9,7 @@ use serde::de::DeserializeOwned;
use serde::Serialize;
use crate::error::{Error, Result};
-use crate::failover::{is_member_level, MemberFailover};
+use crate::failover::{error_kind, is_member_level, MemberFailover};
#[derive(Debug, Clone)]
pub struct StoreConfig {
@@ -155,8 +155,17 @@ impl EtcdStore {
fn observe<T>(
&self,
+ op: &'static str,
result: std::result::Result<T, etcd_client::Error>,
) -> std::result::Result<T, etcd_client::Error> {
+ if let Err(err) = &result {
+ metrics::counter!(
+ "assignment_coordination_etcd_errors_total",
+ "op" => op,
+ "kind" => error_kind(err),
+ )
+ .increment(1);
+ }
self.observe_member_outcome(result.as_ref().map(|_| ()));
result
}
@@ -173,7 +182,7 @@ impl EtcdStore {
pub async fn get_raw(&self, key: &str) -> Result<Option<Vec<u8>>> {
let _t = OpTimer::new("get_raw");
- let resp = self.observe(self.client.clone().get(key, None).await)?;
+ let resp = self.observe("get_raw", self.client.clone().get(key, None).await)?;
record_payload_bytes("get_raw", kvs_bytes(resp.kvs()));
Ok(resp.kvs().first().map(|kv| kv.value().to_vec()))
}
@@ -182,7 +191,7 @@ impl EtcdStore {
let _t = OpTimer::new("put_raw");
let value = value.into();
record_payload_bytes("put_raw", key.len() + value.len());
- self.observe(self.client.clone().put(key, value, None).await)?;
+ self.observe("put_raw", self.client.clone().put(key, value, None).await)?;
Ok(())
}
@@ -190,7 +199,7 @@ impl EtcdStore {
pub async fn get<T: DeserializeOwned>(&self, key: &str) -> Result<Option<T>> {
let _t = OpTimer::new("get");
- let resp = self.observe(self.client.clone().get(key, None).await)?;
+ let resp = self.observe("get", self.client.clone().get(key, None).await)?;
record_payload_bytes("get", kvs_bytes(resp.kvs()));
match resp.kvs().first() {
Some(kv) => Ok(Some(serde_json::from_slice(kv.value())?)),
@@ -200,7 +209,7 @@ impl EtcdStore {
pub async fn get_versioned<T: DeserializeOwned>(&self, key: &str) -> Result<Option<(T, i64)>> {
let _t = OpTimer::new("get_versioned");
- let resp = self.observe(self.client.clone().get(key, None).await)?;
+ let resp = self.observe("get_versioned", self.client.clone().get(key, None).await)?;
record_payload_bytes("get_versioned", kvs_bytes(resp.kvs()));
match resp.kvs().first() {
Some(kv) => {
@@ -222,7 +231,10 @@ impl EtcdStore {
key: &str,
) -> Result<Option<(T, i64)>> {
let _t = OpTimer::new("get_with_mod_revision");
- let resp = self.observe(self.client.clone().get(key, None).await)?;
+ let resp = self.observe(
+ "get_with_mod_revision",
+ self.client.clone().get(key, None).await,
+ )?;
record_payload_bytes("get_with_mod_revision", kvs_bytes(resp.kvs()));
match resp.kvs().first() {
Some(kv) => {
@@ -243,7 +255,10 @@ impl EtcdStore {
key: &str,
) -> Result<(Option<T>, i64)> {
let _t = OpTimer::new("get_with_revision");
- let resp = self.observe(self.client.clone().get(key, None).await)?;
+ let resp = self.observe(
+ "get_with_revision",
+ self.client.clone().get(key, None).await,
+ )?;
record_payload_bytes("get_with_revision", kvs_bytes(resp.kvs()));
let revision = resp.header().map(|h| h.revision()).unwrap_or(0);
match resp.kvs().first() {
@@ -263,7 +278,10 @@ impl EtcdStore {
pub async fn list_keys(&self, prefix: &str) -> Result<Vec<String>> {
let _t = OpTimer::new("list_keys");
let options = GetOptions::new().with_prefix().with_keys_only();
- let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;
+ let resp = self.observe(
+ "list_keys",
+ self.client.clone().get(prefix, Some(options)).await,
+ )?;
record_payload_bytes("list_keys", kvs_bytes(resp.kvs()));
Ok(resp
.kvs()
@@ -277,7 +295,10 @@ impl EtcdStore {
pub async fn list_keys_with_mod_revisions(&self, prefix: &str) -> Result<Vec<(String, i64)>> {
let _t = OpTimer::new("list_keys_with_mod_revisions");
let options = GetOptions::new().with_prefix().with_keys_only();
- let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;
+ let resp = self.observe(
+ "list_keys_with_mod_revisions",
+ self.client.clone().get(prefix, Some(options)).await,
+ )?;
record_payload_bytes("list_keys_with_mod_revisions", kvs_bytes(resp.kvs()));
Ok(resp
.kvs()
@@ -301,7 +322,10 @@ impl EtcdStore {
) -> Result<(Vec<T>, i64)> {
let _t = OpTimer::new("list_with_revision");
let options = GetOptions::new().with_prefix();
- let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;
+ let resp = self.observe(
+ "list_with_revision",
+ self.client.clone().get(prefix, Some(options)).await,
+ )?;
record_payload_bytes("list_with_revision", kvs_bytes(resp.kvs()));
let revision = resp.header().map(|h| h.revision()).unwrap_or(0);
let items = resp
@@ -321,7 +345,10 @@ impl EtcdStore {
) -> Result<Vec<(T, i64)>> {
let _t = OpTimer::new("list_with_mod_revisions");
let options = GetOptions::new().with_prefix();
- let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;
+ let resp = self.observe(
+ "list_with_mod_revisions",
+ self.client.clone().get(prefix, Some(options)).await,
+ )?;
record_payload_bytes("list_with_mod_revisions", kvs_bytes(resp.kvs()));
resp.kvs()
.iter()
@@ -352,27 +379,33 @@ impl EtcdStore {
let value = serde_json::to_string(value)?;
record_payload_bytes("put", key.len() + value.len());
let options = lease_id.map(|id| PutOptions::new().with_lease(id));
- self.observe(self.client.clone().put(key, value, options).await)?;
+ self.observe("put", self.client.clone().put(key, value, options).await)?;
Ok(())
}
pub async fn delete(&self, key: &str) -> Result<()> {
let _t = OpTimer::new("delete");
- self.observe(self.client.clone().delete(key, None).await)?;
+ self.observe("delete", self.client.clone().delete(key, None).await)?;
Ok(())
}
pub async fn delete_prefix(&self, prefix: &str) -> Result<()> {
let _t = OpTimer::new("delete_prefix");
let options = DeleteOptions::new().with_prefix();
- self.observe(self.client.clone().delete(prefix, Some(options)).await)?;
+ self.observe(
+ "delete_prefix",
+ self.client.clone().delete(prefix, Some(options)).await,
+ )?;
Ok(())
}
pub async fn watch(&self, prefix: &str) -> Result<WatchStream> {
let _t = OpTimer::new("watch");
let options = WatchOptions::new().with_prefix();
- let stream = self.observe(self.client.clone().watch(prefix, Some(options)).await)?;
+ let stream = self.observe(
+ "watch",
+ self.client.clone().watch(prefix, Some(options)).await,
+ )?;
Ok(stream)
}
@@ -388,7 +421,10 @@ impl EtcdStore {
let options = WatchOptions::new()
.with_prefix()
.with_start_revision(start_revision);
- let stream = self.observe(self.client.clone().watch(prefix, Some(options)).await)?;
+ let stream = self.observe(
+ "watch_from",
+ self.client.clone().watch(prefix, Some(options)).await,
+ )?;
Ok(stream)
}
@@ -398,7 +434,10 @@ impl EtcdStore {
pub async fn watch_key_from(&self, key: &str, start_revision: i64) -> Result<WatchStream> {
let _t = OpTimer::new("watch_key_from");
let options = WatchOptions::new().with_start_revision(start_revision);
- let stream = self.observe(self.client.clone().watch(key, Some(options)).await)?;
+ let stream = self.observe(
+ "watch_key_from",
+ self.client.clone().watch(key, Some(options)).await,
+ )?;
Ok(stream)
}
@@ -413,7 +452,7 @@ impl EtcdStore {
/// request side directly instead.
pub async fn txn(&self, txn: Txn) -> Result<TxnResponse> {
let _t = OpTimer::new("txn");
- Ok(self.observe(self.client.clone().txn(txn).await)?)
+ Ok(self.observe("txn", self.client.clone().txn(txn).await)?)
}
/// Atomically create `key` bound to `lease_id`, only if it does not
@@ -429,7 +468,7 @@ impl EtcdStore {
value,
Some(PutOptions::new().with_lease(lease_id)),
)]);
- let resp = self.observe(self.client.clone().txn(txn).await)?;
+ let resp = self.observe("put_if_absent", self.client.clone().txn(txn).await)?;
Ok(resp.succeeded())
}
@@ -437,7 +476,10 @@ impl EtcdStore {
pub async fn grant_lease(&self, ttl: i64) -> Result<i64> {
let _t = OpTimer::new("grant_lease");
- let resp = self.observe(self.client.clone().lease_grant(ttl, None).await)?;
+ let resp = self.observe(
+ "grant_lease",
+ self.client.clone().lease_grant(ttl, None).await,
+ )?;
Ok(resp.id())
}
@@ -446,14 +488,19 @@ impl EtcdStore {
lease_id: i64,
) -> Result<(etcd_client::LeaseKeeper, etcd_client::LeaseKeepAliveStream)> {
let _t = OpTimer::new("keep_alive");
- let (keeper, stream) =
- self.observe(self.client.clone().lease_keep_alive(lease_id).await)?;
+ let (keeper, stream) = self.observe(
+ "keep_alive",
+ self.client.clone().lease_keep_alive(lease_id).await,
+ )?;
Ok((keeper, stream))
}
pub async fn revoke_lease(&self, lease_id: i64) -> Result<()> {
let _t = OpTimer::new("revoke_lease");
- self.observe(self.client.clone().lease_revoke(lease_id).await)?;
+ self.observe(
+ "revoke_lease",
+ self.client.clone().lease_revoke(lease_id).await,
+ )?;
Ok(())
}
diff --git a/rust/common/metrics/src/lib.rs b/rust/common/metrics/src/lib.rs
index d17043e0edf5..d629c1da9fa6 100644
--- a/rust/common/metrics/src/lib.rs
+++ b/rust/common/metrics/src/lib.rs
@@ -112,6 +112,11 @@ pub const ETCD_PAYLOAD_SIZE_BUCKETS_BYTES: &[f64] = &[
1024.0, 8192.0, 65536.0, 262144.0, 524288.0, 1048576.0, 1572864.0, 2097152.0, 4194304.0,
];
+/// Pinned like the payload ladder, since every `EtcdStore` user emits the metric.
+pub const ETCD_OP_LATENCY_BUCKETS_MS: &[f64] = &[
+ 1.0, 5.0, 10.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2000.0, 5000.0, 10000.0,
+];
+
fn build_prometheus_builder(product: Option<String>) -> PrometheusBuilder {
const BUCKETS: &[f64] = &[
1.0, 5.0, 10.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2000.0, 5000.0, 10000.0,
@@ -124,6 +129,11 @@ fn build_prometheus_builder(product: Option<String>) -> PrometheusBuilder {
Matcher::Full("assignment_coordination_etcd_payload_bytes".into()),
ETCD_PAYLOAD_SIZE_BUCKETS_BYTES,
)
+ .unwrap()
+ .set_buckets_for_metric(
+ Matcher::Full("assignment_coordination_etcd_op_ms".into()),
+ ETCD_OP_LATENCY_BUCKETS_MS,
+ )
.unwrap();
if let Some(product) = product {
builder = builder.add_global_label("product", product);
diff --git a/rust/personhog-identity/src/main.rs b/rust/personhog-identity/src/main.rs
index 23555f3d4bba..8cbe5c7fcec9 100644
--- a/rust/personhog-identity/src/main.rs
+++ b/rust/personhog-identity/src/main.rs
@@ -4,6 +4,7 @@ use std::time::Duration;
use assignment_coordination::store::{EtcdStore, StoreConfig};
use axum::{routing::get, Router};
use common_database::get_pool_with_config;
+use common_metrics::{ETCD_OP_LATENCY_BUCKETS_MS, ETCD_PAYLOAD_SIZE_BUCKETS_BYTES};
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
@@ -232,6 +233,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.unwrap()
.set_buckets_for_metric(Matcher::Suffix("_per_call".into()), PER_CALL_BUCKETS)
.unwrap()
+ .set_buckets_for_metric(
+ Matcher::Full("assignment_coordination_etcd_op_ms".into()),
+ ETCD_OP_LATENCY_BUCKETS_MS,
+ )
+ .unwrap()
+ .set_buckets_for_metric(
+ Matcher::Full("assignment_coordination_etcd_payload_bytes".into()),
+ ETCD_PAYLOAD_SIZE_BUCKETS_BYTES,
+ )
+ .unwrap()
.install_recorder()
.expect("Failed to install metrics recorder");
{