:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(error-tracking): bound cymbal symbol set parses by estimated memory (#119505)
Hugues Pouillot78ac656a2f79
diff --git a/rust/cymbal/src/core/config.rs b/rust/cymbal/src/core/config.rsindex 8ada2a845bb5..9979ea8f251b 100644--- a/rust/cymbal/src/core/config.rs+++ b/rust/cymbal/src/core/config.rs@@ -1,3 +1,5 @@+use std::num::NonZeroUsize;+use aws_config::{BehaviorVersion, Region};use common_database::{get_pool_with_config, PoolConfig};use envconfig::Envconfig;@@ -102,14 +104,23 @@ pub struct ResolverConfig {#[envconfig(default = "64")]pub symbol_resolution_concurrency: usize,- // Parsing a symbol set can need many times its fetched size in memory, and the symbol- // store cache does not count that memory. Parses of fetched blobs at least this large- // share a small concurrency limit, so a few of them cannot run the pod out of memory.+ // Parsing a symbol set can need many times its decompressed size in memory, and the symbol+ // store cache does not count that memory. Parses of symbol sets at least this large+ // (decompressed) reserve their estimated peak memory from a shared budget, so together+ // they cannot run the pod out of memory.#[envconfig(default = "10000000")]- pub symbol_set_large_parse_bytes: usize,+ pub symbol_set_large_parse_bytes: NonZeroUsize,#[envconfig(default = "2")]- pub symbol_set_max_concurrent_large_parses: usize,+ pub symbol_set_max_concurrent_large_parses: NonZeroUsize,++ // Set this below the memory limit minus the symbol store cache budget and the steady-state usage.+ #[envconfig(default = "1000000000")]+ pub symbol_set_parse_memory_budget_bytes: NonZeroUsize,++ // Estimated peak parse memory, as a multiple of the decompressed symbol set size.+ #[envconfig(default = "8")]+ pub symbol_set_parse_memory_multiplier: NonZeroUsize,// Shared secret authenticating the cymbal <-> cymbal-resolution gRPC seam.#[envconfig(default = "")]@@ -176,3 +187,26 @@ pub async fn get_aws_config(config: &ResolverConfig) -> aws_sdk_s3::Config {.build()}}++#[cfg(test)]+mod tests {+ use std::collections::HashMap;++ use super::*;++ #[test]+ fn parse_limiter_settings_must_be_positive() {+ for name in [+ "SYMBOL_SET_LARGE_PARSE_BYTES",+ "SYMBOL_SET_MAX_CONCURRENT_LARGE_PARSES",+ "SYMBOL_SET_PARSE_MEMORY_BUDGET_BYTES",+ "SYMBOL_SET_PARSE_MEMORY_MULTIPLIER",+ ] {+ let values = HashMap::from([(name.to_owned(), "0".to_owned())]);+ assert!(matches!(+ ResolverConfig::init_from_hashmap(&values),+ Err(envconfig::Error::ParseError { name: field }) if field == name+ ));+ }+ }+}diff --git a/rust/cymbal/src/core/metric_consts.rs b/rust/cymbal/src/core/metric_consts.rsindex 64e2af4c51f6..cc42b9582df0 100644--- a/rust/cymbal/src/core/metric_consts.rs+++ b/rust/cymbal/src/core/metric_consts.rs@@ -56,6 +56,7 @@ pub const SOURCEMAP_PARSE: &str = "cymbal_sourcemap_parse";pub const SYMBOL_SET_DECOMPRESSED_BYTES: &str = "cymbal_symbol_set_decompressed_bytes";// Time a large symbol set parse waits for a `ParseLimiter` permit, in milliseconds.pub const SYMBOL_SET_LARGE_PARSE_WAIT_MS: &str = "cymbal_symbol_set_large_parse_wait_ms";+pub const SYMBOL_SET_PARSE_INPUT_BYTES: &str = "cymbal_symbol_set_parse_input_bytes";// Histogram buckets for the byte-shaped metrics above. The default// `common_metrics` buckets are tuned for milliseconds of latency and saturatediff --git a/rust/cymbal/src/core/resolver.rs b/rust/cymbal/src/core/resolver.rsindex 5912b49e6ba3..283fec219adb 100644--- a/rust/cymbal/src/core/resolver.rs+++ b/rust/cymbal/src/core/resolver.rs@@ -59,8 +59,10 @@ pub fn build_catalog(config.symbol_store_cache_max_bytes,)));let parse_limiter = ParseLimiter::new(- config.symbol_set_large_parse_bytes,- config.symbol_set_max_concurrent_large_parses,+ config.symbol_set_large_parse_bytes.get(),+ config.symbol_set_max_concurrent_large_parses.get(),+ config.symbol_set_parse_memory_budget_bytes.get(),+ config.symbol_set_parse_memory_multiplier.get(),);let smp = SourcemapProvider::new(config).with_chunk_id_rescue(diff --git a/rust/cymbal/src/core/symbolication/symbol_store/caching.rs b/rust/cymbal/src/core/symbolication/symbol_store/caching.rsindex ec23fec71051..6f1278b9809f 100644--- a/rust/cymbal/src/core/symbolication/symbol_store/caching.rs+++ b/rust/cymbal/src/core/symbolication/symbol_store/caching.rs@@ -7,7 +7,7 @@ use tracing::info;use crate::metric_consts::{STORE_CACHED_BYTES, STORE_CACHE_EVICTIONS, STORE_CACHE_EVICTION_RUNS, STORE_CACHE_HITS,- STORE_CACHE_MISSES, SYMBOL_SET_LARGE_PARSE_WAIT_MS,+ STORE_CACHE_MISSES, SYMBOL_SET_LARGE_PARSE_WAIT_MS, SYMBOL_SET_PARSE_INPUT_BYTES,};use super::{chunk_id::SymbolSetCacheKey, Fetcher, ParsePermit, Parser, Provider};@@ -26,32 +26,94 @@ pub struct Caching<P> {parse_limiter: Option<ParseLimiter>,}-// Bounds how many large symbol sets are parsed at once. A parse can need many times the-// fetched size in transient memory, and the cache budget does not count it. Clones share-// one set of permits, so a single limiter covers every provider it is given to.+// The parse memory budget is counted in MiB, because a tokio semaphore can acquire at most+// `u32::MAX` permits in one call.+const PARSE_BUDGET_UNIT_BYTES: usize = 1 << 20;++// Bounds the transient memory of large symbol set parses. A parse can need many times its+// decompressed size, and the cache budget does not count it. Each large parse reserves its+// estimated peak memory from a shared budget. It also reserves at least an equal share of the+// budget, so no more than `max_concurrent_large_parses` run at once. A parse estimated above+// the whole budget reserves all of it and runs alone. Clones share one budget, so a single+// limiter covers every provider it is given to.#[derive(Clone)]pub struct ParseLimiter {large_parse_bytes: usize,+ memory_multiplier: usize,+ budget_units: u32,+ min_units: u32,permits: Arc<Semaphore>,}impl ParseLimiter {- pub fn new(large_parse_bytes: usize, max_concurrent_large_parses: usize) -> Self {+ pub fn new(+ large_parse_bytes: usize,+ max_concurrent_large_parses: usize,+ memory_budget_bytes: usize,+ memory_multiplier: usize,+ ) -> Self {+ assert!(+ large_parse_bytes > 0,+ "SYMBOL_SET_LARGE_PARSE_BYTES must be positive"+ );+ assert!(+ max_concurrent_large_parses > 0,+ "SYMBOL_SET_MAX_CONCURRENT_LARGE_PARSES must be positive"+ );+ assert!(+ memory_budget_bytes > 0,+ "SYMBOL_SET_PARSE_MEMORY_BUDGET_BYTES must be positive"+ );+ assert!(+ memory_multiplier > 0,+ "SYMBOL_SET_PARSE_MEMORY_MULTIPLIER must be positive"+ );+ let budget_units = units_for_bytes(memory_budget_bytes);+ let max_concurrent = u32::try_from(max_concurrent_large_parses).unwrap_or(u32::MAX);Self {large_parse_bytes,- permits: Arc::new(Semaphore::new(max_concurrent_large_parses.max(1))),+ memory_multiplier,+ budget_units,+ min_units: budget_units.div_ceil(max_concurrent),+ permits: Arc::new(Semaphore::new(budget_units as usize)),}}- async fn acquire(&self, required: bool) -> ParsePermit {- if !required {+ // Returns None when the parse is small enough to skip the limiter. When the decompressed+ // size is unknown, the parse counts as large and at least as big as its fetched size.+ fn estimate_parse_bytes(+ &self,+ input_bytes: Option<usize>,+ fetched_bytes: usize,+ ) -> Option<usize> {+ let input_bytes = match input_bytes {+ Some(bytes) if bytes < self.large_parse_bytes => return None,+ Some(bytes) => bytes,+ None => fetched_bytes.max(self.large_parse_bytes),+ };+ Some(input_bytes.saturating_mul(self.memory_multiplier))+ }++ async fn acquire(&self, estimated_bytes: Option<usize>) -> ParsePermit {+ let Some(estimated_bytes) = estimated_bytes else {return ParsePermit::none();- }+ };+ let units = units_for_bytes(estimated_bytes).clamp(self.min_units, self.budget_units);// UNWRAP - we never close the semaphore- ParsePermit::limited(self.permits.clone().acquire_owned().await.unwrap())+ ParsePermit::limited(+ self.permits+ .clone()+ .acquire_many_owned(units)+ .await+ .unwrap(),+ )}}+fn units_for_bytes(bytes: usize) -> u32 {+ u32::try_from(bytes.div_ceil(PARSE_BUDGET_UNIT_BYTES)).unwrap_or(u32::MAX)+}+impl<P> Caching<P>// This where clause exists exclusively to give more obvious compiler errors in cases where// the passed P doesn't cause Provider to be implemented for this Caching<P> - for example,@@ -105,12 +167,19 @@ wherelet fetched_bytes = found.byte_count();let set_type = set_type_name::<P::Set>();+ let parse_input_bytes = found.parse_input_bytes();+ if let Some(bytes) = parse_input_bytes {+ metrics::histogram!(SYMBOL_SET_PARSE_INPUT_BYTES, "kind" => set_type)+ .record(bytes as f64);+ }+ let estimated_parse_bytes = self+ .parse_limiter+ .as_ref()+ .and_then(|limiter| limiter.estimate_parse_bytes(parse_input_bytes, fetched_bytes));+let wait_start = Instant::now();let permit = match &self.parse_limiter {- Some(limiter) => {- let required = found.requires_parse_permit(limiter.large_parse_bytes);- limiter.acquire(required).await- }+ Some(limiter) => limiter.acquire(estimated_parse_bytes).await,None => ParsePermit::none(),};let wait_ms = wait_start.elapsed().as_millis() as u64;@@ -118,21 +187,31 @@ wheremetrics::histogram!(SYMBOL_SET_LARGE_PARSE_WAIT_MS).record(wait_ms as f64);}- if fetched_bytes >= LARGE_FETCHED_BYTES {+ let log_large_parse = fetched_bytes >= LARGE_FETCHED_BYTES+ || parse_input_bytes.is_some_and(|bytes| bytes >= LARGE_PARSED_BYTES);+ if log_large_parse {info!(team_id,- cache_key, set_type, fetched_bytes, wait_ms, "Parsing large symbol set"+ cache_key,+ set_type,+ fetched_bytes,+ parse_input_bytes,+ estimated_parse_bytes,+ wait_ms,+ "Parsing large symbol set");}let parse_start = Instant::now();let parsed = self.inner.parse(found, permit).await?;let bytes = parsed.byte_count();- if fetched_bytes >= LARGE_FETCHED_BYTES || bytes >= LARGE_PARSED_BYTES {+ if log_large_parse || bytes >= LARGE_PARSED_BYTES {info!(team_id,cache_key,set_type,fetched_bytes,+ parse_input_bytes,+ estimated_parse_bytes,cached_bytes = bytes,parse_ms = parse_start.elapsed().as_millis() as u64,"Parsed large symbol set"@@ -241,8 +320,9 @@ struct CachedSymbolSet {pub trait Countable {fn byte_count(&self) -> usize;- fn requires_parse_permit(&self, threshold: usize) -> bool {- self.byte_count() >= threshold+ // The size of the data the parser works on, after any decompression. None means unknown.+ fn parse_input_bytes(&self) -> Option<usize> {+ Some(self.byte_count())}}@@ -257,10 +337,10 @@ impl Countable for Bytes {self.len()}- fn requires_parse_permit(&self, threshold: usize) -> bool {+ fn parse_input_bytes(&self) -> Option<usize> {match posthog_symbol_data::symbol_data_known_decompressed_size(self) {- Ok(Some(size)) => size.max(self.len()) >= threshold,- Ok(None) | Err(_) => true,+ Ok(Some(size)) => Some(size.max(self.len())),+ Ok(None) | Err(_) => None,}}}@@ -438,8 +518,16 @@ mod tests {}#[tokio::test(flavor = "current_thread")]- async fn parse_limiter_bounds_only_large_parses() {- for (fetched_bytes, expected_max) in [(100, 1), (99, 3)] {+ async fn parse_limiter_bounds_large_parses_by_count_and_memory() {+ // (fetched_bytes, max_concurrent, memory_multiplier, expected_max). The budget is+ // 3 MiB, so a 100 byte input times 20_000 estimates 2 MiB and runs alone.+ for (fetched_bytes, max_concurrent, memory_multiplier, expected_max) in [+ (100, 1, 1, 1),+ (99, 1, 1, 3),+ (100, 2, 1, 1),+ (100, 3, 1, 3),+ (100, 3, 20_000, 1),+ ] {let max_in_flight = Arc::new(AtomicUsize::new(0));let provider = SizedProvider {fetched_bytes,@@ -447,8 +535,12 @@ mod tests {max_in_flight: max_in_flight.clone(),};let cache = Arc::new(Mutex::new(SymbolSetCache::new(1 << 20)));- let caching =- Caching::new(provider, cache).with_parse_limiter(ParseLimiter::new(100, 1));+ let caching = Caching::new(provider, cache).with_parse_limiter(ParseLimiter::new(+ 100,+ max_concurrent,+ 3 << 20,+ memory_multiplier,+ ));futures::future::join_all((0..3).map(|i| caching.lookup(1, OrChunkId::<Url>::chunk_id(format!("set-{i}")))),@@ -458,7 +550,8 @@ mod tests {assert_eq!(max_in_flight.load(Ordering::SeqCst),expected_max,- "fetched_bytes = {fetched_bytes}"+ "fetched_bytes = {fetched_bytes}, max_concurrent = {max_concurrent}, \+ memory_multiplier = {memory_multiplier}");}@@ -470,7 +563,7 @@ mod tests {.unwrap(),);assert!(compressed.len() < 100);- assert!(compressed.requires_parse_permit(100));+ assert!(compressed.parse_input_bytes().unwrap() >= 200);let mut raw_zip = Cursor::new(Vec::new());{@@ -482,7 +575,7 @@ mod tests {writer.finish().unwrap();}let raw_zip = Bytes::from(raw_zip.into_inner());- assert!(raw_zip.requires_parse_permit(raw_zip.len() + 1));+ assert_eq!(raw_zip.parse_input_bytes(), None);}#[tokio::test]@@ -495,7 +588,7 @@ mod tests {started: started_tx,release: release.clone(),};- let limiter = ParseLimiter::new(1, 1);+ let limiter = ParseLimiter::new(1, 1, 1 << 20, 1);let caching = Arc::new(Caching::new(provider, Arc::new(Mutex::new(SymbolSetCache::new(1024)))).with_parse_limiter(limiter.clone()),diff --git a/rust/cymbal/src/core/symbolication/symbol_store/saving.rs b/rust/cymbal/src/core/symbolication/symbol_store/saving.rsindex 57016cdefa43..d2d8ad978038 100644--- a/rust/cymbal/src/core/symbolication/symbol_store/saving.rs+++ b/rust/cymbal/src/core/symbolication/symbol_store/saving.rs@@ -128,8 +128,8 @@ impl Countable for Saveable {self.data.len()}- fn requires_parse_permit(&self, threshold: usize) -> bool {- self.data.requires_parse_permit(threshold)+ fn parse_input_bytes(&self) -> Option<usize> {+ self.data.parse_input_bytes()}}diff --git a/rust/cymbal/src/modes/processing/server.rs b/rust/cymbal/src/modes/processing/server.rsindex 7f6b65213b82..70f9f03ac3bf 100644--- a/rust/cymbal/src/modes/processing/server.rs+++ b/rust/cymbal/src/modes/processing/server.rs@@ -8,6 +8,7 @@ use crate::{metric_consts::{BYTE_HISTOGRAM_BUCKETS, ISSUE_CREATED_EVENT_PROPERTIES_BYTES, S3_FETCHED_BYTES,S3_PUT_BYTES, SOURCEMAP_EXTERNAL_BYTES, SYMBOL_SET_DECOMPRESSED_BYTES,+ SYMBOL_SET_PARSE_INPUT_BYTES,},modes::processing::config::ProcessingConfig,router::get_router,@@ -33,6 +34,10 @@ pub async fn start_server(config: ProcessingConfig, context: Arc<AppContext>) ->Matcher::Full(SYMBOL_SET_DECOMPRESSED_BYTES.into()),BYTE_HISTOGRAM_BUCKETS,),+ (+ Matcher::Full(SYMBOL_SET_PARSE_INPUT_BYTES.into()),+ BYTE_HISTOGRAM_BUCKETS,+ ),];let router = setup_metrics_routes_with_overrides(router, bucket_overrides);let bind = format!("{}:{}", config.host, config.port);diff --git a/rust/cymbal/src/modes/resolution/README.md b/rust/cymbal/src/modes/resolution/README.mdindex c62547cbb4ba..a470ba3558b9 100644--- a/rust/cymbal/src/modes/resolution/README.md+++ b/rust/cymbal/src/modes/resolution/README.md@@ -123,14 +123,18 @@ All variables are prefixed `CYMBAL_REMOTE_RESOLUTION_` and live on `cymbal::conf| `INTERNAL_API_SECRET` | _empty_ | Shared secret required as `X-Internal-Api-Secret` metadata on every gRPC request. Empty configuration rejects all RPCs. || `MAX_CONCURRENT_REQUESTS` | `256` | Hard cap on in-flight gRPC streams. Excess returns `UNAVAILABLE` from the gRPC load-shed layer. `0` disables the cap. || `SYMBOL_RESOLUTION_CONCURRENCY` | `64` | Cap on concurrent symbol-resolution operations across all in-flight items. |-| `SYMBOL_SET_LARGE_PARSE_BYTES` | `10000000` | Fetched size, in bytes, at which a symbol set parse counts as large. Large parses share the limit below, because their transient memory is not counted by the symbol store cache. |-| `SYMBOL_SET_MAX_CONCURRENT_LARGE_PARSES` | `2` | Cap on concurrent large symbol set parses across all providers. Other large parses wait. |+| `SYMBOL_SET_LARGE_PARSE_BYTES` | `10000000` | Decompressed size, in bytes, at which a symbol set parse counts as large. Large parses share the memory budget below, because their transient memory is not counted by the symbol store cache. |+| `SYMBOL_SET_MAX_CONCURRENT_LARGE_PARSES` | `2` | Cap on concurrent large symbol set parses across all providers. Each large parse reserves at least this share of the memory budget. |+| `SYMBOL_SET_PARSE_MEMORY_BUDGET_BYTES` | `1000000000` | Shared budget for the estimated peak memory of large parses. A parse waits until its estimate fits. A parse estimated above the whole budget runs alone. |+| `SYMBOL_SET_PARSE_MEMORY_MULTIPLIER` | `8` | Estimated peak parse memory, as a multiple of the decompressed symbol set size. || `MAX_ITEM_CONCURRENCY` | `64` | Process-wide cap on concurrently processed exception items. Excess items receive `ERROR_KIND_OVERLOADED`. || `SERVICE_INSTANCE_ID` | random UUID | Identifier surfaced to callers via `LoadEvent` on the Subscribe stream. || `SUBSCRIBE_TICK_INTERVAL_MS` | `1000` | Default `LoadEvent` heartbeat cadence when callers do not suggest one. Load events may be emitted earlier when draining changes or in-flight load crosses coarse thresholds. || `SUBSCRIBE_MIN_TICK_MS` | `100` | Lower bound for the Subscribe tick cadence. || `SUBSCRIBE_MAX_TICK_MS` | `10000` | Upper bound for the Subscribe tick cadence. |+All four symbol set parse settings must be positive integers. Invalid values prevent the service from starting.+The server inherits the narrow subset of cymbal's env-var surface needed by `build_symbol_resolver` and internal gRPC authentication: `INTERNAL_API_SECRET`, Postgres, object storage, symbol-store cache/resolver tuning, and outbound HTTP controls. Frame result cache knobs such as `FRAME_RESOLVED_TTL_SECONDS` and `FRAME_UNRESOLVED_TTL_SECONDS` are read through cymbal's shared symbol resolver config, not duplicated in `cymbal-resolution`'s own `Config`. It does not connect to Kafka, Redis, or the signals API.Shared-secret callers are service-scoped today, so authenticated cymbal pods are permitted to resolve all teams. If future callers carry a team-scoped identity, the `Resolve` handler must reject any `ResolveItem.team_id` outside the caller's allowed team set before scheduling resolution work.@@ -150,6 +154,7 @@ Shared-secret callers are service-scoped today, so authenticated cymbal pods areThese metric names are exported by the cymbal client unless noted. Definitions live in [`cymbal/src/metric_consts.rs`](../cymbal/src/metric_consts.rs) and the server's Resolve/LoadMonitor modules.+- **Symbol set parse sizes**: `cymbal_symbol_set_parse_input_bytes{kind}` records known sizes before parsing, including decompressed sizes for supported symbol data. Use its byte-sized buckets to inspect the distribution without relying on per-parse logs. Unknown sizes do not enter this histogram. Logs before parsing still identify fetched blobs of at least 10 MB and known parse inputs of at least 50 MB. These sizes do not measure peak memory.- **Item attempts and latency**: `cymbal_remote_resolution_requests_total{outcome, reason?}` and `cymbal_remote_resolution_latency_ms`. The counter is emitted on logical item attempts; `reason` is only a bounded transport status tag.- **Reroute shape**: `cymbal_remote_resolution_reroute_depth{outcome}` records how many endpoint changes happened before the terminal item result. The legacy attempts histogram may still be emitted for dashboard continuity, but new alerts should use reroute depth.- **Protocol error taxonomy**: client-observed `cymbal_remote_resolution_error_kinds_total{kind}` and server-emitted `cymbal_remote_resolution_server_error_kinds_total{kind}` count `ErrorKind` values with bounded labels.diff --git a/rust/cymbal/src/modes/resolution/mod.rs b/rust/cymbal/src/modes/resolution/mod.rsindex c87aa0b6a358..80e004cad34d 100644--- a/rust/cymbal/src/modes/resolution/mod.rs+++ b/rust/cymbal/src/modes/resolution/mod.rs@@ -11,6 +11,7 @@ use std::sync::{use std::time::Duration;use axum::{http::StatusCode, routing::get, Router};+use common_metrics::Matcher;use cymbal_proto::cymbal::resolution::v1::cymbal_resolution_server::CymbalResolutionServer;use cymbal_proto::CYMBAL_RESOLUTION_MAX_MESSAGE_SIZE_BYTES;use personhog_common::grpc::{tracked_tcp_incoming, GrpcLoadShedLayer, GrpcMetricsLayer};@@ -21,6 +22,9 @@ use tonic::transport::Server;use tracing::{info, warn};use crate::core::{config::ResolverConfig, shutdown::wait_for_shutdown};+use crate::metric_consts::{+ BYTE_HISTOGRAM_BUCKETS, SYMBOL_SET_DECOMPRESSED_BYTES, SYMBOL_SET_PARSE_INPUT_BYTES,+};pub mod app_context;pub mod auth;@@ -125,7 +129,20 @@ fn spawn_metrics_server("/_readiness",get(move || readiness(readiness_draining.clone())),);- let router = common_metrics::setup_metrics_routes_for_product(router, "cymbal-resolution");+ let router = common_metrics::setup_metrics_routes_for_product_with_overrides(+ router,+ "cymbal-resolution",+ &[+ (+ Matcher::Full(SYMBOL_SET_DECOMPRESSED_BYTES.into()),+ BYTE_HISTOGRAM_BUCKETS,+ ),+ (+ Matcher::Full(SYMBOL_SET_PARSE_INPUT_BYTES.into()),+ BYTE_HISTOGRAM_BUCKETS,+ ),+ ],+ );let bind = format!("0.0.0.0:{port}");info!("Metrics server listening on {}", bind);