Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cymbal): record /process latency in milliseconds (#119511)

James Gilla6d88e7838dc
diff --git a/rust/cymbal/src/core/metric_consts.rs b/rust/cymbal/src/core/metric_consts.rs
index 8c9511399520..64e2af4c51f6 100644
--- a/rust/cymbal/src/core/metric_consts.rs
+++ b/rust/cymbal/src/core/metric_consts.rs
@@ -115,7 +115,7 @@ pub const JAVA_EXCEPTION_REMAP_FAILED: &str = "cymbal_java_exception_remap_faile
// HTTP /process observability metrics
pub const PROCESS_REQUESTS_TOTAL: &str = "cymbal_process_requests_total";
-pub const PROCESS_REQUEST_DURATION_SECONDS: &str = "cymbal_process_request_duration_seconds";
+pub const PROCESS_REQUEST_DURATION_MS: &str = "cymbal_process_request_duration_ms";
pub const PROCESS_BATCH_EVENTS: &str = "cymbal_process_batch_events";
pub const PROCESS_IN_FLIGHT: &str = "cymbal_process_in_flight";
diff --git a/rust/cymbal/src/modes/processing/router/event.rs b/rust/cymbal/src/modes/processing/router/event.rs
index 95efd61714cf..a892241b631d 100644
--- a/rust/cymbal/src/modes/processing/router/event.rs
+++ b/rust/cymbal/src/modes/processing/router/event.rs
@@ -17,7 +17,7 @@ use crate::{
error::UnhandledError,
metric_consts::{
ERRORS, PROCESS_BATCH_EVENTS, PROCESS_IN_FLIGHT, PROCESS_REQUESTS_TOTAL,
- PROCESS_REQUEST_DURATION_SECONDS,
+ PROCESS_REQUEST_DURATION_MS,
},
stages::http_pipeline::HttpEventPipeline,
types::{batch::Batch, event::AnyEvent, stage::Stage},
@@ -137,10 +137,9 @@ pub async fn process_events(
let duration = started_at.elapsed();
let duration_ms = duration.as_millis() as u64;
- let duration_s = duration.as_secs_f64();
let is_slow = duration_ms >= slow_log_threshold_ms;
- metrics::histogram!(PROCESS_REQUEST_DURATION_SECONDS).record(duration_s);
+ metrics::histogram!(PROCESS_REQUEST_DURATION_MS).record(duration.as_secs_f64() * 1000.0);
match &output {
Ok(batch) => {
diff --git a/rust/cymbal/src/modes/resolution/service/resolve.rs b/rust/cymbal/src/modes/resolution/service/resolve.rs
index 57f2f16d04ea..8a10af3463a0 100644
--- a/rust/cymbal/src/modes/resolution/service/resolve.rs
+++ b/rust/cymbal/src/modes/resolution/service/resolve.rs
@@ -22,7 +22,6 @@ use cymbal_proto::cymbal::resolution::v1::{
use super::codes;
use crate::modes::resolution::load_monitor::LoadMonitor;
-const RESOLVE_REQUEST_DURATION_MS: &str = "cymbal_remote_resolution_server_request_duration_ms";
const SERVER_ERROR_KINDS: &str = "cymbal_remote_resolution_server_error_kinds_total";
const SERVER_ITEM_DURATION_MS: &str = "cymbal_remote_resolution_server_item_duration_ms";
const SERVER_ITEMS_TOTAL: &str = "cymbal_remote_resolution_server_items_total";
@@ -33,12 +32,9 @@ pub(super) async fn run_resolve(
tx: mpsc::Sender<Result<ResolveOutcome, Status>>,
load_monitor: LoadMonitor,
) {
- let started_at = Instant::now();
-
loop {
let item = tokio::select! {
_ = tx.closed() => {
- record_resolve_duration(started_at, "cancelled");
return;
}
maybe_item = input.message() => {
@@ -47,7 +43,6 @@ pub(super) async fn run_resolve(
Ok(None) => break,
Err(err) => {
warn!(error = %err, "resolve input stream failed");
- record_resolve_duration(started_at, "input_error");
return;
}
}
@@ -59,7 +54,6 @@ pub(super) async fn run_resolve(
if item.deadline_ms == 0 {
record_item_metrics(item_started_at, "error", "timeout");
if !send_overloaded(&tx, item.id, "item deadline expired", 0).await {
- record_resolve_duration(started_at, "cancelled");
return;
}
continue;
@@ -68,7 +62,6 @@ pub(super) async fn run_resolve(
if !load_monitor.try_admit() {
record_item_metrics(item_started_at, "error", "overloaded");
if !send_overloaded(&tx, item.id, "server overloaded", 0).await {
- record_resolve_duration(started_at, "cancelled");
return;
}
continue;
@@ -78,7 +71,6 @@ pub(super) async fn run_resolve(
let item_stage = stage.clone();
if !send_accepted(&tx, item.id).await {
load_monitor.decrement_in_flight();
- record_resolve_duration(started_at, "cancelled");
return;
}
let in_flight_guard = InFlightGuard::new(load_monitor.clone());
@@ -92,8 +84,6 @@ pub(super) async fn run_resolve(
.await;
});
}
-
- record_resolve_duration(started_at, "completed");
}
async fn send_accepted(tx: &mpsc::Sender<Result<ResolveOutcome, Status>>, id: u64) -> bool {
@@ -218,11 +208,6 @@ fn error_result(kind: codes::ErrorKind, message: String) -> resolve_outcome::Res
})
}
-fn record_resolve_duration(started_at: Instant, outcome: &'static str) {
- metrics::histogram!(RESOLVE_REQUEST_DURATION_MS, "outcome" => outcome)
- .record(started_at.elapsed().as_secs_f64() * 1000.0);
-}
-
fn record_item_metrics(started_at: Instant, outcome: &'static str, kind: &'static str) {
metrics::counter!(SERVER_ITEMS_TOTAL, "outcome" => outcome, "kind" => kind).increment(1);
metrics::histogram!(SERVER_ITEM_DURATION_MS, "outcome" => outcome, "kind" => kind)