Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
Extract hang telemetry into a reusable hang_telemetry crate (#64874)
Anthony Eid1b572a2bd037
diff --git a/Cargo.lock b/Cargo.lockindex 3eb7bbd0f00..47b48dc74d8 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -7993,6 +7993,17 @@ dependencies = ["thiserror 1.0.69",]+[[package]]+name = "hang_telemetry"+version = "0.1.0"+dependencies = [+ "gpui",+ "hdrhistogram",+ "log",+ "serde_json",+ "telemetry_events",+]+[[package]]name = "harfrust"version = "0.5.0"@@ -22824,7 +22835,7 @@ dependencies = ["gpui","gpui_platform","gpui_tokio",- "hdrhistogram",+ "hang_telemetry","http_client","image","image_viewer",diff --git a/Cargo.toml b/Cargo.tomlindex 1d44d21597f..a8e1cd85ab1 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -104,6 +104,7 @@ members = ["crates/gpui_wgpu","crates/gpui_windows","crates/grammars",+ "crates/hang_telemetry","crates/html_to_markdown","crates/http_client","crates/http_client_tls",@@ -374,6 +375,7 @@ gpui_web = { path = "crates/gpui_web" }gpui_wgpu = { path = "crates/gpui_wgpu" }gpui_windows = { path = "crates/gpui_windows", default-features = false }grammars = { path = "crates/grammars" }+hang_telemetry = { path = "crates/hang_telemetry" }html_to_markdown = { path = "crates/html_to_markdown" }http_client = { path = "crates/http_client" }http_client_tls = { path = "crates/http_client_tls" }diff --git a/crates/hang_telemetry/Cargo.toml b/crates/hang_telemetry/Cargo.tomlnew file mode 100644index 00000000000..8555fcf66aa--- /dev/null+++ b/crates/hang_telemetry/Cargo.toml@@ -0,0 +1,20 @@+[package]+name = "hang_telemetry"+version = "0.1.0"+edition.workspace = true+publish.workspace = true+license = "GPL-3.0-or-later"++[lints]+workspace = true++[lib]+path = "src/hang_telemetry.rs"+doctest = false++[dependencies]+gpui = { workspace = true, features = ["profiler"] }+hdrhistogram.workspace = true+log.workspace = true+serde_json.workspace = true+telemetry_events.workspace = truediff --git a/crates/hang_telemetry/LICENSE-GPL b/crates/hang_telemetry/LICENSE-GPLnew file mode 120000index 00000000000..89e542f750c--- /dev/null+++ b/crates/hang_telemetry/LICENSE-GPL@@ -0,0 +1 @@+../../LICENSE-GPL\ No newline at end of filediff --git a/crates/hang_telemetry/src/hang_telemetry.rs b/crates/hang_telemetry/src/hang_telemetry.rsnew file mode 100644index 00000000000..c3b5769e0b4--- /dev/null+++ b/crates/hang_telemetry/src/hang_telemetry.rs@@ -0,0 +1,386 @@+//! Turns GPUI foreground hang incidents into batched "Hang Incidents" telemetry.+//!+//! [`HangTelemetry`] polls a [`HangDetector`] over the app's foreground journal,+//! serializes each incident, and batches them into periodic telemetry events.+//! Applications own delivery: they send the returned [`FlexibleEvent`] through+//! their own telemetry pipeline, so every GPUI app that uses this crate reports+//! hangs with the same thresholds and wire schema.++use std::collections::HashMap;+use std::time::{Duration, Instant};++use gpui::profiler::hang::{+ HangDetector, HangTrigger, MEASUREMENT_VERSION, SerializedHangIncident,+};+use gpui::profiler::journal::ForegroundJournal;+use hdrhistogram::Histogram;+use serde_json::Value;+use telemetry_events::FlexibleEvent;++pub const EVENT_TYPE: &str = "Hang Incidents";++const MAX_SERIALIZED_CONTRIBUTORS: usize = 8;++/// Cap on incidents per telemetry event. When more accrue between sends, the+/// ones with the largest stalls are kept and the incident counts still cover+/// them all.+const MAX_REPORTED_INCIDENTS: usize = 10;++// A long interval keeps hang telemetry a small fraction of event volume even+// for pathologically hang-prone sessions; the on-quit flush covers short ones.+const SEND_INTERVAL: Duration = Duration::from_mins(30);++/// Duration at which a single piece of foreground work counts as a hang.+pub fn hang_threshold() -> Duration {+ if cfg!(debug_assertions) {+ if cfg!(windows) {+ // Windows debug builds are especially slow.+ Duration::from_secs(30)+ } else {+ Duration::from_secs(5)+ }+ } else {+ // Will be lowered over time or turned into a setting.+ Duration::from_millis(100)+ }+}++/// Total foreground spend within one interval that counts as a hang.+pub fn frame_budget() -> Duration {+ if cfg!(debug_assertions) {+ // Unoptimized builds routinely spend more than a release frame budget+ // on ordinary frames; keep dev builds from reporting constantly.+ Duration::from_millis(100)+ } else {+ // At least one dropped frame on any display. Generous while budget+ // incidents are plentiful; lower it as they get fixed.+ Duration::from_millis(24)+ }+}++/// Detects foreground hangs and batches them into telemetry events.+///+/// Polling and sending are driven by the caller, typically from a dedicated+/// OS thread so reporting keeps working while GPUI's executors are stalled.+pub struct HangTelemetry {+ detector: HangDetector,+ reporter: Reporter,+ startup: Instant,+}++impl HangTelemetry {+ /// Starts observing `journal` with [`hang_threshold`] and [`frame_budget`].+ ///+ /// Incident timestamps are reported relative to `startup`.+ pub fn new(journal: ForegroundJournal, startup: Instant) -> Self {+ Self {+ detector: HangDetector::new(journal, hang_threshold(), frame_budget()),+ reporter: Reporter::new(),+ startup,+ }+ }++ /// Adds incidents sealed since the previous call to the pending batch.+ ///+ /// Returns the newly collected incidents so callers can also surface them+ /// elsewhere, such as in feedback reports.+ pub fn collect(&mut self) -> Vec<SerializedHangIncident> {+ let incidents = self.detector.poll();+ let first_present_at = self.detector.first_present_at();+ let serialized_incidents = incidents+ .iter()+ .map(|incident| {+ SerializedHangIncident::convert(+ self.startup,+ incident,+ MAX_SERIALIZED_CONTRIBUTORS,+ first_present_at,+ )+ })+ .collect::<Vec<_>>();+ for incident in &serialized_incidents {+ self.reporter.add(incident.clone());+ }+ serialized_incidents+ }++ /// Takes the pending batch as an event once the send interval has elapsed.+ pub fn take_event_if_due(&mut self) -> Option<FlexibleEvent> {+ (self.reporter.last_send.elapsed() > SEND_INTERVAL).then(|| self.take_event())+ }++ /// Takes the pending batch as an event, even when it has no incidents.+ ///+ /// Empty events still matter: summing `report_window_seconds` gives the+ /// observed time when computing incident rates.+ pub fn take_event(&mut self) -> FlexibleEvent {+ self.reporter.take_event().into_flexible_event()+ }+}++struct Reporter {+ last_send: Instant,+ pending: Vec<SerializedHangIncident>,+ threshold_incidents: u64,+ budget_incidents: u64,+ /// Every incident's stall, in milliseconds; `pending` keeps only the largest.+ stalls: Histogram<u64>,+}++impl Reporter {+ fn new() -> Self {+ Self {+ last_send: Instant::now(),+ pending: Vec::new(),+ threshold_incidents: 0,+ budget_incidents: 0,+ stalls: Histogram::new(3).expect("3 significant figures is a valid histogram"),+ }+ }++ fn add(&mut self, incident: SerializedHangIncident) {+ match incident.trigger {+ HangTrigger::Threshold => self.threshold_incidents += 1,+ HangTrigger::Budget => self.budget_incidents += 1,+ }+ self.stalls.record(incident.stall_ms as u64).ok();+ self.pending.push(incident);+ if self.pending.len() > MAX_REPORTED_INCIDENTS {+ self.pending+ .sort_by(|first, second| second.stall_ms.total_cmp(&first.stall_ms));+ self.pending.truncate(MAX_REPORTED_INCIDENTS);+ }+ }++ fn take_event(&mut self) -> HangIncidentsEvent {+ let now = Instant::now();+ let report_window_seconds = now.duration_since(self.last_send).as_secs();+ self.last_send = now;+ let mut incidents = std::mem::take(&mut self.pending);+ incidents.sort_by(|first, second| second.stall_ms.total_cmp(&first.stall_ms));+ let threshold_incidents = std::mem::take(&mut self.threshold_incidents);+ let budget_incidents = std::mem::take(&mut self.budget_incidents);+ let event = HangIncidentsEvent {+ incidents,+ total_incidents: threshold_incidents + budget_incidents,+ threshold_incidents,+ budget_incidents,+ stall_p50_ms: self.stalls.value_at_quantile(0.5),+ stall_p95_ms: self.stalls.value_at_quantile(0.95),+ stall_max_ms: self.stalls.max(),+ report_window_seconds,+ };+ self.stalls.reset();+ event+ }+}++struct HangIncidentsEvent {+ incidents: Vec<SerializedHangIncident>,+ /// Predates the threshold/budget split; existing queries key on it.+ total_incidents: u64,+ threshold_incidents: u64,+ budget_incidents: u64,+ stall_p50_ms: u64,+ stall_p95_ms: u64,+ stall_max_ms: u64,+ report_window_seconds: u64,+}++impl HangIncidentsEvent {+ fn into_flexible_event(self) -> FlexibleEvent {+ FlexibleEvent {+ event_type: EVENT_TYPE.to_string(),+ event_properties: HashMap::from([+ ("incidents".to_string(), to_value(&self.incidents)),+ ("total_incidents".to_string(), self.total_incidents.into()),+ (+ "threshold_incidents".to_string(),+ self.threshold_incidents.into(),+ ),+ ("budget_incidents".to_string(), self.budget_incidents.into()),+ ("stall_p50_ms".to_string(), self.stall_p50_ms.into()),+ ("stall_p95_ms".to_string(), self.stall_p95_ms.into()),+ ("stall_max_ms".to_string(), self.stall_max_ms.into()),+ (+ "report_window_seconds".to_string(),+ self.report_window_seconds.into(),+ ),+ (+ "measurement_version".to_string(),+ MEASUREMENT_VERSION.into(),+ ),+ ]),+ }+ }+}++fn to_value(incidents: &[SerializedHangIncident]) -> Value {+ match serde_json::to_value(incidents) {+ Ok(value) => value,+ Err(error) => {+ log::error!("failed to serialize hang incidents: {error}");+ Value::Null+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn reporter_keeps_largest_incidents_and_counts_all() {+ let mut reporter = Reporter::new();+ for stall_ms in 0..12 {+ let mut incident = serialized_incident(f64::from(stall_ms));+ if stall_ms % 2 == 0 {+ incident.trigger = HangTrigger::Budget;+ }+ reporter.add(incident);+ }++ let event = reporter.take_event();+ let stalls = event+ .incidents+ .iter()+ .map(|incident| incident.stall_ms)+ .collect::<Vec<_>>();++ assert_eq!(event.total_incidents, 12);+ assert_eq!(event.threshold_incidents, 6);+ assert_eq!(event.budget_incidents, 6);+ assert_eq!(event.stall_p50_ms, 5);+ assert_eq!(event.stall_p95_ms, 11);+ assert_eq!(event.stall_max_ms, 11);+ assert_eq!(+ stalls,+ vec![11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0]+ );+ }++ #[test]+ fn reporter_resets_counts_and_histogram_between_events() {+ let mut reporter = Reporter::new();+ reporter.add(serialized_incident(100.0));+ reporter.take_event();++ let mut incident = serialized_incident(20.9);+ incident.trigger = HangTrigger::Budget;+ reporter.add(incident);+ let event = reporter.take_event();++ assert_eq!(event.total_incidents, 1);+ assert_eq!(event.threshold_incidents, 0);+ assert_eq!(event.budget_incidents, 1);+ assert_eq!(event.incidents[0].stall_ms, 20.9);+ assert_eq!(event.stall_p50_ms, 20);+ assert_eq!(event.stall_p95_ms, 20);+ assert_eq!(event.stall_max_ms, 20);++ let empty = reporter.take_event();+ assert!(empty.incidents.is_empty());+ assert_eq!(empty.total_incidents, 0);+ assert_eq!(empty.threshold_incidents, 0);+ assert_eq!(empty.budget_incidents, 0);+ assert_eq!(empty.stall_p50_ms, 0);+ assert_eq!(empty.stall_p95_ms, 0);+ assert_eq!(empty.stall_max_ms, 0);+ }++ #[test]+ fn reporter_reports_empty_observation_windows() {+ let mut reporter = Reporter::new();+ let last_send = Instant::now() - Duration::from_secs(60);+ reporter.last_send = last_send;+ let event = reporter.take_event().into_flexible_event();+ let report_window_seconds = reporter.last_send.duration_since(last_send).as_secs();++ assert!(report_window_seconds >= 60);+ assert_eq!(event.event_type, "Hang Incidents");+ assert_eq!(+ serde_json::to_value(event.event_properties).unwrap(),+ json!({+ "incidents": [],+ "total_incidents": 0,+ "threshold_incidents": 0,+ "budget_incidents": 0,+ "stall_p50_ms": 0,+ "stall_p95_ms": 0,+ "stall_max_ms": 0,+ "report_window_seconds": report_window_seconds,+ "measurement_version": 2+ })+ );+ }++ #[test]+ fn hang_incidents_event_uses_expected_wire_shape() {+ let event = HangIncidentsEvent {+ incidents: vec![serialized_incident(125.0)],+ total_incidents: 3,+ threshold_incidents: 2,+ budget_incidents: 1,+ stall_p50_ms: 25,+ stall_p95_ms: 125,+ stall_max_ms: 125,+ report_window_seconds: 1800,+ }+ .into_flexible_event();++ assert_eq!(event.event_type, "Hang Incidents");+ assert_eq!(+ serde_json::to_value(event.event_properties).unwrap(),+ json!({+ "incidents": [{+ "measurement_version": 2,+ "phase": "steady",+ "trigger": "threshold",+ "start_ms": 10.0,+ "active_ms": 125.0,+ "stall_ms": 125.0,+ "dirty_to_present_ms": null,+ "sealed_by": "idle",+ "busy_fraction": 1.0,+ "event_count": 1,+ "small_poll_count": 0,+ "small_poll_total_ms": 0.0,+ "dropped_events": 0,+ "journal_discontinuous": false,+ "contributors": [],+ "contributors_elided": 0+ }],+ "total_incidents": 3,+ "threshold_incidents": 2,+ "budget_incidents": 1,+ "stall_p50_ms": 25,+ "stall_p95_ms": 125,+ "stall_max_ms": 125,+ "report_window_seconds": 1800,+ "measurement_version": 2+ })+ );+ }++ fn serialized_incident(stall_ms: f64) -> SerializedHangIncident {+ SerializedHangIncident {+ measurement_version: MEASUREMENT_VERSION,+ phase: "steady",+ trigger: HangTrigger::Threshold,+ start_ms: 10.0,+ active_ms: stall_ms,+ stall_ms,+ dirty_to_present_ms: None,+ sealed_by: "idle",+ busy_fraction: 1.0,+ event_count: 1,+ small_poll_count: 0,+ small_poll_total_ms: 0.0,+ dropped_events: 0,+ journal_discontinuous: false,+ contributors: Vec::new(),+ contributors_elided: 0,+ }+ }+}diff --git a/crates/zed/Cargo.toml b/crates/zed/Cargo.tomlindex a17c2ed31bc..5363ac60959 100644--- a/crates/zed/Cargo.toml+++ b/crates/zed/Cargo.toml@@ -128,7 +128,7 @@ gpui_platform = { workspace = true, features = ["x11",] }gpui_tokio.workspace = true-hdrhistogram.workspace = true+hang_telemetry.workspace = trueimage = { workspace = true, optional = true }spin = "0.10.0"itertools.workspace = truediff --git a/crates/zed/src/reliability/hang_detection.rs b/crates/zed/src/reliability/hang_detection.rsindex 55423269ae1..f6793cf3e7e 100644--- a/crates/zed/src/reliability/hang_detection.rs+++ b/crates/zed/src/reliability/hang_detection.rs@@ -3,16 +3,14 @@ use std::thread;use std::time::{Duration, Instant};use client::Client;-use gpui::profiler::hang::{HangDetector, SerializedHangIncident};use gpui::{AppContext, TasksIncluded, profiler};-use parking_lot::Mutex;+use hang_telemetry::HangTelemetry;use ui::App;use crate::STARTUP_TIME;mod logging;mod task_traces;-mod telemetry;gpui::actions!(dev,@@ -26,37 +24,14 @@ gpui::actions!(]);-const MAX_SERIALIZED_CONTRIBUTORS: usize = 8;-pub(crate) fn start(client: Arc<Client>, cx: &mut App) {- let hang_time = if cfg!(debug_assertions) {- if cfg!(windows) {- // yes windows debug builds are horribly slow- Duration::from_secs(30)- } else {- Duration::from_secs(5)- }- } else {- // will be lowered over time or turned into a setting- Duration::from_millis(100)- };-- let frame_budget = if cfg!(debug_assertions) {- // Unoptimized builds routinely spend more than a release frame- // budget on ordinary frames; keep dev builds from reporting- // constantly.- Duration::from_millis(100)- } else {- // At least one dropped frame on any display. Generous while budget- // incidents are plentiful; lower it as they get fixed.- Duration::from_millis(24)- };+ let hang_time = hang_telemetry::hang_threshold();if cfg!(debug_assertions) {log::warn!("debug build, only reporting hangs longer then {hang_time:?}");}- start_hang_detection(hang_time, frame_budget, client, cx);+ start_hang_detection(hang_time, client, cx);cx.on_action(move |_: &HangAction, _| {log::warn!(@@ -91,44 +66,25 @@ pub(crate) fn start(client: Arc<Client>, cx: &mut App) {});}-fn start_hang_detection(- report_longer_then: Duration,- frame_budget: Duration,- client: Arc<Client>,- cx: &App,-) {+fn start_hang_detection(report_longer_then: Duration, client: Arc<Client>, cx: &App) {let foreground_thread = thread::current().id();let monitor_interval = Duration::from_secs(1);- let telemetry = Arc::new(Mutex::new(telemetry::Reporter::new()));- let incident_detector = Arc::new(spin::Mutex::new(HangDetector::new(- cx.foreground_journal(),- report_longer_then,- frame_budget,- )));let started = Instant::now();let startup = *STARTUP_TIME.get().unwrap_or(&started);+ let hang_telemetry = Arc::new(spin::Mutex::new(HangTelemetry::new(+ cx.foreground_journal(),+ startup,+ )));let mut log = logging::Reporter::new(monitor_interval, report_longer_then, foreground_thread);cx.on_app_quit({- let telemetry = Arc::clone(&telemetry);- let incident_detector = Arc::clone(&incident_detector);+ let hang_telemetry = Arc::clone(&hang_telemetry);move |_| {- let mut incident_detector = incident_detector.lock();- let incidents = incident_detector.poll();- let first_present_at = incident_detector.first_present_at();- drop(incident_detector);-- let mut telemetry = telemetry.lock();- for incident in &incidents {- telemetry.add(SerializedHangIncident::convert(- startup,- incident,- MAX_SERIALIZED_CONTRIBUTORS,- first_present_at,- ));- }- telemetry.send();- drop(telemetry);+ let mut hang_telemetry = hang_telemetry.lock();+ hang_telemetry.collect();+ let event = hang_telemetry.take_event();+ drop(hang_telemetry);+ telemetry::send_event(event);client.telemetry().flush_events()}})@@ -147,23 +103,13 @@ fn start_hang_detection(let task_stats = profiler::take_all_stats(TasksIncluded::CompletedAndRunning);let action_stats = profiler::take_action_stats();- {- let mut incident_detector = incident_detector.lock();- let incidents = incident_detector.poll();- let first_present_at = incident_detector.first_present_at();- drop(incident_detector);-- let mut telemetry = telemetry.lock();- for incident in &incidents {- let serialized_incident = SerializedHangIncident::convert(- startup,- incident,- MAX_SERIALIZED_CONTRIBUTORS,- first_present_at,- );- telemetry.add(serialized_incident);- }- telemetry.send_periodically();+ let event = {+ let mut hang_telemetry = hang_telemetry.lock();+ hang_telemetry.collect();+ hang_telemetry.take_event_if_due()+ };+ if let Some(event) = event {+ telemetry::send_event(event);}let should_write_trace = log.check_and_report(&task_stats, &action_stats);diff --git a/crates/zed/src/reliability/hang_detection/telemetry.rs b/crates/zed/src/reliability/hang_detection/telemetry.rsdeleted file mode 100644index 32f2e08af41..00000000000--- a/crates/zed/src/reliability/hang_detection/telemetry.rs+++ /dev/null@@ -1,82 +0,0 @@-use std::time::{Duration, Instant};--use gpui::profiler::hang::{HangTrigger, SerializedHangIncident};-use hdrhistogram::Histogram;--/// Cap on incidents per telemetry event. When more accrue between sends, the-/// ones with the largest stalls are kept and the incident counts still cover-/// them all.-const MAX_REPORTED_INCIDENTS: usize = 10;--// A long interval keeps hang telemetry a small fraction of event volume even-// for pathologically hang-prone sessions; the on-quit flush covers short ones.-const SEND_INTERVAL: Duration = Duration::from_mins(30);--pub struct Reporter {- last_send: Instant,- pending: Vec<SerializedHangIncident>,- threshold_incidents: u64,- budget_incidents: u64,- // Every incident's stall, in milliseconds; `pending` keeps only the largest.- stalls: Histogram<u64>,-}--impl Reporter {- pub fn new() -> Self {- Self {- last_send: Instant::now(),- pending: Vec::new(),- threshold_incidents: 0,- budget_incidents: 0,- stalls: Histogram::new(3).expect("3 significant figures is a valid histogram"),- }- }-- pub fn add(&mut self, incident: SerializedHangIncident) {- match incident.trigger {- HangTrigger::Threshold => self.threshold_incidents += 1,- HangTrigger::Budget => self.budget_incidents += 1,- }- self.stalls.record(incident.stall_ms as u64).ok();- self.pending.push(incident);- if self.pending.len() > MAX_REPORTED_INCIDENTS {- self.pending- .sort_by(|a, b| b.stall_ms.total_cmp(&a.stall_ms));- self.pending.truncate(MAX_REPORTED_INCIDENTS);- }- }-- pub fn send_periodically(&mut self) {- if self.last_send.elapsed() > SEND_INTERVAL {- self.send();- }- }-- // Sends even without incidents so summing `report_window_seconds` gives the- // observed time when computing incident rates.- pub fn send(&mut self) {- let now = Instant::now();- let report_window_seconds = now.duration_since(self.last_send).as_secs();- self.last_send = now;- let mut incidents = std::mem::take(&mut self.pending);- incidents.sort_by(|a, b| b.stall_ms.total_cmp(&a.stall_ms));- let threshold_incidents = std::mem::take(&mut self.threshold_incidents);- let budget_incidents = std::mem::take(&mut self.budget_incidents);- // `total_incidents` predates the split; existing queries key on it.- let total_incidents = threshold_incidents + budget_incidents;-- telemetry::event!(- "Hang Incidents",- incidents,- total_incidents,- threshold_incidents,- budget_incidents,- stall_p50_ms = self.stalls.value_at_quantile(0.5),- stall_p95_ms = self.stalls.value_at_quantile(0.95),- stall_max_ms = self.stalls.max(),- report_window_seconds,- measurement_version = gpui::profiler::hang::MEASUREMENT_VERSION- );- self.stalls.reset();- }-}