Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
Add a GPUI-owned hang monitor and make hang_telemetry a consumer (#64944)
Anthony Eid0f9c923e674e
diff --git a/Cargo.lock b/Cargo.lockindex d4f8f75ca4c..e54024c2e27 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -22911,7 +22911,6 @@ dependencies = ["smol","snippet_provider","snippets_ui",- "spin 0.10.0","svg_preview","sysinfo 0.39.6","system_specs",diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rsindex 2a05f8d17c0..24e0ea05daf 100644--- a/crates/gpui/src/app.rs+++ b/crates/gpui/src/app.rs@@ -756,6 +756,8 @@ pub struct App {pub(crate) foreground_executor: ForegroundExecutor,#[cfg(feature = "profiler")]foreground_journal: crate::profiler::journal::ForegroundJournal,+ #[cfg(all(feature = "profiler", not(target_family = "wasm")))]+ hang_monitor: Option<crate::profiler::hang::HangMonitor>,pub(crate) entities: EntityMap,pub(crate) new_entity_observers: SubscriberSet<TypeId, NewEntityListener>,pub(crate) windows: SlotMap<WindowId, Option<Box<Window>>>,@@ -882,6 +884,8 @@ impl App {foreground_executor,#[cfg(feature = "profiler")]foreground_journal,+ #[cfg(all(feature = "profiler", not(target_family = "wasm")))]+ hang_monitor: None,svg_renderer: SvgRenderer::new(asset_source.clone()),loading_assets: Default::default(),asset_source,@@ -1058,12 +1062,32 @@ impl App {/// [`SHUTDOWN_TIMEOUT`] to complete. WebAssembly runs them asynchronously as best-effort cleanup/// because its event-loop thread cannot block.pub fn shutdown(&mut self) {- let mut futures = Vec::new();+ // Requested first so the final hang poll overlaps with the quit+ // handlers. It's awaited alongside them, within `SHUTDOWN_TIMEOUT`.+ #[cfg(all(feature = "profiler", not(target_family = "wasm")))]+ let hang_monitor_flush = self+ .hang_monitor+ .as_ref()+ .and_then(|hang_monitor| hang_monitor.request_flush());++ let mut futures: Vec<LocalBoxFuture<'static, ()>> = Vec::new();for observer in self.quit_observers.remove(&()) {futures.push(observer(self));}+ #[cfg(all(feature = "profiler", not(target_family = "wasm")))]+ if let Some(flushed) = hang_monitor_flush {+ futures.push(+ async move {+ if flushed.await.is_err() {+ log::warn!("hang monitor exited before flushing");+ }+ }+ .boxed_local(),+ );+ }+self.windows.clear();self.window_handles.clear();self.flush_effects();@@ -2032,6 +2056,45 @@ impl App {self.foreground_journal.clone()}+ /// Starts detecting foreground hangs on a dedicated thread.+ ///+ /// Nothing is spawned unless the app calls this. The thread polls a+ /// detector over this app's foreground journal every `config.interval`+ /// and passes each poll's incidents, including empty polls, to `on_poll`+ /// on that thread. When the app quits, a final poll with+ /// [`HangMonitorPollReason::Flush`] runs during shutdown, concurrently+ /// with quit handlers and within [`SHUTDOWN_TIMEOUT`], so `on_poll` can+ /// deliver batched results.+ ///+ /// # Errors+ ///+ /// Fails if the monitor was already started or its thread can't be+ /// spawned.+ ///+ /// [`HangMonitorPollReason::Flush`]: crate::profiler::hang::HangMonitorPollReason::Flush+ #[cfg(all(feature = "profiler", not(target_family = "wasm")))]+ pub fn start_hang_monitor(+ &mut self,+ config: crate::profiler::hang::HangMonitorConfig,+ on_poll: impl FnMut(crate::profiler::hang::HangMonitorPoll) + Send + 'static,+ ) -> Result<(), crate::profiler::hang::HangMonitorError> {+ use crate::profiler::hang::{HangDetector, HangMonitor, HangMonitorError};++ if self.hang_monitor.is_some() {+ debug_assert!(false, "the hang monitor was started twice");+ return Err(HangMonitorError::AlreadyStarted);+ }+ let detector = HangDetector::new(+ self.foreground_journal(),+ config.threshold,+ config.frame_budget,+ );+ let monitor = HangMonitor::spawn(detector, config.interval, on_poll)+ .map_err(HangMonitorError::Spawn)?;+ self.hang_monitor = Some(monitor);+ Ok(())+ }+/// Spawns the future returned by the given function on the main thread. The closure will be invoked/// with [AsyncApp], which allows the application state to be accessed across await points.#[track_caller]diff --git a/crates/gpui/src/profiler/hang.rs b/crates/gpui/src/profiler/hang.rsindex 23d63aa658b..e251f64176b 100644--- a/crates/gpui/src/profiler/hang.rs+++ b/crates/gpui/src/profiler/hang.rs@@ -15,6 +15,13 @@ use std::time::Duration;use scheduler::Instant;use serde::Serialize;+#[cfg(not(target_family = "wasm"))]+mod monitor;+#[cfg(not(target_family = "wasm"))]+pub(crate) use monitor::HangMonitor;+#[cfg(not(target_family = "wasm"))]+pub use monitor::{HangMonitorConfig, HangMonitorError, HangMonitorPoll, HangMonitorPollReason};+/// Version of the power/visibility-aware measurement rules.pub const MEASUREMENT_VERSION: u32 = 2;@@ -629,6 +636,106 @@ mod tests {));}+ #[test]+ fn hang_monitor_polls_on_its_interval_and_on_flush() {+ use super::{HangMonitor, HangMonitorPollReason};++ let (journal, _guard) = install_test_foreground_journal(64, 4);+ // Each monitor reports on its own channel, which disconnects once the+ // monitor's thread has exited and dropped its callback.+ let spawn_monitor = |journal, interval| {+ let (sender, receiver) = std::sync::mpsc::channel();+ let monitor = HangMonitor::spawn(+ HangDetector::new(journal, HANG_THRESHOLD, FRAME_BUDGET),+ interval,+ move |poll| {+ if !poll.incidents.is_empty() {+ sender.send((poll.reason, poll.incidents.len())).ok();+ }+ },+ )+ .expect("spawn monitor thread");+ (monitor, receiver)+ };++ let (monitor, receiver) = spawn_monitor(journal.clone(), Duration::from_millis(10));+ simulate_blocked_foreground_poll(HANG_THRESHOLD * 2);+ assert_eq!(+ receiver.recv_timeout(Duration::from_secs(10)).ok(),+ Some((HangMonitorPollReason::Interval, 1)),+ );+ drop(monitor);+ assert_eq!(+ receiver.recv_timeout(Duration::from_secs(10)),+ Err(std::sync::mpsc::RecvTimeoutError::Disconnected),+ );++ // An interval longer than the test isolates the flush path.+ let (monitor, receiver) = spawn_monitor(journal, Duration::from_secs(3600));+ simulate_blocked_foreground_poll(HANG_THRESHOLD * 2);+ let flushed = monitor.request_flush().expect("monitor thread is running");+ futures::executor::block_on(flushed).expect("flush completes");+ assert_eq!(+ receiver.try_recv().ok(),+ Some((HangMonitorPollReason::Flush, 1)),+ );++ drop(monitor);+ assert_eq!(+ receiver.recv_timeout(Duration::from_secs(10)),+ Err(std::sync::mpsc::RecvTimeoutError::Disconnected),+ );+ }++ #[gpui::test]+ fn app_flushes_its_hang_monitor_on_shutdown(cx: &mut TestAppContext) {+ use super::{HangMonitorConfig, HangMonitorPollReason};++ let (sender, receiver) = std::sync::mpsc::channel();+ cx.update(|cx| {+ cx.start_hang_monitor(+ HangMonitorConfig {+ threshold: HANG_THRESHOLD,+ frame_budget: FRAME_BUDGET,+ // Longer than the test, so only shutdown can poll.+ interval: Duration::from_secs(3600),+ },+ move |poll| {+ sender.send(poll.reason).ok();+ },+ )+ .expect("start hang monitor");+ });+ assert!(receiver.try_recv().is_err());++ cx.update(|cx| cx.shutdown());+ assert_eq!(receiver.try_recv().ok(), Some(HangMonitorPollReason::Flush));+ }++ #[gpui::test]+ fn starting_the_hang_monitor_twice_is_rejected(cx: &mut TestAppContext) {+ use super::{HangMonitorConfig, HangMonitorError};++ let config = HangMonitorConfig {+ threshold: HANG_THRESHOLD,+ frame_budget: FRAME_BUDGET,+ interval: Duration::from_secs(3600),+ };+ cx.update(|cx| {+ cx.start_hang_monitor(config, |_| {})+ .expect("start hang monitor");+ let second = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {+ cx.start_hang_monitor(config, |_| {})+ }));+ // Debug builds also assert, so misuse is caught during development.+ if cfg!(debug_assertions) {+ assert!(second.is_err());+ } else {+ assert!(matches!(second, Ok(Err(HangMonitorError::AlreadyStarted))));+ }+ });+ }+#[test]fn serialized_incident_reports_idle_seal_fields() {let startup = scheduler::Instant::now();diff --git a/crates/gpui/src/profiler/hang/monitor.rs b/crates/gpui/src/profiler/hang/monitor.rsnew file mode 100644index 00000000000..566b32d3dc0--- /dev/null+++ b/crates/gpui/src/profiler/hang/monitor.rs@@ -0,0 +1,113 @@+use std::sync::mpsc::{self, RecvTimeoutError};+use std::time::Duration;++use futures::channel::oneshot;+use scheduler::Instant;++use super::{HangDetector, HangIncident};++/// Detection thresholds and polling cadence for+/// [`crate::App::start_hang_monitor`].+#[derive(Debug, Clone, Copy)]+pub struct HangMonitorConfig {+ /// Duration at which a single piece of foreground work counts as a hang.+ pub threshold: Duration,+ /// Total foreground spend within one interval that counts as a hang.+ pub frame_budget: Duration,+ /// How often the monitor thread drains the journal.+ pub interval: Duration,+}++/// Why [`crate::App::start_hang_monitor`] failed.+#[derive(Debug, thiserror::Error)]+pub enum HangMonitorError {+ /// The monitor was already started for this app.+ #[error("the hang monitor was already started")]+ AlreadyStarted,+ /// The monitor thread couldn't be spawned.+ #[error("failed to spawn the hang monitor thread: {0}")]+ Spawn(#[source] std::io::Error),+}++/// Why a [`HangMonitor`] polled its detector.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum HangMonitorPollReason {+ /// The monitor's regular interval elapsed.+ Interval,+ /// A poll was requested before the application quits. Consumers should+ /// deliver any batched results now.+ Flush,+}++/// The incidents a [`HangMonitor`] collected in one poll.+pub struct HangMonitorPoll {+ /// Incidents sealed since the previous poll, possibly none.+ pub incidents: Vec<HangIncident>,+ /// See [`HangDetector::first_present_at`].+ pub first_present_at: Option<Instant>,+ /// Why this poll happened.+ pub reason: HangMonitorPollReason,+}++enum Request {+ Flush { done: oneshot::Sender<()> },+}++/// Polls a [`HangDetector`] on a dedicated OS thread.+///+/// Detection runs off the foreground thread so incidents are still collected+/// and reported while GPUI's executors are stalled. The thread owns the+/// detector and the callback; other threads only send it requests. Dropping+/// the monitor stops the thread.+pub(crate) struct HangMonitor {+ requests: mpsc::Sender<Request>,+}++impl HangMonitor {+ /// Starts a thread that polls `detector` every `interval` and passes each+ /// poll's result, including empty ones, to `on_poll` on that thread.+ ///+ /// # Errors+ ///+ /// Returns an error when the thread can't be spawned.+ pub(crate) fn spawn<F>(+ mut detector: HangDetector,+ interval: Duration,+ mut on_poll: F,+ ) -> std::io::Result<Self>+ where+ F: FnMut(HangMonitorPoll) + Send + 'static,+ {+ let (requests, receiver) = mpsc::channel();+ std::thread::Builder::new()+ .name("HangDetection".to_string())+ .spawn(move || {+ loop {+ let (reason, done) = match receiver.recv_timeout(interval) {+ Ok(Request::Flush { done }) => (HangMonitorPollReason::Flush, Some(done)),+ Err(RecvTimeoutError::Timeout) => (HangMonitorPollReason::Interval, None),+ Err(RecvTimeoutError::Disconnected) => break,+ };+ let incidents = detector.poll();+ on_poll(HangMonitorPoll {+ incidents,+ first_present_at: detector.first_present_at(),+ reason,+ });+ if let Some(done) = done {+ done.send(()).ok();+ }+ }+ })?;+ Ok(Self { requests })+ }++ /// Asks the monitor thread to poll now with [`HangMonitorPollReason::Flush`]+ /// without waiting for it. The returned receiver completes once the+ /// callback has returned. Returns `None` if the thread has exited.+ pub(crate) fn request_flush(&self) -> Option<oneshot::Receiver<()>> {+ let (done, finished) = oneshot::channel();+ self.requests.send(Request::Flush { done }).ok()?;+ Some(finished)+ }+}diff --git a/crates/hang_telemetry/src/hang_telemetry.rs b/crates/hang_telemetry/src/hang_telemetry.rsindex c3b5769e0b4..97ed89f2614 100644--- a/crates/hang_telemetry/src/hang_telemetry.rs+++ b/crates/hang_telemetry/src/hang_telemetry.rs@@ -1,18 +1,20 @@//! Turns GPUI foreground hang incidents into batched "Hang Incidents" telemetry.//!-//! [`HangTelemetry`] polls a [`HangDetector`] over the app's foreground journal,-//! serializes each incident, and batches them into periodic telemetry events.-//! Applications own delivery: they send the returned [`FlexibleEvent`] through-//! their own telemetry pipeline, so every GPUI app that uses this crate reports-//! hangs with the same thresholds and wire schema.+//! [`HangTelemetry`] consumes the polls of GPUI's hang monitor+//! ([`gpui::App::start_hang_monitor`]), serializes each incident, and batches+//! incidents into periodic telemetry events. Applications own delivery by+//! supplying a sink for the resulting [`FlexibleEvent`]s, so every GPUI app+//! that uses this crate reports hangs with the same thresholds and wire+//! schema.use std::collections::HashMap;use std::time::{Duration, Instant};+use gpui::App;use gpui::profiler::hang::{- HangDetector, HangTrigger, MEASUREMENT_VERSION, SerializedHangIncident,+ HangMonitorConfig, HangMonitorError, HangMonitorPoll, HangMonitorPollReason, HangTrigger,+ MEASUREMENT_VERSION, SerializedHangIncident,};-use gpui::profiler::journal::ForegroundJournal;use hdrhistogram::Histogram;use serde_json::Value;use telemetry_events::FlexibleEvent;@@ -30,6 +32,8 @@ const MAX_REPORTED_INCIDENTS: usize = 10;// for pathologically hang-prone sessions; the on-quit flush covers short ones.const SEND_INTERVAL: Duration = Duration::from_mins(30);+const MONITOR_INTERVAL: Duration = Duration::from_secs(1);+/// Duration at which a single piece of foreground work counts as a hang.pub fn hang_threshold() -> Duration {if cfg!(debug_assertions) {@@ -58,66 +62,91 @@ pub fn frame_budget() -> Duration {}}-/// Detects foreground hangs and batches them into telemetry events.+/// The monitor configuration every app using this crate reports with.+pub fn monitor_config() -> HangMonitorConfig {+ HangMonitorConfig {+ threshold: hang_threshold(),+ frame_budget: frame_budget(),+ interval: MONITOR_INTERVAL,+ }+}++/// Serializes and batches hang incidents into "Hang Incidents" events.///-/// Polling and sending are driven by the caller, typically from a dedicated-/// OS thread so reporting keeps working while GPUI's executors are stalled.+/// Every 30 minutes, and when the app quits, the current batch is passed to+/// the event sink, including empty batches, whose `report_window_seconds`+/// still counts toward observed time.pub struct HangTelemetry {- detector: HangDetector,- reporter: Reporter,startup: Instant,+ reporter: Reporter,+ send_event: Box<dyn Fn(FlexibleEvent) + Send>,+ observe_incidents: Option<Box<dyn FnMut(&[SerializedHangIncident]) + Send>>,}impl HangTelemetry {- /// Starts observing `journal` with [`hang_threshold`] and [`frame_budget`].- ///- /// Incident timestamps are reported relative to `startup`.- pub fn new(journal: ForegroundJournal, startup: Instant) -> Self {+ /// Creates telemetry that reports incident timestamps relative to+ /// `startup` and delivers events through `send_event`, which is called on+ /// the monitor thread and must not block.+ pub fn new(startup: Instant, send_event: impl Fn(FlexibleEvent) + Send + 'static) -> Self {Self {- detector: HangDetector::new(journal, hang_threshold(), frame_budget()),- reporter: Reporter::new(),startup,+ reporter: Reporter::new(),+ send_event: Box::new(send_event),+ observe_incidents: None,}}- /// Adds incidents sealed since the previous call to the pending batch.- ///- /// Returns the newly collected incidents so callers can also surface them- /// elsewhere, such as in feedback reports.- pub fn collect(&mut self) -> Vec<SerializedHangIncident> {- let incidents = self.detector.poll();- let first_present_at = self.detector.first_present_at();- let serialized_incidents = incidents- .iter()- .map(|incident| {- SerializedHangIncident::convert(- self.startup,- incident,- MAX_SERIALIZED_CONTRIBUTORS,- first_present_at,- )- })- .collect::<Vec<_>>();- for incident in &serialized_incidents {- self.reporter.add(incident.clone());- }- serialized_incidents+ /// Also passes each poll's serialized incidents to `observer` on the+ /// monitor thread, e.g. to attach recent hangs to feedback reports.+ pub fn with_incident_observer(+ mut self,+ observer: impl FnMut(&[SerializedHangIncident]) + Send + 'static,+ ) -> Self {+ self.observe_incidents = Some(Box::new(observer));+ self}- /// Takes the pending batch as an event once the send interval has elapsed.- pub fn take_event_if_due(&mut self) -> Option<FlexibleEvent> {- (self.reporter.last_send.elapsed() > SEND_INTERVAL).then(|| self.take_event())+ /// Starts `cx`'s hang monitor with [`monitor_config`], reporting its+ /// polls through this telemetry.+ ///+ /// # Errors+ ///+ /// See [`App::start_hang_monitor`].+ pub fn start(mut self, cx: &mut App) -> Result<(), HangMonitorError> {+ cx.start_hang_monitor(monitor_config(), move |poll| self.handle_poll(poll))}- /// Takes the pending batch as an event, even when it has no incidents.- ///- /// Empty events still matter: summing `report_window_seconds` gives the- /// observed time when computing incident rates.- pub fn take_event(&mut self) -> FlexibleEvent {- self.reporter.take_event().into_flexible_event()+ fn handle_poll(&mut self, poll: HangMonitorPoll) {+ let flush = poll.reason == HangMonitorPollReason::Flush;+ let incidents = serialize_incidents(self.startup, poll);+ if !incidents.is_empty()+ && let Some(observe_incidents) = self.observe_incidents.as_mut()+ {+ observe_incidents(&incidents);+ }+ for incident in incidents {+ self.reporter.add(incident);+ }+ if flush || self.reporter.last_send.elapsed() > SEND_INTERVAL {+ (self.send_event)(self.reporter.take_event().into_flexible_event());+ }}}+fn serialize_incidents(startup: Instant, poll: HangMonitorPoll) -> Vec<SerializedHangIncident> {+ poll.incidents+ .iter()+ .map(|incident| {+ SerializedHangIncident::convert(+ startup,+ incident,+ MAX_SERIALIZED_CONTRIBUTORS,+ poll.first_present_at,+ )+ })+ .collect()+}+struct Reporter {last_send: Instant,pending: Vec<SerializedHangIncident>,diff --git a/crates/zed/Cargo.toml b/crates/zed/Cargo.tomlindex 5363ac60959..f759f2fc937 100644--- a/crates/zed/Cargo.toml+++ b/crates/zed/Cargo.toml@@ -130,7 +130,6 @@ gpui_platform = { workspace = true, features = [gpui_tokio.workspace = truehang_telemetry.workspace = trueimage = { workspace = true, optional = true }-spin = "0.10.0"itertools.workspace = truerayon.workspace = truesemver.workspace = truediff --git a/crates/zed/src/reliability/hang_detection.rs b/crates/zed/src/reliability/hang_detection.rsindex f6793cf3e7e..6d3d36242fd 100644--- a/crates/zed/src/reliability/hang_detection.rs+++ b/crates/zed/src/reliability/hang_detection.rs@@ -66,34 +66,26 @@ pub(crate) fn start(client: Arc<Client>, cx: &mut App) {});}-fn start_hang_detection(report_longer_then: Duration, client: Arc<Client>, cx: &App) {+fn start_hang_detection(report_longer_then: Duration, client: Arc<Client>, cx: &mut App) {let foreground_thread = thread::current().id();let monitor_interval = Duration::from_secs(1);let started = Instant::now();let startup = *STARTUP_TIME.get().unwrap_or(&started);- let hang_telemetry = Arc::new(spin::Mutex::new(HangTelemetry::new(- cx.foreground_journal(),- startup,- )));- let mut log = logging::Reporter::new(monitor_interval, report_longer_then, foreground_thread);-- cx.on_app_quit({- let hang_telemetry = Arc::clone(&hang_telemetry);- move |_| {- let mut hang_telemetry = hang_telemetry.lock();- hang_telemetry.collect();- let event = hang_telemetry.take_event();- drop(hang_telemetry);- telemetry::send_event(event);- client.telemetry().flush_events()- }- })- .detach();+ // GPUI's final `Flush` poll runs during shutdown, concurrently with this+ // handler and within `SHUTDOWN_TIMEOUT`, so the last batch may miss this+ // flush.+ match HangTelemetry::new(startup, telemetry::send_event).start(cx) {+ Ok(()) => cx+ .on_app_quit(move |_| client.telemetry().flush_events())+ .detach(),+ Err(error) => log::error!("failed to start hang reporting: {error}"),+ }- // an OS thread to insulate detection and reporting from hangs on the fore- // or background.+ let mut log = logging::Reporter::new(monitor_interval, report_longer_then, foreground_thread);+ // An OS thread keeps the legacy hang logs and task traces working while+ // the foreground or background executors are hung.thread::Builder::new()- .name("HangDetection".to_string())+ .name("HangLogging".to_string()).spawn(move || {// allow "bad" tasks during startup. Not because we should but since here// they are not observed by the user and to lower on clutter from the reporter@@ -103,15 +95,6 @@ fn start_hang_detection(report_longer_then: Duration, client: Arc<Client>, cx: &let task_stats = profiler::take_all_stats(TasksIncluded::CompletedAndRunning);let action_stats = profiler::take_action_stats();- let event = {- let mut hang_telemetry = hang_telemetry.lock();- hang_telemetry.collect();- hang_telemetry.take_event_if_due()- };- if let Some(event) = event {- telemetry::send_event(event);- }-let should_write_trace = log.check_and_report(&task_stats, &action_stats);if should_write_trace {if let Some(path) = task_traces::save_any(foreground_thread) {