Skip to content

SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.

Lost GPU: the old process exits instead of hanging in the driver, after the settings are saved (#9044) (#9054)

froglemonade2fc499441e27f
diff --git a/apps/solidcraft/src/device_lost.rs b/apps/solidcraft/src/device_lost.rs
index b892723c3c..2f598c69c1 100644
--- a/apps/solidcraft/src/device_lost.rs
+++ b/apps/solidcraft/src/device_lost.rs
@@ -5,12 +5,25 @@
//! for recovery as a forced quit does, and starts again ([`restart`]), and the new start says what
//! happened ([`note`]). A device lost again soon after such a restart starts the next one on the
//! software renderer, which doesn't depend on the graphics driver.
+//!
+//! The old process ends without tearing down its GPU objects ([`end`]): on a lost NVIDIA device
+//! destroying the window's swap chain busy-waits in the driver and never returns (#9044).
use crate::startup;
/// Set on the start after a lost device: when the device was lost (seconds since 1970).
pub const LOST_ENV: &str = "SOLIDCRAFT_DEVICE_LOST";
+/// Set on the start after a lost device: why it was lost, as wgpu said it. The old process's
+/// session journal, which recorded it, is gone by then.
+pub const REASON_ENV: &str = "SOLIDCRAFT_DEVICE_LOST_REASON";
+
+/// The longest reason passed on ([`REASON_ENV`]) and shown, in characters.
+const REASON_CHARS: usize = 300;
+
+/// How long the process may take to end ([`end`]) before it is aborted.
+const END_GRACE: std::time::Duration = std::time::Duration::from_secs(3);
+
/// A loss this recent (seconds) counts for the start after it: its note, and a second loss
/// within it starts on the software renderer. Inherited by later starts, an older stamp is
/// ignored.
@@ -32,27 +45,54 @@ pub fn restart_on_software(after_recent_loss: bool, cpu_renderer: bool) -> bool
after_recent_loss || cpu_renderer
}
+/// `reason` on one line, at most [`REASON_CHARS`] long; None when there is nothing to say.
+pub fn reason(reason: Option<&str>) -> Option<String> {
+ let line = reason?.split_whitespace().collect::<Vec<_>>().join(" ");
+ (!line.is_empty()).then(|| line.chars().take(REASON_CHARS).collect())
+}
+
/// What the start after a lost device tells the user, `software` when it runs on the software
-/// renderer.
-pub fn note(software: bool) -> String {
+/// renderer, with what the driver reported ([`REASON_ENV`]).
+pub fn note(software: bool, reason: Option<&str>) -> String {
let mut note = String::from(
"The graphics device was reset (the graphics driver stopped responding), so SolidCraft closed and started again. Unsaved changes were kept for recovery.",
);
if software {
note.push_str(" This start uses software rendering, which doesn't depend on the graphics driver.");
}
+ if let Some(reason) = self::reason(reason) {
+ note.push_str(&format!(" The graphics driver reported: {reason}"));
+ }
note
}
-/// Start SolidCraft again with the same arguments, the loss stamped at `lost_at`, on the software
-/// renderer when `software`.
-pub fn restart(software: bool, lost_at: u64) {
- let Ok(exe) = std::env::current_exe() else { return };
+/// The command that starts `exe` again with `args`, the loss stamped at `lost_at` and its
+/// `reason`, on the software renderer when `software`.
+fn command(
+ exe: std::path::PathBuf,
+ args: impl IntoIterator<Item = std::ffi::OsString>,
+ software: bool,
+ lost_at: u64,
+ reason: &str,
+) -> std::process::Command {
let mut command = std::process::Command::new(exe);
- command.args(std::env::args_os().skip(1)).env(LOST_ENV, lost_at.to_string());
+ command.args(args).env(LOST_ENV, lost_at.to_string());
+ match self::reason(Some(reason)) {
+ Some(reason) => command.env(REASON_ENV, reason),
+ // Not one inherited from an earlier loss.
+ None => command.env_remove(REASON_ENV),
+ };
if software {
command.env(startup::RENDERER_ENV, "cpu");
}
+ command
+}
+
+/// Start SolidCraft again with the same arguments, the loss stamped at `lost_at` and its `reason`,
+/// on the software renderer when `software`.
+pub fn restart(software: bool, lost_at: u64, reason: &str) {
+ let Ok(exe) = std::env::current_exe() else { return };
+ let mut command = command(exe, std::env::args_os().skip(1), software, lost_at, reason);
#[cfg(windows)]
{
use std::os::windows::process::CommandExt;
@@ -64,6 +104,23 @@ pub fn restart(software: bool, lost_at: u64) {
}
}
+/// End the process now, without tearing down what is left on the lost device: the window's swap
+/// chain, the viewport's buffers and pipelines. On NVIDIA's Vulkan driver destroying the swap
+/// chain of a lost device busy-waits forever, which left the old window frozen at 100% of a core
+/// beside the new one (#9044). Should the exit itself get stuck in a library's exit handler, the
+/// process is aborted after [`END_GRACE`].
+pub fn end() -> ! {
+ let watchdog = std::thread::Builder::new().name("solidcraft-exit".into()).spawn(|| {
+ std::thread::sleep(END_GRACE);
+ eprintln!("SolidCraft graphics: still closing after the device was lost; aborting");
+ std::process::abort();
+ });
+ if let Err(e) = watchdog {
+ log::warn!("graphics: no exit watchdog: {e}");
+ }
+ std::process::exit(0)
+}
+
#[cfg(test)]
mod tests {
/// Only a recent, well-formed stamp counts: a later start that inherited it says nothing.
@@ -84,8 +141,34 @@ mod tests {
assert!(!super::restart_on_software(false, false));
assert!(super::restart_on_software(true, false));
assert!(super::restart_on_software(false, true));
- assert!(super::note(true).contains("software rendering"));
- assert!(!super::note(false).contains("software rendering"));
- assert!(super::note(false).contains("kept for recovery"));
+ assert!(super::note(true, None).contains("software rendering"));
+ assert!(!super::note(false, None).contains("software rendering"));
+ assert!(super::note(false, None).contains("kept for recovery"));
+ }
+
+ /// The start after a loss is told why (#9044): the old process's journal held the reason, and
+ /// it is deleted as that process closes.
+ #[test]
+ fn the_next_start_is_told_why_the_device_was_lost() {
+ let exe = std::path::PathBuf::from("solidcraft");
+ let args = ["--control".into(), "7979".into()];
+ let command = super::command(exe.clone(), args, true, 1_000, "Lost: Device lost\n (vkQueueSubmit)");
+ let env = |command: &std::process::Command, name: &str| {
+ command.get_envs().find(|(k, _)| *k == name).map(|(_, v)| v.map(|v| v.to_string_lossy().into_owned()))
+ };
+ assert_eq!(env(&command, super::LOST_ENV), Some(Some("1000".into())));
+ assert_eq!(env(&command, super::REASON_ENV), Some(Some("Lost: Device lost (vkQueueSubmit)".into())), "on one line");
+ assert_eq!(env(&command, crate::startup::RENDERER_ENV), Some(Some("cpu".into())));
+ assert_eq!(command.get_args().collect::<Vec<_>>(), ["--control", "7979"]);
+ // No reason: none inherited from an earlier loss either.
+ let command = super::command(exe, Vec::new(), false, 1_000, " ");
+ assert_eq!(env(&command, super::REASON_ENV), Some(None));
+ assert_eq!(env(&command, crate::startup::RENDERER_ENV), None);
+
+ let note = super::note(false, Some("Lost: Device lost"));
+ assert!(note.ends_with("The graphics driver reported: Lost: Device lost"), "{note}");
+ assert!(!super::note(false, Some(" \n ")).contains("reported"));
+ let long = "x".repeat(1_000);
+ assert_eq!(super::reason(Some(&long)).map(|r| r.chars().count()), Some(300));
}
}
diff --git a/apps/solidcraft/src/main.rs b/apps/solidcraft/src/main.rs
index d9c6fbc9f6..c4cfcf40f8 100644
--- a/apps/solidcraft/src/main.rs
+++ b/apps/solidcraft/src/main.rs
@@ -204,13 +204,29 @@ impl eframe::App for App {
}
solidcraft_ui_egui::recovery_ui::close(&mut self.solve);
solidcraft_ui_egui::open_elsewhere::close(&mut self.solve);
- // Before a restart below starts the next instance, which would otherwise get nothing from
- // the 3D mouse driver (`solidcraft_3dx_client::stop`).
+ // Before a restart (`Drop for App`) starts the next instance, which would otherwise get
+ // nothing from the 3D mouse driver (`solidcraft_3dx_client::stop`).
#[cfg(target_os = "macos")]
solidcraft_3dx_client::stop();
solidcraft_engine::crash::end_session();
- if let Some(lost_at) = self.gpu.as_ref().and_then(|g| g.lost_at) {
- device_lost::restart(self.gpu.as_ref().is_some_and(|g| g.restart_on_software), lost_at);
+ // The next start (after a lost device, or into an update) begins once the app is dropped
+ // (`Drop for App`), when the settings are on disk.
+ }
+}
+
+impl Drop for App {
+ /// eframe drops its settings storage first, which waits for `app.ron` to be written, then the
+ /// app, then the window and its GPU surface. So the next start begins here: started from
+ /// `on_exit`, it read `app.ron` while the old process was still writing it, found it
+ /// unreadable and moved it aside, resetting every setting (#9044).
+ fn drop(&mut self) {
+ if let Some(gpu) = &self.gpu
+ && let Some(lost_at) = gpu.lost_at
+ {
+ device_lost::restart(gpu.restart_on_software, lost_at, &gpu.reason);
+ // The window's GPU surface is still to be dropped, and on a lost device that may
+ // never return (#9044).
+ device_lost::end();
}
// An update asked to restart into the new version or to run its installer.
if let Some((program, args)) = solidcraft_ui_egui::update::take_launch(&mut self.solve) {
@@ -238,6 +254,8 @@ struct LostWatch {
restart_on_software: bool,
/// When the loss was seen: the app is closing to start again.
lost_at: Option<u64>,
+ /// Why, as wgpu said it: passed to the next start ([`device_lost::REASON_ENV`]).
+ reason: String,
}
impl App {
@@ -251,6 +269,7 @@ impl App {
eprintln!("SolidCraft graphics: the device was lost ({reason}); starting again");
solidcraft_engine::crash::phase(&format!("graphics device lost: {reason}"));
gpu.lost_at = Some(device_lost::now_secs());
+ gpu.reason = reason;
self.solve.quit_requested = true;
}
}
@@ -839,7 +858,11 @@ fn main() -> eframe::Result {
// The launch before this one closed because its graphics device was lost (`App::watch_device`).
let after_loss = device_lost::recent(std::env::var(device_lost::LOST_ENV).ok().as_deref(), device_lost::now_secs());
if after_loss {
- notes.push(device_lost::note(cpu_renderer));
+ let reason = std::env::var(device_lost::REASON_ENV).ok();
+ if let Some(reason) = device_lost::reason(reason.as_deref()) {
+ log::warn!("graphics: the last launch lost its device ({reason})");
+ }
+ notes.push(device_lost::note(cpu_renderer, reason.as_deref()));
}
if safe_mode {
notes.push("Safe mode: default settings (yours are kept for the next normal start), no recovery offer, software rendering.".into());
@@ -979,6 +1002,7 @@ fn main() -> eframe::Result {
lost: rs.device_lost.clone(),
restart_on_software: device_lost::restart_on_software(after_loss, cpu_renderer),
lost_at: None,
+ reason: String::new(),
});
if !safe_mode {
app.preferences.graphics_adapter =
diff --git a/docs/book/src/guide/troubleshooting.md b/docs/book/src/guide/troubleshooting.md
index edfe0c9b49..685ee7ca21 100644
--- a/docs/book/src/guide/troubleshooting.md
+++ b/docs/book/src/guide/troubleshooting.md
@@ -158,9 +158,12 @@ When the graphics driver stops responding, the system resets it (on Windows, "Di
stopped responding and has recovered"), and the graphics device SolidCraft draws with is gone
for good: the window can't draw anything more. SolidCraft then closes, keeps your unsaved
changes for recovery as it does after a crash, and starts again; the new window says what
-happened and offers the changes back. If the device is reset again within ten minutes of that
-restart, the next start uses software rendering, which doesn't depend on the graphics driver.
-Updating the graphics driver from the GPU maker usually stops the resets.
+happened, with what the graphics driver reported, and offers the changes back. The old window
+closes on its own; SolidCraft doesn't wait for the driver to release it. If the device is reset
+again within ten minutes of that restart, the next start uses software rendering, which doesn't
+depend on the graphics driver. Updating the graphics driver from the GPU maker usually stops the
+resets. Laptops that lose the device on waking from sleep (often with an NVIDIA GPU that doesn't
+drive the screen) can keep SolidCraft on the integrated GPU instead: Preferences › Graphics.
## A design saved by 0.10 won't open in an older version