Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

test: real-time machine-freeze sentinel in the stimulus process

Richie McIlroyb59f61b5bc84
diff --git a/crates/verify-stimulus/src/main.rs b/crates/verify-stimulus/src/main.rs
index 5e783bca6..528d9692f 100644
--- a/crates/verify-stimulus/src/main.rs
+++ b/crates/verify-stimulus/src/main.rs
@@ -79,6 +79,7 @@ mod app {
let log = Arc::new(Log {
file: Mutex::new(File::create(&log_path)?),
});
+ let sentinel = Sentinel::start(log_path.with_file_name(SENTINEL_FILE));
use winit::platform::macos::{ActivationPolicy, EventLoopBuilderExtMacOS};
let event_loop = EventLoop::<Command>::with_user_event()
@@ -135,12 +136,79 @@ mod app {
result?;
drop(observer);
drop(audio);
+ sentinel.finish()?;
if let Some(error) = app.error {
anyhow::bail!(error);
}
Ok(())
}
+ use cap_verify::stimulus::{SENTINEL_FILE, SENTINEL_PERIOD_SECS};
+
+ /// A real-time thread that only wakes every `SENTINEL_PERIOD_SECS` and
+ /// records each wake (host ticks), for the whole life of the stimulus.
+ /// At priority 97 it preempts every timeshare thread of every process,
+ /// so a long gap between its wakes means the machine itself did not run
+ /// (the hypervisor descheduled the virtual CPUs, or the kernel did not
+ /// schedule), not that the recorder was busy.
+ struct Sentinel {
+ stop: Arc<std::sync::atomic::AtomicBool>,
+ thread: std::thread::JoinHandle<(sched::Realtime, Option<sched::ThreadState>, Vec<u64>)>,
+ path: PathBuf,
+ }
+
+ impl Sentinel {
+ fn start(path: PathBuf) -> Self {
+ let stop = Arc::new(std::sync::atomic::AtomicBool::new(false));
+ let thread = {
+ let stop = stop.clone();
+ std::thread::Builder::new()
+ .name("verify-sentinel".into())
+ .spawn(move || {
+ let realtime = sched::request_realtime(SENTINEL_PERIOD_SECS, 0.0005);
+ let me = sched::current_thread();
+ let mut wakes = Vec::with_capacity(64 * 1024);
+ let mut last = host_now();
+ wakes.push(last);
+ while !stop.load(Ordering::Acquire) {
+ sched::wait_until(host_add_secs(last, SENTINEL_PERIOD_SECS));
+ last = host_now();
+ wakes.push(last);
+ }
+ (realtime, sched::thread_state(me), wakes)
+ })
+ .expect("spawn sentinel")
+ };
+ Self { stop, thread, path }
+ }
+
+ /// Writes a JSON header line (scheduling, timebase, first wake),
+ /// then one line per wake: ticks since the previous wake.
+ fn finish(self) -> anyhow::Result<()> {
+ self.stop.store(true, Ordering::Release);
+ let (realtime, end, wakes) = self
+ .thread
+ .join()
+ .map_err(|_| anyhow::anyhow!("sentinel panicked"))?;
+ let info = cidre::mach::TimeBaseInfo::new();
+ let mut out = serde_json::json!({
+ "period_secs": SENTINEL_PERIOD_SECS,
+ "timebase": [info.numer, info.denom],
+ "first": wakes.first(),
+ "realtime": realtime,
+ "end": end,
+ })
+ .to_string();
+ out.push('\n');
+ for w in wakes.windows(2) {
+ out.push_str(&(w[1] - w[0]).to_string());
+ out.push('\n');
+ }
+ std::fs::write(&self.path, out)?;
+ Ok(())
+ }
+ }
+
/// Stops the observer stream when dropped.
struct Observer {
stop: Arc<std::sync::atomic::AtomicBool>,
diff --git a/crates/verify/src/report.rs b/crates/verify/src/report.rs
index 5987c9950..4bf3eec7b 100644
--- a/crates/verify/src/report.rs
+++ b/crates/verify/src/report.rs
@@ -62,6 +62,9 @@ pub struct Case {
started: Instant,
pub streams: Vec<StreamReport>,
pub problems: Vec<String>,
+ /// The class of each entry of `problems` (a stream report's classes,
+ /// comma-separated).
+ pub problem_classes: Vec<String>,
/// Failure classes seen (see `expectations.toml`).
pub classes: BTreeSet<String>,
/// First problem per class (the draft reason when blessing).
@@ -84,6 +87,7 @@ impl Case {
started: Instant::now(),
streams: Vec::new(),
problems: Vec::new(),
+ problem_classes: Vec::new(),
classes: Default::default(),
class_problems: Default::default(),
notes: Vec::new(),
@@ -115,6 +119,7 @@ impl Case {
self.class_problems
.entry(class.to_string())
.or_insert_with(|| text.clone());
+ self.problem_classes.push(class.to_string());
self.problems.push(text);
}
@@ -134,6 +139,7 @@ impl Case {
pub fn stream(&mut self, report: StreamReport) {
if !report.passed() {
+ let mut classes = Vec::new();
for (class, n) in [
("stale", report.stale),
("future", report.ahead),
@@ -148,8 +154,10 @@ impl Case {
self.class_problems
.entry(class.to_string())
.or_insert_with(|| report.summary());
+ classes.push(class);
}
}
+ self.problem_classes.push(classes.join(","));
self.problems.push(report.summary());
}
self.streams.push(report);
diff --git a/crates/verify/src/stimulus.rs b/crates/verify/src/stimulus.rs
index 2a7cb97e3..0820d75b2 100644
--- a/crates/verify/src/stimulus.rs
+++ b/crates/verify/src/stimulus.rs
@@ -123,6 +123,54 @@ pub fn read_log(path: &Path) -> anyhow::Result<Vec<StimulusEvent>> {
.collect()
}
+/// The stimulus's real-time sentinel log, next to its event log.
+pub const SENTINEL_FILE: &str = "sentinel.txt";
+/// The sentinel's wake period.
+pub const SENTINEL_PERIOD_SECS: f64 = 0.002;
+
+/// Every wake of the stimulus's real-time sentinel thread (host ticks) and
+/// the scheduling it ran under.
+#[cfg(target_os = "macos")]
+#[derive(Clone, Debug)]
+pub struct SentinelLog {
+ pub realtime: crate::sched::Realtime,
+ pub end: Option<crate::sched::ThreadState>,
+ pub wakes: Vec<u64>,
+}
+
+#[cfg(target_os = "macos")]
+impl SentinelLog {
+ pub fn read(path: &std::path::Path) -> anyhow::Result<Self> {
+ #[derive(serde::Deserialize)]
+ struct Header {
+ first: Option<u64>,
+ realtime: crate::sched::Realtime,
+ end: Option<crate::sched::ThreadState>,
+ }
+ let text = std::fs::read_to_string(path)?;
+ let mut lines = text.lines();
+ let header: Header = serde_json::from_str(lines.next().unwrap_or_default())?;
+ let mut wakes = Vec::new();
+ if let Some(mut at) = header.first {
+ wakes.push(at);
+ for line in lines {
+ at += line.trim().parse::<u64>()?;
+ wakes.push(at);
+ }
+ }
+ Ok(Self {
+ realtime: header.realtime,
+ end: header.end,
+ wakes,
+ })
+ }
+
+ /// Gaps between consecutive wakes: (start host, seconds).
+ pub fn gaps(&self) -> impl Iterator<Item = (u64, f64)> + '_ {
+ self.wakes.windows(2).map(|w| (w[0], host_secs(w[1], w[0])))
+ }
+}
+
#[cfg(target_os = "macos")]
pub fn host_now() -> u64 {
cidre::mach::abs_time()