Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix: require explanations for new expectation proposals [skip ci]

Richie McIlroy03f325f6aa76
diff --git a/crates/verify-expect/src/lib.rs b/crates/verify-expect/src/lib.rs
index 6182b127a..160c94548 100644
--- a/crates/verify-expect/src/lib.rs
+++ b/crates/verify-expect/src/lib.rs
@@ -760,8 +760,8 @@ pub fn judge_with(
}
/// Rewrites `xfails` so that this platform's run passes exactly, without
-/// touching what other platforms or cases that did not run expect. Reasons
-/// and links of replaced entries are kept; new failures get the reason
+/// touching what other platforms or cases that did not run expect. Links
+/// of replaced entries are kept; new proposals get the reason
/// `TODO: <problem>` from `problems`.
pub fn bless(
xfails: &[Xfail],
@@ -899,16 +899,11 @@ pub fn bless_with(
} else {
Bound::new()
},
- reason: previous.map_or_else(
- || {
- format!(
- "TODO: {}",
- problems
- .get(&(case.clone(), class.clone()))
- .map_or("describe the failure", |p| p.as_str())
- )
- },
- |x| x.reason.clone(),
+ reason: format!(
+ "TODO: {}",
+ problems
+ .get(&(case.clone(), class.clone()))
+ .map_or("describe the failure", |p| p.as_str())
),
link: previous.map_or_else(String::new, |x| x.link.clone()),
};
@@ -1308,8 +1303,15 @@ mod tests {
.iter()
.filter(|x| x.case == "b" && x.applies_on("macos"))
.collect();
- assert_eq!(b_mac.len(), 1);
- assert_eq!(b_mac[0].seeds, Some(set(&[2, 3, 40])));
+ assert_eq!(b_mac.len(), 2);
+ assert!(
+ b_mac
+ .iter()
+ .any(|x| x.seeds == Some(set(&[40])) && x.reason == "r")
+ );
+ assert!(b_mac.iter().any(|x| {
+ x.seeds == Some(set(&[2, 3])) && x.reason == "TODO: describe the failure"
+ }));
let d = blessed.iter().find(|x| x.case == "d").unwrap();
assert_eq!(d.reason, "TODO: 3 frames missing");
assert_eq!(d.platforms, vec!["macos"]);
@@ -1321,12 +1323,14 @@ mod tests {
}
#[test]
- fn bless_merges_platforms_back() {
+ fn bless_requires_an_explanation_on_a_new_platform() {
let xfails = vec![entry("a", "stale", &["linux", "windows"], None)];
let ran = Ran::from([("a".to_string(), None)]);
let observed = Observed::from([(("a".into(), "stale".into()), set(&[]))]);
let blessed = bless(&xfails, "macos", &ran, &observed, &BTreeMap::new());
- assert_eq!(blessed, vec![entry("a", "stale", &[], None)]);
+ let mut proposed = entry("a", "stale", &["macos"], None);
+ proposed.reason = "TODO: describe the failure".into();
+ assert_eq!(blessed, vec![xfails[0].clone(), proposed]);
}
fn measured(case: &str, class: &str, metrics: &[(&str, f64, f64)]) -> Evidence {
diff --git a/crates/verify-expect/src/main.rs b/crates/verify-expect/src/main.rs
index d32bc92fd..c0dad0602 100644
--- a/crates/verify-expect/src/main.rs
+++ b/crates/verify-expect/src/main.rs
@@ -192,3 +192,64 @@ fn main() -> ExitCode {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use cap_verify_expect::{Metrics, Range};
+
+ #[test]
+ fn expanded_signature_requires_a_new_explanation() {
+ let entries = parse(
+ "[[xfail]]\ncase = \"a\"\nclass = \"sync\"\nbound = { ms = [-5, 5] }\nreason = \"known offset\"\n",
+ )
+ .unwrap();
+ let ran = Ran::from([("a".into(), None)]);
+ let observed = Observed::from([(("a".into(), "sync".into()), BTreeSet::new())]);
+ let evidence = Evidence {
+ measures: BTreeMap::from([(
+ ("a".into(), "sync".into()),
+ Metrics::from([("ms".into(), Range::at(10.0))]),
+ )]),
+ ..Default::default()
+ };
+ let proposals = bless_with(
+ &entries,
+ "macos",
+ &ran,
+ &observed,
+ &evidence,
+ &BTreeMap::new(),
+ );
+ assert_eq!(proposals.len(), 2);
+ assert!(proposals.contains(&entries[0]));
+ let path = std::env::temp_dir().join(format!(
+ "cap-verify-explanation-{}-{}.toml",
+ std::process::id(),
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_nanos()
+ ));
+ std::fs::write(&path, render(&proposals)).unwrap();
+ let status = check_command(&[path.to_string_lossy().into_owned()]);
+ std::fs::remove_file(path).unwrap();
+ assert_eq!(status, ExitCode::FAILURE);
+
+ let retained = bless_with(
+ &entries,
+ "macos",
+ &ran,
+ &observed,
+ &Evidence {
+ measures: BTreeMap::from([(
+ ("a".into(), "sync".into()),
+ Metrics::from([("ms".into(), Range::at(3.0))]),
+ )]),
+ ..Default::default()
+ },
+ &BTreeMap::new(),
+ );
+ assert_eq!(retained, entries);
+ }
+}