Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix: scope expectation proposals to recorded source hosts [skip ci]

Richie McIlroy4fbe8ce4beba
diff --git a/crates/verify-expect/src/lib.rs b/crates/verify-expect/src/lib.rs
index 160c94548..33030b5cf 100644
--- a/crates/verify-expect/src/lib.rs
+++ b/crates/verify-expect/src/lib.rs
@@ -200,8 +200,13 @@ pub struct Xfail {
impl Xfail {
pub fn applies_on(&self, platform: &str) -> bool {
+ self.applies_on_host(platform, host().as_deref())
+ }
+
+ fn applies_on_host(&self, platform: &str, source_host: Option<&str>) -> bool {
(self.platforms.is_empty() || self.platforms.iter().any(|p| p == platform))
- && (self.hosts.is_empty() || host().is_some_and(|h| self.hosts.contains(&h)))
+ && (self.hosts.is_empty()
+ || source_host.is_some_and(|h| self.hosts.iter().any(|candidate| candidate == h)))
}
pub fn matches(&self, case: &str, platform: &str) -> bool {
@@ -623,12 +628,16 @@ fn covered_by<'a>(
case: &str,
class: &str,
platform: &str,
+ source_host: Option<&str>,
seeds: &BTreeSet<u64>,
evidence: &Evidence,
) -> BTreeSet<u64> {
let mut covered = BTreeSet::new();
for x in xfails.filter(|x| {
- x.class == class && x.matches(case, platform) && excuses(x, case, class, evidence).is_ok()
+ x.class == class
+ && x.applies_on_host(platform, source_host)
+ && glob_match(&x.case, case)
+ && excuses(x, case, class, evidence).is_ok()
}) {
match &x.seeds {
None => return seeds.clone(),
@@ -694,6 +703,7 @@ pub fn judge_with(
case,
class,
platform,
+ host().as_deref(),
seeds,
evidence,
);
@@ -759,9 +769,9 @@ pub fn judge_with(
verdict
}
-/// Rewrites `xfails` so that this platform's run passes exactly, without
-/// touching what other platforms or cases that did not run expect. Links
-/// of replaced entries are kept; new proposals get the reason
+/// Proposes changes for this platform and host without narrowing entries
+/// for unobserved hosts, platforms or cases. Links of replaced entries
+/// are kept; new proposals get the reason
/// `TODO: <problem>` from `problems`.
pub fn bless(
xfails: &[Xfail],
@@ -794,15 +804,26 @@ pub fn bless_observing(
unobserved: unobserved.clone(),
..Default::default()
};
- bless_with(xfails, platform, ran, observed, &evidence, problems)
+ bless_with(
+ xfails,
+ platform,
+ host().as_deref(),
+ ran,
+ observed,
+ &evidence,
+ problems,
+ )
}
/// [`bless`] with measurements: a failure outside every entry's bound gets a
/// new entry (TODO reason) whose bound is the measured range, to be
-/// reviewed against the mechanism.
+/// reviewed against the mechanism. Unknown-host observations cannot remove
+/// entries. All-host entries are retained because finite host observations
+/// cannot establish their behavior on every possible host.
pub fn bless_with(
xfails: &[Xfail],
platform: &str,
+ source_host: Option<&str>,
ran: &Ran,
observed: &Observed,
evidence: &Evidence,
@@ -819,9 +840,15 @@ pub fn bless_with(
let unmeasured = matched
.iter()
.any(|(case, _)| unobserved.contains(&((*case).clone(), x.class.clone())));
- if !x.applies_on(platform) || matched.is_empty() || x.flaky || unmeasured {
+ if source_host.is_none()
+ || x.hosts.is_empty()
+ || !x.applies_on_host(platform, source_host)
+ || matched.is_empty()
+ || x.flaky
+ || unmeasured
+ {
out.push(x.clone());
- if x.applies_on(platform) {
+ if x.applies_on_host(platform, source_host) {
kept_here.push(x.clone());
}
continue;
@@ -847,6 +874,19 @@ pub fn bless_with(
if !others.is_empty() {
out.push(x.with_platforms(others));
}
+ let other_hosts: Vec<String> = x
+ .hosts
+ .iter()
+ .filter(|h| Some(h.as_str()) != source_host)
+ .cloned()
+ .collect();
+ if !other_hosts.is_empty() {
+ out.push(Xfail {
+ platforms: vec![platform.to_string()],
+ hosts: other_hosts,
+ ..x.clone()
+ });
+ }
if let Some(listed) = &x.seeds {
for (case, ran_seeds) in &matched {
let not_ran: BTreeSet<u64> = listed
@@ -859,6 +899,7 @@ pub fn bless_with(
case: (*case).clone(),
seeds: Some(not_ran),
platforms: vec![platform.to_string()],
+ hosts: source_host.into_iter().map(str::to_string).collect(),
..x.clone()
};
out.push(entry.clone());
@@ -868,10 +909,19 @@ pub fn bless_with(
}
}
for ((case, class), seeds) in observed {
- let covered = covered_by(kept_here.iter(), case, class, platform, seeds, evidence);
+ let covered = covered_by(
+ kept_here.iter(),
+ case,
+ class,
+ platform,
+ source_host,
+ seeds,
+ evidence,
+ );
let fully = kept_here.iter().any(|x| {
x.class == *class
- && x.matches(case, platform)
+ && x.applies_on_host(platform, source_host)
+ && glob_match(&x.case, case)
&& x.seeds.is_none()
&& excuses(x, case, class, evidence).is_ok()
});
@@ -879,9 +929,11 @@ pub fn bless_with(
continue;
}
let missing: BTreeSet<u64> = seeds.difference(&covered).copied().collect();
- let previous = xfails
- .iter()
- .find(|x| x.class == *class && glob_match(&x.case, case));
+ let previous = xfails.iter().find(|x| {
+ x.class == *class
+ && x.applies_on_host(platform, source_host)
+ && glob_match(&x.case, case)
+ });
let entry = Xfail {
case: case.clone(),
class: class.clone(),
@@ -889,7 +941,7 @@ pub fn bless_with(
seeds: (!seeds.is_empty()).then_some(missing),
flaky: false,
starved: false,
- hosts: Vec::new(),
+ hosts: source_host.into_iter().map(str::to_string).collect(),
bound: if seeds.is_empty() {
evidence
.measures
@@ -1055,6 +1107,7 @@ fn write_records_to(dir: &Path, records: &[(Record, serde_json::Value)]) {
.expect("open report file");
for (record, extra) in records {
let mut value = serde_json::to_value(record).expect("serialize report record");
+ value["host"] = serde_json::to_value(host()).expect("serialize report host");
if let (Some(map), Some(extra)) = (value.as_object_mut(), extra.as_object()) {
for (k, v) in extra {
map.entry(k.clone()).or_insert_with(|| v.clone());
@@ -1082,7 +1135,7 @@ mod tests {
seeds: seeds.map(set),
flaky: false,
starved: false,
- hosts: Vec::new(),
+ hosts: host().into_iter().collect(),
bound: Bound::new(),
reason: "r".into(),
link: String::new(),
@@ -1122,7 +1175,13 @@ mod tests {
repro: String::new(),
seconds: 0.0,
};
- write_records_to(dir, &[(record, serde_json::json!({"writer": writer}))]);
+ write_records_to(
+ dir,
+ &[(
+ record,
+ serde_json::json!({"writer": writer, "host": "incorrect"}),
+ )],
+ );
}
});
}
@@ -1132,6 +1191,8 @@ mod tests {
let cases: BTreeSet<String> = text
.lines()
.map(|line| {
+ let value: serde_json::Value = serde_json::from_str(line).unwrap();
+ assert_eq!(value["host"], serde_json::json!(host()));
let record: Record = serde_json::from_str(line).unwrap();
assert_eq!(record.outcome, Outcome::Unsupported);
record.case
@@ -1333,6 +1394,90 @@ mod tests {
assert_eq!(blessed, vec![xfails[0].clone(), proposed]);
}
+ #[test]
+ fn bless_preserves_unobserved_host_and_platform_slices() {
+ let xfail = Xfail {
+ hosts: vec!["github-hosted".into(), "local".into()],
+ ..entry("a", "stale", &["macos", "windows"], None)
+ };
+ let ran = Ran::from([("a".into(), None)]);
+ let blessed = bless_with(
+ &[xfail.clone()],
+ "macos",
+ Some("local"),
+ &ran,
+ &Observed::new(),
+ &Evidence::default(),
+ &BTreeMap::new(),
+ );
+ assert_eq!(blessed.len(), 2);
+ assert!(
+ blessed
+ .iter()
+ .any(|x| x.platforms == ["windows"] && x.hosts == xfail.hosts)
+ );
+ assert!(
+ blessed
+ .iter()
+ .any(|x| x.platforms == ["macos"] && x.hosts == ["github-hosted"])
+ );
+ assert!(
+ !blessed
+ .iter()
+ .any(|x| x.applies_on_host("macos", Some("local")))
+ );
+
+ let untested = bless_with(
+ &[xfail.clone()],
+ "macos",
+ Some("blacksmith"),
+ &ran,
+ &Observed::new(),
+ &Evidence::default(),
+ &BTreeMap::new(),
+ );
+ assert_eq!(untested, vec![xfail]);
+ }
+
+ #[test]
+ fn bless_unknown_hosts_cannot_remove_or_narrow_entries() {
+ let xfails = vec![
+ entry("a", "stale", &["macos"], Some(&[1, 2])),
+ Xfail {
+ hosts: Vec::new(),
+ ..entry("b", "stale", &["macos"], None)
+ },
+ ];
+ let ran = Ran::from([("a".into(), Some(set(&[1, 2]))), ("b".into(), None)]);
+ for observed in [
+ Observed::new(),
+ Observed::from([(("a".into(), "stale".into()), set(&[1]))]),
+ ] {
+ let blessed = bless_with(
+ &xfails,
+ "macos",
+ None,
+ &ran,
+ &observed,
+ &Evidence::default(),
+ &BTreeMap::new(),
+ );
+ for entry in &xfails {
+ assert!(blessed.contains(entry));
+ }
+ }
+ let blessed = bless_with(
+ &xfails[1..],
+ "macos",
+ Some("blacksmith"),
+ &ran,
+ &Observed::new(),
+ &Evidence::default(),
+ &BTreeMap::new(),
+ );
+ assert_eq!(blessed, xfails[1..]);
+ }
+
fn measured(case: &str, class: &str, metrics: &[(&str, f64, f64)]) -> Evidence {
let mut evidence = Evidence::default();
evidence.measures.insert(
@@ -1450,7 +1595,15 @@ mod tests {
let ran = Ran::from([("a".to_string(), None)]);
let observed = Observed::from([(("a".into(), "sync".into()), set(&[]))]);
let evidence = measured("a", "sync", &[("ms", -201.0, -199.0)]);
- let blessed = bless_with(&[], "macos", &ran, &observed, &evidence, &BTreeMap::new());
+ let blessed = bless_with(
+ &[],
+ "macos",
+ host().as_deref(),
+ &ran,
+ &observed,
+ &evidence,
+ &BTreeMap::new(),
+ );
assert_eq!(blessed.len(), 1);
assert_eq!(
blessed[0].bound["ms"],
diff --git a/crates/verify-expect/src/main.rs b/crates/verify-expect/src/main.rs
index c0dad0602..f1cdf6342 100644
--- a/crates/verify-expect/src/main.rs
+++ b/crates/verify-expect/src/main.rs
@@ -44,6 +44,90 @@ struct Run {
problems: BTreeMap<(String, String), String>,
}
+#[derive(serde::Deserialize)]
+struct SourceRecord {
+ #[serde(flatten)]
+ record: Record,
+ #[serde(default)]
+ host: Option<String>,
+}
+
+type HostRuns = BTreeMap<(String, Option<String>), Run>;
+type Runs = BTreeMap<String, HostRuns>;
+
+fn add_record(runs: &mut Runs, source: SourceRecord) {
+ let SourceRecord { record, host } = source;
+ if matches!(record.outcome, Outcome::Unsupported | Outcome::Inconclusive) {
+ return;
+ }
+ let host = host
+ .map(|h| h.trim().to_string())
+ .filter(|h| !h.is_empty() && h != "unknown");
+ let run = runs
+ .entry(record.expectations.clone())
+ .or_default()
+ .entry((record.platform.clone(), host))
+ .or_default();
+ let ran = run.ran.entry(record.case.clone()).or_insert(None);
+ if let Some(seeds) = &record.seeds_ran {
+ ran.get_or_insert_with(BTreeSet::new).extend(seeds);
+ }
+ for class in record.unobserved.keys() {
+ run.evidence
+ .unobserved
+ .insert((record.case.clone(), class.clone()));
+ }
+ if record.starved {
+ run.evidence.starved.insert(record.case.clone());
+ }
+ for (class, metrics) in &record.measures {
+ let entry = run
+ .evidence
+ .measures
+ .entry((record.case.clone(), class.clone()))
+ .or_default();
+ for (metric, range) in metrics {
+ entry
+ .entry(metric.clone())
+ .and_modify(|r| {
+ r.include(range.min);
+ r.include(range.max);
+ })
+ .or_insert(*range);
+ }
+ }
+ for (class, seeds) in &record.failures {
+ run.observed
+ .entry((record.case.clone(), class.clone()))
+ .or_default()
+ .extend(seeds);
+ if let Some(problem) = record.problems.get(class) {
+ run.problems
+ .entry((record.case.clone(), class.clone()))
+ .or_insert_with(|| problem.clone());
+ }
+ }
+}
+
+fn proposals(
+ entries: &[cap_verify_expect::Xfail],
+ runs: &HostRuns,
+) -> Vec<cap_verify_expect::Xfail> {
+ let mut entries = entries.to_vec();
+ for ((platform, host), run) in runs {
+ entries = bless_with(
+ &entries,
+ platform,
+ host.as_deref(),
+ &run.ran,
+ &run.observed,
+ &run.evidence,
+ &run.problems,
+ );
+ }
+ entries
+}
+
fn bless_command(inputs: &[String]) -> ExitCode {
let mut files = Vec::new();
for input in inputs {
@@ -53,64 +137,18 @@ fn bless_command(inputs: &[String]) -> ExitCode {
eprintln!("no .jsonl records under {inputs:?}");
return ExitCode::FAILURE;
}
- let mut runs: BTreeMap<String, BTreeMap<String, Run>> = BTreeMap::new();
+ let mut runs = Runs::new();
for file in &files {
let text = std::fs::read_to_string(file).unwrap_or_default();
for line in text.lines().filter(|l| !l.trim().is_empty()) {
- let record: Record = match serde_json::from_str(line) {
+ let record: SourceRecord = match serde_json::from_str(line) {
Ok(record) => record,
Err(e) => {
eprintln!("{}: skipping a record: {e}", file.display());
continue;
}
};
- if matches!(record.outcome, Outcome::Unsupported | Outcome::Inconclusive) {
- continue;
- }
- let run = runs
- .entry(record.expectations.clone())
- .or_default()
- .entry(record.platform.clone())
- .or_default();
- let ran = run.ran.entry(record.case.clone()).or_insert(None);
- if let Some(seeds) = &record.seeds_ran {
- ran.get_or_insert_with(BTreeSet::new).extend(seeds);
- }
- for class in record.unobserved.keys() {
- run.evidence
- .unobserved
- .insert((record.case.clone(), class.clone()));
- }
- if record.starved {
- run.evidence.starved.insert(record.case.clone());
- }
- for (class, metrics) in &record.measures {
- let entry = run
- .evidence
- .measures
- .entry((record.case.clone(), class.clone()))
- .or_default();
- for (metric, range) in metrics {
- entry
- .entry(metric.clone())
- .and_modify(|r| {
- r.include(range.min);
- r.include(range.max);
- })
- .or_insert(*range);
- }
- }
- for (class, seeds) in &record.failures {
- run.observed
- .entry((record.case.clone(), class.clone()))
- .or_default()
- .extend(seeds);
- if let Some(problem) = record.problems.get(class) {
- run.problems
- .entry((record.case.clone(), class.clone()))
- .or_insert_with(|| problem.clone());
- }
- }
+ add_record(&mut runs, record);
}
}
let root = repo_root();
@@ -118,20 +156,10 @@ fn bless_command(inputs: &[String]) -> ExitCode {
let path = root.join(expectations);
let text = std::fs::read_to_string(&path).unwrap_or_default();
let before = load(&path);
- let mut entries = before.clone();
- for (platform, run) in platforms {
- entries = bless_with(
- &entries,
- platform,
- &run.ran,
- &run.observed,
- &run.evidence,
- &run.problems,
- );
- }
+ let entries = proposals(&before, platforms);
let added = entries.iter().filter(|e| !before.contains(e)).count();
let removed = before.iter().filter(|e| !entries.contains(e)).count();
- let platform_list: Vec<&String> = platforms.keys().collect();
+ let platform_list: Vec<_> = platforms.keys().collect();
if added == 0 && removed == 0 {
println!("{expectations}: unchanged ({platform_list:?})");
continue;
@@ -198,6 +226,62 @@ mod tests {
use super::*;
use cap_verify_expect::{Metrics, Range};
+ fn source_record(source_host: Option<&str>, failure: Option<f64>) -> SourceRecord {
+ let mut value = serde_json::json!({
+ "suite": "recording",
+ "expectations": "expectations.toml",
+ "case": "a",
+ "platform": "macos",
+ "outcome": if failure.is_some() { "xfail" } else { "pass" },
+ });
+ if let Some(host) = source_host {
+ value["host"] = host.into();
+ }
+ if let Some(value_seen) = failure {
+ value["failures"] = serde_json::json!({ "sync": [] });
+ value["measures"] = serde_json::json!({ "sync": { "ms": [value_seen, value_seen] } });
+ }
+ serde_json::from_value(value).unwrap()
+ }
+
+ #[test]
+ fn mixed_host_records_keep_separate_measurements_and_expectations() {
+ let entries = parse(
+ "[[xfail]]\ncase = \"a\"\nclass = \"sync\"\nplatforms = [\"macos\"]\nhosts = [\"local\", \"github-hosted\"]\nbound = { ms = [-5, 5] }\nreason = \"known offset\"\n",
+ )
+ .unwrap();
+ let mut runs = Runs::new();
+ add_record(&mut runs, source_record(Some("local"), Some(3.0)));
+ add_record(&mut runs, source_record(Some("blacksmith"), Some(10.0)));
+ add_record(&mut runs, source_record(Some("github-hosted"), None));
+ let host_runs = &runs["expectations.toml"];
+ assert_eq!(host_runs.len(), 3);
+ let proposed = proposals(&entries, host_runs);
+ assert_eq!(proposed.len(), 2);
+ let local = proposed.iter().find(|x| x.hosts == ["local"]).unwrap();
+ assert_eq!(local.bound, entries[0].bound);
+ assert_eq!(local.reason, entries[0].reason);
+ let blacksmith = proposed.iter().find(|x| x.hosts == ["blacksmith"]).unwrap();
+ assert_eq!(blacksmith.bound["ms"], Range::at(10.0));
+ assert!(blacksmith.reason.starts_with("TODO:"));
+ }
+
+ #[test]
+ fn legacy_and_unknown_host_records_never_use_the_invoking_host() {
+ let entries = parse(
+ "[[xfail]]\ncase = \"a\"\nclass = \"sync\"\nhosts = [\"local\", \"github-hosted\"]\nreason = \"known offset\"\n",
+ )
+ .unwrap();
+ for host in [None, Some(""), Some("unknown")] {
+ let mut runs = Runs::new();
+ add_record(&mut runs, source_record(host, None));
+ add_record(&mut runs, source_record(Some("blacksmith"), None));
+ let host_runs = &runs["expectations.toml"];
+ assert!(host_runs.contains_key(&("macos".into(), None)));
+ assert_eq!(proposals(&entries, host_runs), entries);
+ }
+ }
+
#[test]
fn expanded_signature_requires_a_new_explanation() {
let entries = parse(
@@ -216,6 +300,7 @@ mod tests {
let proposals = bless_with(
&entries,
"macos",
+ Some("local"),
&ran,
&observed,
&evidence,
@@ -239,6 +324,7 @@ mod tests {
let retained = bless_with(
&entries,
"macos",
+ Some("local"),
&ran,
&observed,
&Evidence {