Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

ci(path-check): reuse cargo metadata across PRs with the same manifests (#9846)

teo3ddfe579c8ef
diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.yml
index e622e88f27..043c6cfa40 100644
--- a/.github/workflows/code_check_cloud_storage.yml
+++ b/.github/workflows/code_check_cloud_storage.yml
@@ -272,7 +272,31 @@ jobs:
fi
ensure_filter_bin
- packages="$("$filter_bin" /tmp/changed-files "$(< /tmp/base-revision)")"
+
+ # cargo metadata takes ~15 s on its first run on this runner and ~1 s after, so its
+ # output is saved per manifest state. The key covers every input of that output; the
+ # filter ignores the file when the change set can alter the graph.
+ metadata_file="$RUNNER_TEMP/cargo-metadata.json"
+ metadata_url=""
+ metadata_restored=false
+ if [ -n "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then
+ metadata_key="$( { pwd; git ls-files --stage -- Cargo.lock rust-toolchain.toml ':(glob)**/Cargo.toml'; } | sha256sum | cut -c1-16)"
+ metadata_url="$SCCACHE_WEBDAV_ENDPOINT/nextest-filter/metadata-$metadata_key.json.gz"
+ if filter_cache_request "$metadata_url" 2>/dev/null | gunzip > "$metadata_file"; then
+ metadata_restored=true
+ echo "Using saved cargo metadata $metadata_key"
+ else
+ rm -f "$metadata_file"
+ fi
+ fi
+
+ packages="$("$filter_bin" /tmp/changed-files "$(< /tmp/base-revision)" "$metadata_file")"
+
+ # Losing the upload costs the next job about 15 s, not a failure.
+ if [ -n "$metadata_url" ] && [ "$metadata_restored" = false ] && [ -f "$metadata_file" ]; then
+ gzip < "$metadata_file" | filter_cache_request --upload-file - "$metadata_url" \
+ || echo "::warning::Could not save cargo metadata to the remote cache"
+ fi
# One package list drives both clippy and tests, so a clippy.toml edit can
# only skip tests when nothing else selected a package.
diff --git a/tooling/xtask/crates/xtask_graph/src/lib.rs b/tooling/xtask/crates/xtask_graph/src/lib.rs
index ef18660d34..fd3fbd3431 100644
--- a/tooling/xtask/crates/xtask_graph/src/lib.rs
+++ b/tooling/xtask/crates/xtask_graph/src/lib.rs
@@ -6,8 +6,8 @@
//! the task works from anywhere in the repo.
use anyhow::{Context, Result};
-use guppy::MetadataCommand;
use guppy::graph::PackageGraph;
+use guppy::{CargoMetadata, MetadataCommand};
use std::path::Path;
/// Build the workspace [`PackageGraph`] via `cargo metadata`.
@@ -35,3 +35,30 @@ pub fn build_graph_at(workspace_dir: &Path, locked: bool) -> Result<PackageGraph
}
})
}
+
+/// [`build_graph_at`] with `--locked`, reusing the `cargo metadata` output saved at
+/// `metadata_file`, or saving it there when that file is missing or unreadable.
+///
+/// The caller owns the file's identity: it must only name a file produced from the
+/// same manifests, lockfile and workspace path.
+pub fn build_graph_with_saved_metadata(
+ workspace_dir: &Path,
+ metadata_file: &Path,
+) -> Result<PackageGraph> {
+ if let Ok(json) = std::fs::read_to_string(metadata_file)
+ && let Ok(graph) = CargoMetadata::parse_json(&json).and_then(CargoMetadata::build_graph)
+ {
+ return Ok(graph);
+ }
+ let mut cmd = MetadataCommand::new();
+ cmd.current_dir(workspace_dir);
+ cmd.other_options(vec!["--locked".to_owned()]);
+ let metadata = cmd.exec().context("running cargo metadata --locked")?;
+ let mut json = Vec::new();
+ metadata
+ .serialize(&mut json)
+ .context("serializing cargo metadata")?;
+ std::fs::write(metadata_file, json)
+ .with_context(|| format!("saving cargo metadata to {}", metadata_file.display()))?;
+ metadata.build_graph().context("building the package graph")
+}
diff --git a/tooling/xtask/crates/xtask_nextest_filter/src/main.rs b/tooling/xtask/crates/xtask_nextest_filter/src/main.rs
index 9a20ec27a3..7d3c20d02c 100644
--- a/tooling/xtask/crates/xtask_nextest_filter/src/main.rs
+++ b/tooling/xtask/crates/xtask_nextest_filter/src/main.rs
@@ -40,6 +40,7 @@ use tempfile::TempDir;
#[cfg(test)]
use xtask_graph::build_graph;
use xtask_graph::build_graph_at;
+use xtask_graph::build_graph_with_saved_metadata;
#[cfg(test)]
mod test;
@@ -97,13 +98,25 @@ fn main() -> Result<()> {
let base_revision = base_revision
.to_str()
.context("base revision is not valid UTF-8")?;
- run(Path::new(changed_files_path), base_revision)
+ run(Path::new(changed_files_path), base_revision, None)
}
- _ => bail!("usage: cargo x nextest-filter <changed-files-path> <base-revision>"),
+ [changed_files_path, base_revision, metadata_file] => {
+ let base_revision = base_revision
+ .to_str()
+ .context("base revision is not valid UTF-8")?;
+ run(
+ Path::new(changed_files_path),
+ base_revision,
+ Some(Path::new(metadata_file)),
+ )
+ }
+ _ => bail!(
+ "usage: cargo x nextest-filter <changed-files-path> <base-revision> [metadata-file]"
+ ),
}
}
-fn run(changed_files_path: &Path, base_revision: &str) -> Result<()> {
+fn run(changed_files_path: &Path, base_revision: &str, metadata_file: Option<&Path>) -> Result<()> {
let changed_files = std::fs::read_to_string(changed_files_path).with_context(|| {
format!(
"reading changed files from {}",
@@ -118,7 +131,10 @@ fn run(changed_files_path: &Path, base_revision: &str) -> Result<()> {
let workspace_dir = xtask_paths::workspace_root();
if !changed_files.iter().any(|path| changes_cargo_graph(path)) {
- let graph = build_graph_at(&workspace_dir, true)?;
+ let graph = match metadata_file {
+ Some(file) => build_graph_with_saved_metadata(&workspace_dir, file)?,
+ None => build_graph_at(&workspace_dir, true)?,
+ };
println!(
"{}",
select_by_paths(&graph, &changed_files, &repository_rules()?)?
diff --git a/tooling/xtask/crates/xtask_nextest_filter/src/test.rs b/tooling/xtask/crates/xtask_nextest_filter/src/test.rs
index 14e62e3e32..0ea56908f8 100644
--- a/tooling/xtask/crates/xtask_nextest_filter/src/test.rs
+++ b/tooling/xtask/crates/xtask_nextest_filter/src/test.rs
@@ -485,3 +485,35 @@ fn collect_rust_files(dir: &Path, out: &mut BTreeSet<PathBuf>) {
}
}
}
+
+/// The saved metadata must select the same packages as a live `cargo metadata`, and a
+/// damaged file must fall back to one rather than fail or select nothing.
+#[test]
+fn saved_metadata_selects_what_live_metadata_selects() {
+ let dir = tempfile::tempdir().unwrap();
+ let file = dir.path().join("metadata.json");
+ let root = xtask_paths::workspace_root();
+ let changed = [PathBuf::from("crates/macro_user_id/src/lib.rs")];
+ let rules = repository_rules().unwrap();
+ let live = build_graph(true).expect("cargo metadata");
+ let expected = select_by_paths(&live, &changed, &rules).unwrap();
+
+ let saved_now = build_graph_with_saved_metadata(&root, &file).unwrap();
+ assert!(file.exists());
+ assert_eq!(
+ select_by_paths(&saved_now, &changed, &rules).unwrap(),
+ expected
+ );
+ let restored = build_graph_with_saved_metadata(&root, &file).unwrap();
+ assert_eq!(
+ select_by_paths(&restored, &changed, &rules).unwrap(),
+ expected
+ );
+
+ std::fs::write(&file, "{ not metadata").unwrap();
+ let recovered = build_graph_with_saved_metadata(&root, &file).unwrap();
+ assert_eq!(
+ select_by_paths(&recovered, &changed, &rules).unwrap(),
+ expected
+ );
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh
index 9bb946351b..65d4693675 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh
@@ -100,7 +100,31 @@ if ! grep -qvE '^\.sqlx/' /tmp/changed-files; then
fi
ensure_filter_bin
-packages="$("$filter_bin" /tmp/changed-files "$(< /tmp/base-revision)")"
+
+# cargo metadata takes ~15 s on its first run on this runner and ~1 s after, so its
+# output is saved per manifest state. The key covers every input of that output; the
+# filter ignores the file when the change set can alter the graph.
+metadata_file="$RUNNER_TEMP/cargo-metadata.json"
+metadata_url=""
+metadata_restored=false
+if [ -n "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then
+ metadata_key="$( { pwd; git ls-files --stage -- Cargo.lock rust-toolchain.toml ':(glob)**/Cargo.toml'; } | sha256sum | cut -c1-16)"
+ metadata_url="$SCCACHE_WEBDAV_ENDPOINT/nextest-filter/metadata-$metadata_key.json.gz"
+ if filter_cache_request "$metadata_url" 2>/dev/null | gunzip > "$metadata_file"; then
+ metadata_restored=true
+ echo "Using saved cargo metadata $metadata_key"
+ else
+ rm -f "$metadata_file"
+ fi
+fi
+
+packages="$("$filter_bin" /tmp/changed-files "$(< /tmp/base-revision)" "$metadata_file")"
+
+# Losing the upload costs the next job about 15 s, not a failure.
+if [ -n "$metadata_url" ] && [ "$metadata_restored" = false ] && [ -f "$metadata_file" ]; then
+ gzip < "$metadata_file" | filter_cache_request --upload-file - "$metadata_url" \
+ || echo "::warning::Could not save cargo metadata to the remote cache"
+fi
# One package list drives both clippy and tests, so a clippy.toml edit can
# only skip tests when nothing else selected a package.