SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.
Saved model cache: keep fast opening after an update (#7168) (#7354)
semmelrinde9f5ac92faa5e
diff --git a/crates/io/Cargo.toml b/crates/io/Cargo.tomlindex e9a701fe1a..e5f7c4dee8 100644--- a/crates/io/Cargo.toml+++ b/crates/io/Cargo.toml@@ -31,6 +31,10 @@ cadmpeg-core = { workspace = true }cadmpeg-ir = { workspace = true }cadmpeg-registry = { workspace = true }+# The model cache's key (build.rs).+[build-dependencies]+sha2 = { workspace = true }+# A plain fsync where a network share refuses Apple's full flush (src/safe_write.rs); already in# the tree through tempfile.[target.'cfg(target_vendor = "apple")'.dependencies]diff --git a/crates/io/build.rs b/crates/io/build.rsindex 45c77eed47..d135732ea8 100644--- a/crates/io/build.rs+++ b/crates/io/build.rs@@ -1,20 +1,89 @@-//! The commit the model cache is keyed by (`SOLIDCRAFT_SOURCE_COMMIT`): a cache is only the-//! result of evaluating its design when the same source computed it. "unknown" without git (a-//! source tarball), which never trusts a cache.+//! What the model cache is keyed by. `SOLIDCRAFT_EVALUATION_SOURCE`: the digest of the committed+//! source a design's model is computed from: the `doc` crate and every workspace crate it is+//! built from, the patched dependencies, the locked versions of all others, the workspace's build+//! settings and the cache's own reader. A cache is only the result of evaluating its design when+//! that source computed it, and an update that changed none of it (only the interface, the+//! commands or the other file formats, or the version number) computes the same model.+//! `SOLIDCRAFT_SOURCE_COMMIT`: the commit, recorded with the cache. Both are "unknown" without git+//! (a source tarball), which never trusts a cache.+use sha2::{Digest, Sha256};+use std::path::{Path, PathBuf};use std::process::Command;-fn git(args: &[&str]) -> Option<String> {- let out = Command::new("git").args(args).output().ok()?;- out.status.success().then(|| String::from_utf8_lossy(&out.stdout).trim().to_string()).filter(|s| !s.is_empty())+/// The workspace crates a design's model is computed from: `doc` and every workspace crate it+/// depends on (a test checks the list against their manifests).+const EVALUATION_CRATES: [&str; 6] = ["geom", "sketch", "kernel", "render", "derive", "doc"];+/// The other files the saved model depends on, relative to the workspace root.+const EVALUATION_FILES: [&str; 3] = ["crates/io/build.rs", "crates/io/src/design/native_cache.rs", ".cargo/config.toml"];+/// What under `vendor/` does not compute models: notes, and the interface's renderer.+const VENDOR_OUTSIDE: [&str; 2] = ["README.md", "egui-wgpu"];++fn git(root: &Path, args: &[&str]) -> Option<String> {+ let out = Command::new("git").arg("-C").arg(root).args(args).output().ok()?;+ out.status.success().then(|| String::from_utf8_lossy(&out.stdout).into_owned()).filter(|s| !s.trim().is_empty())+}++/// The committed `Cargo.lock` without the workspace's own packages: their version is the+/// application's, which every release changes; the source they are built from is hashed itself.+fn locked_dependencies(lock: &str) -> String {+ lock.split("\n[[package]]").filter(|block| block.lines().any(|l| l.starts_with("source = "))).collect::<Vec<_>>().join("\n[[package]]")+}++/// The committed workspace `Cargo.toml` without the application's version.+fn workspace_settings(manifest: &str) -> String {+ let mut section = "";+ let mut kept = Vec::new();+ for line in manifest.lines() {+ if line.starts_with('[') {+ section = line.trim();+ }+ if !(section == "[workspace.package]" && line.trim_start().starts_with("version")) {+ kept.push(line);+ }+ }+ kept.join("\n")+}++/// The digest of the committed source of the saved model (see the module notes), or `None` when+/// a part of it is not in the commit.+fn evaluation_source(root: &Path) -> Option<String> {+ let mut paths: Vec<String> = EVALUATION_CRATES.iter().map(|c| format!("crates/{c}")).collect();+ paths.extend(EVALUATION_FILES.iter().map(|f| f.to_string()));+ let specs: Vec<String> = paths.iter().map(|p| format!("HEAD:./{p}")).collect();+ let mut args = vec!["rev-parse"];+ args.extend(specs.iter().map(String::as_str));+ let ids = git(root, &args)?;+ if ids.lines().count() != paths.len() {+ return None;+ }+ let mut digest = Sha256::new();+ for (path, id) in paths.iter().zip(ids.lines()) {+ digest.update(format!("{path} {id}\n"));+ }+ // `vendor/`: one line per entry, "<mode> <type> <id>\t<name>".+ for entry in git(root, &["ls-tree", "HEAD", "./vendor/"])?.lines() {+ let name = entry.rsplit(['\t', '/']).next().unwrap_or(entry);+ if !VENDOR_OUTSIDE.contains(&name) {+ digest.update(format!("vendor {entry}\n"));+ }+ }+ digest.update(locked_dependencies(&git(root, &["show", "HEAD:./Cargo.lock"])?));+ digest.update(workspace_settings(&git(root, &["show", "HEAD:./Cargo.toml"])?));+ Some(digest.finalize().iter().map(|b| format!("{b:02x}")).collect())}fn main() {- let commit = git(&["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".into());+ let crate_dir = PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default());+ let root = crate_dir.join("../..");+ let commit = git(&crate_dir, &["rev-parse", "HEAD"]).map(|s| s.trim().to_string()).unwrap_or_else(|| "unknown".into());println!("cargo:rustc-env=SOLIDCRAFT_SOURCE_COMMIT={commit}");+ let source = evaluation_source(&root).unwrap_or_else(|| "unknown".into());+ println!("cargo:rustc-env=SOLIDCRAFT_EVALUATION_SOURCE={source}");+ println!("cargo:rustc-env=SOLIDCRAFT_EVALUATION_CRATES={}", EVALUATION_CRATES.join(","));// A new commit in this checkout (or worktree) restamps the build.- if let Some(dir) = git(&["rev-parse", "--git-dir"]) {- println!("cargo:rerun-if-changed={dir}/logs/HEAD");+ if let Some(dir) = git(&crate_dir, &["rev-parse", "--git-dir"]) {+ println!("cargo:rerun-if-changed={}/logs/HEAD", dir.trim());}println!("cargo:rerun-if-changed=build.rs");}diff --git a/crates/io/src/design.rs b/crates/io/src/design.rsindex ac608f7757..1576e36f5f 100644--- a/crates/io/src/design.rs+++ b/crates/io/src/design.rs@@ -1109,6 +1109,38 @@ mod tests {assert!(native_cache::source_trusted("0123abc", false));}+ /// The model cache is keyed by the source of the crates a model is computed from (#7168):+ /// each one `doc` is built from must be in it, or an update that changed one would open+ /// designs with the model the code before it computed.+ #[test]+ fn the_cache_key_covers_every_crate_the_model_is_computed_from() {+ let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");+ let read = |p: std::path::PathBuf| std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("{}: {e}", p.display()));+ let workspace = read(root.join("Cargo.toml"));+ // `solidcraft-geom = { path = "crates/geom" }`: a workspace crate's folder.+ let folder = |name: &str| {+ let prefix = format!("{name} = {{ path = \"crates/");+ workspace.lines().find_map(|l| l.strip_prefix(prefix.as_str())?.split('"').next().map(str::to_string))+ };+ let keyed: Vec<&str> = env!("SOLIDCRAFT_EVALUATION_CRATES").split(',').collect();+ let (mut todo, mut seen) = (vec!["doc".to_string()], BTreeSet::new());+ while let Some(krate) = todo.pop() {+ if !seen.insert(krate.clone()) {+ continue;+ }+ assert!(keyed.contains(&krate.as_str()), "crates/{krate} computes models: add it to EVALUATION_CRATES in crates/io/build.rs");+ let mut built_with = false;+ for line in read(root.join("crates").join(&krate).join("Cargo.toml")).lines() {+ if line.starts_with('[') {+ built_with = line.ends_with("dependencies]") && !line.contains("dev-dependencies");+ } else if built_with && let Some(dir) = line.split(" = ").next().filter(|n| n.starts_with("solidcraft-")).and_then(folder) {+ todo.push(dir);+ }+ }+ }+ assert_eq!(seen.len(), keyed.len(), "{seen:?} computed from, {keyed:?} keyed");+ }+#[test]fn native_cache_round_trip_and_preview_defer_original_sources() {let mut doc = Document::new("Native cache");@@ -1314,7 +1346,7 @@ mod tests {let doc = document();let bytes = write_design_container_with_cache(&doc, DesignCompression::Fast, None, Some(&native_geometry())).unwrap();let original = parts(&bytes);- for key in ["format", "producer", "app", "build", "document", "payload"] {+ for key in ["format", "producer", "evaluation", "document", "payload"] {let mut entries = original.clone();let mut manifest: Value = serde_json::from_slice(&entries["cache/model.json"]).unwrap();manifest[key] = json!("unsupported-or-stale");@@ -1331,6 +1363,47 @@ mod tests {assert_eq!(crate::read_design(&bytes).unwrap().to_json(), doc.to_json());}+ /// #7168: a design saved by another version of SolidCraft (another version number and commit)+ /// whose model was computed by the same evaluation source opens from its model cache, so an+ /// update that changed only the interface or the commands does not evaluate every design again.+ /// One computed by other evaluation source (a kernel or codec change) is evaluated again.+ #[test]+ fn a_cache_of_another_version_with_the_same_evaluation_source_is_read() {+ let doc = document();+ let geometry = native_geometry();+ let bytes = write_design_container_with_cache(&doc, DesignCompression::Fast, None, Some(&geometry)).unwrap();+ let written = |entries: &BTreeMap<String, Vec<u8>>| serde_json::from_slice::<Value>(&entries["cache/model.json"]).unwrap();+ let original = parts(&bytes);+ let evaluation = written(&original)["evaluation"].as_str().unwrap().to_string();+ assert!(evaluation.ends_with(&format!("/{}-{}", std::env::consts::ARCH, std::env::consts::OS)), "{evaluation}");+ let with = |changes: &[(&str, &str)]| {+ let mut entries = original.clone();+ let mut manifest = written(&entries);+ for (key, value) in changes {+ manifest[*key] = json!(value);+ }+ entries.insert("cache/model.json".into(), serde_json::to_vec(&manifest).unwrap());+ package(&entries)+ };+ let older = with(&[("app", "0.0.1"), ("build", "0123456789abcdef0123456789abcdef01234567/other-platform")]);+ let state = read_design_geometry(&older).expect("another version's cache of the same evaluation source");+ assert_eq!(state.bodies[0].body.face_count(), geometry.bodies[0].face_count());+ assert_eq!(crate::read_design(&older).unwrap().to_json(), doc.to_json());+ // The same version with other evaluation source, or on another platform, evaluates again.+ let other_source = format!("{}/{}", "0".repeat(64), evaluation.rsplit('/').next().unwrap());+ for changed in [other_source.as_str(), &format!("{}/other-platform", evaluation.rsplit_once('/').unwrap().0)] {+ let bytes = with(&[("evaluation", changed)]);+ assert!(read_design_geometry(&bytes).is_none(), "{changed}");+ assert_eq!(crate::read_design(&bytes).unwrap().to_json(), doc.to_json());+ }+ // An older build's cache (no evaluation source recorded) is evaluated again.+ let mut entries = original.clone();+ let mut manifest = written(&entries);+ manifest.as_object_mut().unwrap().remove("evaluation");+ entries.insert("cache/model.json".into(), serde_json::to_vec(&manifest).unwrap());+ assert!(read_design_geometry(&package(&entries)).is_none());+ }+#[test]fn metadata_snapshot_survives_in_place_changes_after_preview() {use std::{cell::RefCell, rc::Rc};diff --git a/crates/io/src/design/native_cache.rs b/crates/io/src/design/native_cache.rsindex 9876d4f49d..37667d5ff5 100644--- a/crates/io/src/design/native_cache.rs+++ b/crates/io/src/design/native_cache.rs@@ -1,7 +1,7 @@//! The model cache: a design's evaluated timeline (every feature's result and the state after//! it, [`SavedModel`]), saved with it so opening need not evaluate it again. Bodies use-//! SolidCraft's exact native B-rep encoding; the key ties the cache to the exact document, build-//! and platform that computed it (docs/design-container.md).+//! SolidCraft's exact native B-rep encoding; the key ties the cache to the exact document, and to+//! the source and platform that computed it (docs/design-container.md).use super::*;use solidcraft_doc::{ModelState, SavedModel};@@ -15,12 +15,19 @@ const MAX_RECORDS: usize = 200_000;const MAX_UNIQUE: usize = 100_000;const MAX_MANIFEST: usize = 32 << 20;-/// The build that computed a cache: the same source, compiled for the same platform, computes the-/// same numbers, so only its own caches are its results.+/// The build that wrote a cache (recorded; earlier builds read only their own caches).fn build() -> String {format!("{}/{}-{}", env!("SOLIDCRAFT_SOURCE_COMMIT"), std::env::consts::ARCH, std::env::consts::OS)}+/// What computed a cache: the same evaluation source (build.rs), compiled for the same platform,+/// computes the same numbers, so only caches of that source are its results. An update that left+/// that source as it was (only the interface or the commands changed) reads the caches of the+/// build before it; one that changed it (a kernel fix, the codec) evaluates the design again.+fn evaluation() -> String {+ format!("{}/{}-{}", env!("SOLIDCRAFT_EVALUATION_SOURCE"), std::env::consts::ARCH, std::env::consts::OS)+}+/// A 4 x 4 matrix as its 16 numbers, in storage order.fn matrix(t: &[[f64; 4]; 4]) -> Vec<f64> {t.iter().flatten().copied().collect()@@ -195,7 +202,7 @@ impl Encoded {}let manifest = serde_json::to_vec(&json!({"format": FORMAT, "producer": PRODUCER, "app": env!("CARGO_PKG_VERSION"), "build": build(),- "document": digest(document), "payload": digest(&data),+ "evaluation": evaluation(), "document": digest(document), "payload": digest(&data),"slices": slices, "shares": shares, "bodies": table, "model": records,})).map_err(invalid)?;@@ -203,8 +210,8 @@ impl Encoded {}}-/// The final state of the model a design keeps, when this build computed it for exactly this-/// document. Any failure is a miss, never a document error.+/// The final state of the model a design keeps, when this build's evaluation source computed it+/// for exactly this document. Any failure is a miss, never a document error.pub fn read_design_geometry(bytes: &[u8]) -> Option<std::sync::Arc<ModelState>> {if bytes.len() > max_design() || !is_design_container(bytes) {return None;@@ -212,15 +219,16 @@ pub fn read_design_geometry(bytes: &[u8]) -> Option<std::sync::Arc<ModelState>>DesignContainerReader::new(Cursor::new(bytes)).ok()?.geometry()?.final_state().ok()}-/// Whether `manifest` is a model cache this build wrote for `document`.+/// Whether `manifest` is a model cache of this build's evaluation source for `document`. The+/// application version and commit that wrote it are not asked: another version of the same+/// evaluation source computed the same model.fn current(manifest: &Value, document: &[u8]) -> bool {let is = |key: &str, value: &str| manifest.get(key).and_then(Value::as_str) == Some(value);is("format", FORMAT)&& is("producer", PRODUCER)- && is("app", env!("CARGO_PKG_VERSION"))- && is("build", &build())+ && is("evaluation", &evaluation())&& is("document", &digest(document))- && source_trusted(env!("SOLIDCRAFT_SOURCE_COMMIT"), cfg!(any(test, feature = "trust-unknown-source")))+ && source_trusted(env!("SOLIDCRAFT_EVALUATION_SOURCE"), cfg!(any(test, feature = "trust-unknown-source")))}/// A build of unknown source (no git) never trusts a cache, unless `in_tests`: test buildsdiff --git a/docs/design-container.md b/docs/design-container.mdindex 7c260b0e49..91bfa66339 100644--- a/docs/design-container.md+++ b/docs/design-container.md@@ -149,12 +149,25 @@ again instead of being meshed on its own.`cache/model.json` records `solidcraft-model-cache/4`, the producer revision`solidcraft-truck-adapter/3`, the application version, the build (source commit,-CPU architecture and OS), the SHA-256 of the exact `document.json` (whose blob-references pin every STEP, mesh and image payload) and of `cache/bodies.bin`, the-slices, each kernel body's slice and display share, and the model's records. A-cache is read back only when every one of these matches: the same source compiled-for the same platform computes the same numbers, so another build's cache is a-miss. A build of unknown source (a tarball without git) never reads a cache.+CPU architecture and OS), the evaluation source (below, with the CPU architecture+and OS), the SHA-256 of the exact `document.json` (whose blob references pin every+STEP, mesh and image payload) and of `cache/bodies.bin`, the slices, each kernel+body's slice and display share, and the model's records. A cache is read back only+when the format, the producer, the evaluation source, the platform and both+digests match: the same source compiled for the same platform computes the same+numbers. The application version and the commit are recorded, not compared.++The evaluation source (`SOLIDCRAFT_EVALUATION_SOURCE`, `crates/io/build.rs`) is a+digest of the committed source a model is computed from: the `doc` crate and every+workspace crate it is built from (a test checks the list against their manifests),+the patched dependencies in `vendor/` (not the interface's renderer), the locked+versions of every other dependency, the workspace `Cargo.toml` without the+application's version, `.cargo/config.toml`, and the cache's own reader. An update+that changed none of it (the interface, the commands, other file formats, a new+version number) opens designs from the caches the build before it wrote (#7168);+one that changed any of it (a kernel fix, the codec, a dependency) evaluates them+again, once, and the next save writes a cache of its own. A build of unknown source+(a tarball without git) never reads a cache.The fixed budgets of earlier caches (4,096 bodies, 100,000 faces, 64 MiB) scalewith the design now: up to 200,000 body records, 100,000 distinct bodies, a 32 MiB@@ -203,9 +216,9 @@ only that.This replaces the earlier rule that a cache never supplies evaluated sessionstate. It is safe because the cache is the result of evaluating exactly this-document with exactly this build: the hashes tie it to the document (and through-it to every source blob) and to its own payload, the build key to the code that-computed it, the fingerprints to every feature's inputs, and the exact encoding+document with exactly this evaluation source: the hashes tie it to the document (and+through it to every source blob) and to its own payload, the evaluation key to the+code that computed it, the fingerprints to every feature's inputs, and the exact encodinggives back the bodies number for number. Hashes are not trust: a crafted file cancarry a cache whose geometry disagrees with its timeline, as a crafted STEP filecan carry any geometry. Every decoded body passes the same structural and resource