SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.
Save designs with large STEP imports: compress the STEP part (#3221) (#3285)
Christian Svensson38c710e72e9e
diff --git a/crates/io/src/design.rs b/crates/io/src/design.rsindex 510d800d7e..42f4fc9277 100644--- a/crates/io/src/design.rs+++ b/crates/io/src/design.rs@@ -20,7 +20,15 @@ const VERSION: &str = "solidcraft-container/1";const LEGACY_VERSION: &str = "solvecraft-container/1";const PREVIEW_VERSION: &str = "solidcraft-preview/1";const MAX_ENTRIES: usize = 10_000;-const MAX_EXPANDED: usize = 512 << 20;+/// An embedded STEP source may be as large as an import accepts. It is always stored+/// Deflate-compressed, so the file itself stays within [`MAX_DESIGN_BYTES`] (#3221).+const MAX_STEP_PART: usize = crate::MAX_STEP_BYTES;+/// Deflate level for STEP sources. Level 1 uses fixed Huffman codes: a 270 MiB STEP export+/// shrank to 70 MB in 1.6 s, at level 2 to 47 MB in 3.1 s; higher levels gain little more.+const STEP_LEVEL: i64 = 2;+/// Every other part is bounded by the file size limit.+const MAX_PART: usize = MAX_DESIGN_BYTES;+const MAX_EXPANDED: usize = MAX_STEP_PART + MAX_DESIGN_BYTES;const MESH_MAGIC: &[u8; 8] = b"SCMESH01";const MAX_PREVIEW_BYTES: usize = 8 << 20;const MAX_PREVIEW_TRIANGLES: usize = 100_000;@@ -65,7 +73,7 @@ struct PackageWriter {impl PackageWriter {fn part(&mut self, name: &str, bytes: &[u8], compressed: bool) -> Result<()> {- if bytes.len() > MAX_DESIGN_BYTES {+ if bytes.len() > if name.ends_with(".step") { MAX_STEP_PART } else { MAX_PART } {return Err(invalid("part too large"));}if !self.names.insert(name.to_owned()) {@@ -76,10 +84,21 @@ impl PackageWriter {return Err(invalid("package too large"));}let method = if compressed { zip::CompressionMethod::Deflated } else { zip::CompressionMethod::Stored };- let options = zip::write::SimpleFileOptions::default().compression_method(method).compression_level(compressed.then_some(1));+ let level = if name.ends_with(".step") { STEP_LEVEL } else { 1 };+ let options = zip::write::SimpleFileOptions::default().compression_method(method).compression_level(compressed.then_some(level));self.zip.start_file(name, options).map_err(invalid)?;self.zip.write_all(bytes).map_err(invalid)}+ /// Would `entries` more optional parts of `extra` bytes, stored uncompressed, still keep+ /// the file within `budget` and its expanded size within the reader's budget?+ fn room(&self, entries: usize, extra: usize, budget: usize) -> bool {+ // The bytes written so far, plus what the open compressed part may still hold back.+ let written = self.zip.get_ref().map_or(usize::MAX, |c| c.get_ref().len()).saturating_add(1 << 20);+ let directory = self.names.len().saturating_add(entries).saturating_mul(256).saturating_add(65_536);+ self.names.len().saturating_add(entries) <= MAX_ENTRIES+ && written.saturating_add(extra).saturating_add(directory) <= budget+ && self.expanded.saturating_add(extra) <= MAX_EXPANDED+ }fn blob(&mut self, bytes: &[u8], extension: &str, compressed: bool) -> Result<Value> {self.resolved =self.resolved.checked_add(bytes.len()).filter(|&n| n <= MAX_EXPANDED).ok_or_else(|| invalid("resolved payloads too large"))?;@@ -89,9 +108,70 @@ impl PackageWriter {}}+/// Counts what is written without keeping it.+#[derive(Default)]+struct Measure {+ position: u64,+ length: u64,+}++impl Write for Measure {+ fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {+ self.position = self.position.saturating_add(bytes.len() as u64);+ self.length = self.length.max(self.position);+ Ok(bytes.len())+ }+ fn flush(&mut self) -> std::io::Result<()> {+ Ok(())+ }+}++impl Seek for Measure {+ fn seek(&mut self, position: SeekFrom) -> std::io::Result<u64> {+ let target = match position {+ SeekFrom::Start(n) => Some(n),+ SeekFrom::End(n) => self.length.checked_add_signed(n),+ SeekFrom::Current(n) => self.position.checked_add_signed(n),+ };+ self.position = target.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "seek out of range"))?;+ Ok(self.position)+ }+}++/// Room kept in a design file for everything besides one large embedded STEP source.+const STEP_RESERVE: usize = 4 << 20;++/// Can an import's STEP text be saved in a design? It must be no larger than an import+/// accepts, and, compressed as a save stores it, leave room in a design file for the rest of+/// the design. Checked when the file is imported, so it never imports and then fails to save.+pub fn check_step_storable(step: &str) -> Result<()> {+ let limit = MAX_DESIGN_BYTES - STEP_RESERVE;+ if step.len() > MAX_STEP_PART {+ return Err(IoError::Invalid(format!("STEP file too large ({} MB, limit {} MB)", step.len() >> 20, MAX_STEP_PART >> 20)));+ }+ // Deflate adds at most a few bytes per 16 KiB block: smaller text always fits.+ if step.len() <= limit - (limit >> 10) {+ return Ok(());+ }+ let mut zip = zip::ZipWriter::new(Measure::default());+ let options = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated).compression_level(Some(STEP_LEVEL));+ zip.start_file("blobs/source.step", options).map_err(invalid)?;+ zip.write_all(step.as_bytes()).map_err(invalid)?;+ let compressed = usize::try_from(zip.finish().map_err(invalid)?.length).unwrap_or(usize::MAX);+ if compressed > limit {+ return Err(IoError::Invalid(format!(+ "STEP file too large to save in a design: {} MB of STEP text compresses to {} MB, over the {} MB a design file may hold",+ step.len() >> 20,+ compressed >> 20,+ limit >> 20+ )));+ }+ Ok(())+}+/// Write a compact container, deduplicating payloads by SHA-256. Mesh arrays are-/// stored directly for low save/open latency; small metadata uses level-one-/// Deflate. Use the explicit Compact option to compress STEP and meshes too.+/// stored directly for low save/open latency; STEP sources and small metadata use+/// fast Deflate. Use the explicit Compact option to compress meshes too.pub fn write_design_container(doc: &Document) -> Result<Vec<u8>> {write_design_container_with_compression(doc, DesignCompression::Fast)}@@ -145,7 +225,9 @@ fn write_container(for feature in &doc.features {let replacements = match &feature.kind {FeatureKind::Import { step, .. } => {- let blob = writer.blob(step.as_bytes(), "step", compression == DesignCompression::Compact)?;+ // STEP is text and shrinks several times: always compressed, so an import as large+ // as MAX_STEP_BYTES still fits in a design file.+ let blob = writer.blob(step.as_bytes(), "step", true)?;fields([("step", Value::String(String::new())), ("step_blob", blob)])}FeatureKind::MeshImport { meshes, .. } => {@@ -190,14 +272,7 @@ fn write_container(// The model cache is optional: one that does not fit the container is left out, never a// failed save.let cache = geometry.and_then(|g| native_cache::encode_model(g, &metadata).ok()).filter(|(manifest, data)| {- let fits = writer.names.len().saturating_add(2) <= MAX_ENTRIES- && writer- .expanded- .saturating_add(manifest.len())- .saturating_add(data.len())- .saturating_add(writer.names.len().saturating_mul(256))- .saturating_add(65_536)- <= budget;+ let fits = writer.room(2, manifest.len().saturating_add(data.len()), budget);if !fits {log::info!("the model cache ({} MB) does not fit the design file: saved without it", (manifest.len() + data.len()) >> 20);}@@ -211,14 +286,7 @@ fn write_container(writer.part(payload_entry, &data, true)?;}if let Some((manifest, data)) = preview.and_then(|p| encode_preview(p, &metadata).ok())- && writer.names.len().saturating_add(2) <= MAX_ENTRIES- && writer- .expanded- .saturating_add(manifest.len())- .saturating_add(data.len())- .saturating_add(writer.names.len().saturating_mul(256))- .saturating_add(65_536)- <= budget+ && writer.room(2, manifest.len().saturating_add(data.len()), budget){writer.part("cache/preview.json", &manifest, false)?;writer.part("cache/preview.bin", &data, false)?;@@ -228,7 +296,7 @@ fn write_container(}let bytes = writer.zip.finish().map_err(invalid)?.into_inner();if bytes.len() > MAX_DESIGN_BYTES {- return Err(invalid("compressed file too large"));+ return Err(invalid(format!("the design would be {} MB, over the {} MB a design file may hold", bytes.len() >> 20, MAX_DESIGN_BYTES >> 20)));}Ok(bytes)}@@ -345,7 +413,8 @@ impl<R: Read + Seek> PackageReader<R> {return Err(invalid("duplicate or invalid archive entry"));}total = total.checked_add(part.size()).ok_or_else(|| invalid("expanded size overflow"))?;- if part.size() > MAX_DESIGN_BYTES as u64 || total > MAX_EXPANDED as u64 {+ let limit = if name.ends_with(".step") { MAX_STEP_PART } else { MAX_PART };+ if part.size() > limit as u64 || total > MAX_EXPANDED as u64 {return Err(invalid("expanded package too large"));}}@@ -360,7 +429,7 @@ impl<R: Read + Seek> PackageReader<R> {self.document.as_ref().filter(|d| d.len() <= limit).cloned().ok_or_else(|| invalid("metadata budget"))}fn part(&mut self, name: &str) -> Result<Vec<u8>> {- self.limited_part(name, MAX_DESIGN_BYTES)+ self.limited_part(name, if name.ends_with(".step") { MAX_STEP_PART } else { MAX_PART })}fn limited_part(&mut self, name: &str, limit: usize) -> Result<Vec<u8>> {let part = self.zip.by_name(name).map_err(|e| invalid(format!("required part `{name}`: {e}")))?;@@ -369,7 +438,9 @@ impl<R: Read + Seek> PackageReader<R> {return Err(invalid("part too large"));}let mut bytes = Vec::with_capacity(expected as usize);- part.take(limit as u64 + 1).read_to_end(&mut bytes).map_err(invalid)?;+ // A compressed part may expand beyond its declared size (a decompression bomb): stop+ // one byte past it instead of at the limit.+ part.take(expected + 1).read_to_end(&mut bytes).map_err(invalid)?;if bytes.len() > limit || bytes.len() as u64 != expected {return Err(invalid("part too large or length mismatch"));}@@ -676,7 +747,7 @@ mod tests {#[test]fn seekable_preview_defers_large_source_blob_until_document_read() {let mut doc = Document::new("Deferred source");- let step = "ISO-10303-21;".repeat(200_000);+ let step = noise(2_600_000);doc.add_feature(FeatureKind::Import { file: "large.step".into(), step: step.clone().into(), components: Vec::new(), skip: Vec::new() }, None).unwrap();let preview = DesignPreview { meshes: vec![mesh()] };@@ -685,12 +756,12 @@ mod tests {let source = CountingReader { source: Cursor::new(bytes), count: count.clone() };let mut reader = DesignContainerReader::new(source).unwrap();assert_eq!(reader.preview().unwrap().meshes[0].positions, preview.meshes[0].positions);- // ZIP directory scans may touch a bounded suffix of a stored blob. They+ // ZIP directory scans may touch a bounded suffix of a source blob. They// must not load its multi-megabyte payload to obtain the saved picture.assert!(count.get() < 256 * 1024, "read {} bytes before display", count.get());let restored = reader.read_document().unwrap();assert_eq!(restored.to_json(), doc.to_json());- assert!(count.get() >= step.len());+ assert!(count.get() >= step.len() / 4);}#[test]@@ -698,9 +769,11 @@ mod tests {let doc = document();let preview = DesignPreview { meshes: vec![mesh()] };let mut bytes = write_design_container_with_preview(&doc, DesignCompression::Fast, Some(&preview)).unwrap();- let marker = b"ISO-10303-21;";- let offset = bytes.windows(marker.len()).position(|part| part == marker).unwrap();- bytes[offset] ^= 1;+ // The first part written is the (compressed) STEP source: damage its data.+ let name = u16::from_le_bytes([bytes[26], bytes[27]]) as usize;+ let extra = u16::from_le_bytes([bytes[28], bytes[29]]) as usize;+ assert!(bytes[30..30 + name].ends_with(b".step"));+ bytes[30 + name + extra + 2] ^= 0x55;let mut reader = DesignContainerReader::new(Cursor::new(bytes)).unwrap();assert!(reader.preview().is_some());assert!(reader.read_document().is_err());@@ -752,7 +825,7 @@ mod tests {#[test]fn native_cache_round_trip_and_preview_defer_original_sources() {let mut doc = Document::new("Native cache");- let step = "ISO-10303-21;".repeat(200_000);+ let step = noise(2_600_000);doc.add_feature(FeatureKind::Import { file: "large.step".into(), step: step.clone().into(), components: Vec::new(), skip: Vec::new() }, None).unwrap();let bodies = native_geometry();@@ -767,7 +840,7 @@ mod tests {assert!(!reader.preview().unwrap().meshes.is_empty());assert!(count.get() < 256 * 1024, "read {} bytes before native display", count.get());assert_eq!(reader.read_document().unwrap().to_json(), doc.to_json());- assert!(count.get() >= step.len());+ assert!(count.get() >= step.len() / 4);}#[test]@@ -1159,7 +1232,7 @@ mod tests {assert!(crate::read_design(&duplicate).is_err());let mut bomb = bytes.clone();let central = bomb.windows(4).position(|v| v == b"PK\x01\x02").unwrap();- bomb[central + 24..central + 28].copy_from_slice(&((MAX_DESIGN_BYTES + 1) as u32).to_le_bytes());+ bomb[central + 24..central + 28].copy_from_slice(&((MAX_STEP_PART + 1) as u32).to_le_bytes());assert!(crate::read_design(&bomb).unwrap_err().to_string().contains("expanded package too large"));let mut excessive = bytes;let end = excessive.windows(4).rposition(|v| v == b"PK\x05\x06").unwrap();@@ -1167,6 +1240,114 @@ mod tests {assert!(crate::read_design(&excessive).is_err());}+ /// Printable text that compresses poorly (93 characters: about 6.5 bits each). No `*`, so+ /// it never ends a STEP comment.+ fn noise(length: usize) -> String {+ let mut state = 0x2545_f491_4f6c_dd1du64;+ (0..length)+ .map(|_| {+ state ^= state << 13;+ state ^= state >> 7;+ state ^= state << 17;+ let c = b'!' + ((state >> 32) % 93) as u8;+ char::from(if c >= b'*' { c + 1 } else { c })+ })+ .collect()+ }++ /// A valid STEP cube, padded with a comment to `length` bytes. `noisy` padding barely+ /// compresses; otherwise it is repetitive like the text of a real large export.+ fn padded_step(length: usize, noisy: bool) -> String {+ use solidcraft_geom::Vec3;+ let cube = solidcraft_kernel::box_solid(Vec3::ZERO, Vec3::new(10.0, 20.0, 30.0)).unwrap();+ let step = solidcraft_kernel::step_export(&[&cube], "padded").unwrap();+ let (head, tail) = step.split_at(step.find("DATA;").unwrap() + 5);+ let end = length - tail.len() - 2;+ let mut out = String::with_capacity(length);+ out.push_str(head);+ out.push_str("\n/*");+ if noisy {+ out.push_str(&noise(end - out.len()));+ } else {+ let mut line = 0u64;+ while out.len() + 64 < end {+ line += 1;+ out.push_str(&format!("#{line}=CARTESIAN_POINT('',({}.,{}.5,-{}.25));\n", line % 997, line % 89, line % 13));+ }+ out.extend(std::iter::repeat_n(' ', end - out.len()));+ }+ out.push_str("*/");+ out.push_str(tail);+ assert_eq!(out.len(), length);+ out+ }++ /// #3221: a STEP file larger than a design part used to import, then fail to save.+ #[test]+ fn step_over_the_part_limit_imports_saves_compressed_and_reopens() {+ let step = padded_step(MAX_DESIGN_BYTES + (1 << 20), false);+ let imported = crate::step_import_feature(step.as_bytes(), "large.step").unwrap();+ drop(step);+ let mut doc = Document::new("Large import");+ doc.add_feature(imported.kind, None).unwrap();+ let bytes = write_design_container(&doc).unwrap();+ assert!(bytes.len() < MAX_DESIGN_BYTES / 2, "saved {} MB", bytes.len() >> 20);+ let restored = crate::read_design(&bytes).unwrap();+ let (Some(FeatureKind::Import { step: before, .. }), Some(FeatureKind::Import { step: after, .. })) =+ (doc.features.first().map(|f| &f.kind), restored.features.first().map(|f| &f.kind))+ else {+ panic!("the reopened design lost its import");+ };+ assert!(before.len() > MAX_DESIGN_BYTES && before == after, "the reopened STEP text differs");+ let reopened = crate::step_import_feature(after.as_bytes(), "large.step").unwrap();+ assert_eq!(reopened.body_names, imported.body_names);+ assert_eq!(reopened.import.bodies.len(), 1);+ }++ #[test]+ fn step_that_cannot_be_saved_even_compressed_is_refused_at_import() {+ let step = padded_step(MAX_DESIGN_BYTES + (80 << 20), true);+ let message = crate::step_import_feature(step.as_bytes(), "noisy.step").err().unwrap().to_string();+ assert!(message.contains("too large to save in a design"), "{message}");+ }++ #[test]+ fn designs_with_uncompressed_step_sources_still_open() {+ // Saves before #3221 stored the STEP source without compression.+ let doc = document();+ let entries = parts(&write_design_container(&doc).unwrap());+ let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));+ for (name, bytes) in &entries {+ let method = if name.ends_with(".step") { zip::CompressionMethod::Stored } else { zip::CompressionMethod::Deflated };+ zip.start_file(name, zip::write::SimpleFileOptions::default().compression_method(method)).unwrap();+ zip.write_all(bytes).unwrap();+ }+ let old = zip.finish().unwrap().into_inner();+ assert!(old.windows(13).any(|v| v == b"ISO-10303-21;"));+ assert_eq!(crate::read_design(&old).unwrap(), doc);+ }++ #[test]+ fn decompression_bomb_is_refused_without_expanding_it() {+ // 64 MiB that deflate to a few hundred kilobytes, declared as a 1 KiB STEP source.+ let mut doc = Document::new("Bomb");+ let step = format!("ISO-10303-21;{}", " ".repeat(64 << 20));+ doc.add_feature(FeatureKind::Import { file: "bomb.step".into(), step: step.into(), components: Vec::new(), skip: Vec::new() }, None).unwrap();+ let honest = write_design_container(&doc).unwrap();+ assert!(honest.len() < 1 << 20);+ let central = honest.windows(4).position(|v| v == b"PK\x01\x02").unwrap();+ let mut bomb = honest.clone();+ // The STEP source is the first part: its local header is at 0.+ bomb[22..26].copy_from_slice(&1024u32.to_le_bytes());+ bomb[central + 24..central + 28].copy_from_slice(&1024u32.to_le_bytes());+ let message = crate::read_design(&bomb).unwrap_err().to_string();+ assert!(message.contains("declared uncompressed size") || message.contains("length mismatch"), "{message}");+ // An honest declaration over the limit is refused before the part is read at all.+ let mut over = honest;+ over[central + 24..central + 28].copy_from_slice(&((MAX_STEP_PART + 1) as u32).to_le_bytes());+ assert!(crate::read_design(&over).unwrap_err().to_string().contains("expanded package too large"));+ }+#[test]fn rejects_invalid_references_and_repeated_payload_expansion() {let bytes = write_design_container(&document()).unwrap();diff --git a/crates/io/src/design/thumbnail.rs b/crates/io/src/design/thumbnail.rsindex ff375e67a2..6d2b018c1c 100644--- a/crates/io/src/design/thumbnail.rs+++ b/crates/io/src/design/thumbnail.rs@@ -17,15 +17,7 @@ pub(super) fn write(writer: &mut PackageWriter, png: &[u8], document: &[u8]) ->}let key = serde_json::to_vec(&json!({"format": FORMAT, "app": env!("CARGO_PKG_VERSION"), "document": digest(document), "payload": digest(png)})).map_err(invalid)?;- if writer.names.len().saturating_add(2) > MAX_ENTRIES- || writer- .expanded- .saturating_add(key.len())- .saturating_add(png.len())- .saturating_add(writer.names.len().saturating_mul(256))- .saturating_add(65_536)- > MAX_DESIGN_BYTES- {+ if !writer.room(2, key.len().saturating_add(png.len()), MAX_DESIGN_BYTES) {return Ok(());}writer.part("cache/thumbnail.json", &key, false)?;diff --git a/crates/io/src/lib.rs b/crates/io/src/lib.rsindex 0594b98826..5ab6ca9df3 100644--- a/crates/io/src/lib.rs+++ b/crates/io/src/lib.rs@@ -216,6 +216,8 @@ pub fn step_import_feature_named(bytes: &[u8], file: &str, names: StepNames) ->let text = step_text(bytes);if is_iges_path(file) { solidcraft_kernel::iges_to_step(&text).map_err(IoError::Invalid)? } else { (text, Vec::new()) }};+ // The feature keeps this text and a save stores it: refuse now what could not be saved.+ design::check_step_storable(&text)?;let imp = solidcraft_kernel::step_import_shared(&text)?;let stem = std::path::Path::new(file).file_stem()