DesignCraft: An open-source page and layout design editor written by scratch in pure Rust
Assets: index placed files by content so importing many graphics stays linear (#2181)
Dylan M. Taylor74eab18d1676
diff --git a/crates/doc/src/assets.rs b/crates/doc/src/assets.rsindex 1533b8b4b..6ddd52b1e 100644--- a/crates/doc/src/assets.rs+++ b/crates/doc/src/assets.rs@@ -112,8 +112,13 @@ pub fn find_same_asset(assets: &BTreeMap<AssetId, Arc<Asset>>, asset: &Asset) ->/// Add `asset` to `assets` unless they already hold the same file; returns the id placements/// should reference. A new asset with the bytes of an existing one shares its buffer.+///+/// This scans `assets`; an importer adding many files uses an [`AssetIndex`] instead.pub fn insert_asset(assets: &mut BTreeMap<AssetId, Arc<Asset>>, mut asset: Asset) -> AssetId {- if let Some(id) = find_same_asset(assets, &asset) {+ // An empty, unlinked asset never holds the same file as another: nothing to search for.+ if (!asset.data.is_empty() || asset.link.is_some())+ && let Some(id) = find_same_asset(assets, &asset)+ {return id;}if !asset.data.is_empty()@@ -126,6 +131,51 @@ pub fn insert_asset(assets: &mut BTreeMap<AssetId, Arc<Asset>>, mut asset: Assetid}+/// A lookup of the assets added through it, by content, so that adding many files (an import)+/// costs time in proportion to their number rather than its square. It only knows the assets+/// it inserted: use one index per asset map, and add every asset through it.+#[derive(Debug, Default)]+pub struct AssetIndex {+ fingerprint: std::collections::hash_map::RandomState,+ /// Candidates for [`Asset::same_file`]: (bytes fingerprint, type, page, link).+ files: HashMap<(u64, String, u32, Option<String>), Vec<AssetId>>,+ /// One asset per distinct non-empty byte content, for buffer sharing.+ bytes: HashMap<u64, Vec<AssetId>>,+}++impl AssetIndex {+ pub fn new() -> Self {+ Self::default()+ }++ /// [`insert_asset`], looking up candidates by fingerprint instead of scanning `assets`.+ /// A fingerprint only selects candidates; equality still compares the actual files.+ pub fn insert_asset(&mut self, assets: &mut BTreeMap<AssetId, Arc<Asset>>, mut asset: Asset) -> AssetId {+ // Empty, unlinked assets never compare equal: don't search for or index them.+ if asset.data.is_empty() && asset.link.is_none() {+ let id = asset.id;+ assets.insert(id, Arc::new(asset));+ return id;+ }+ let hash = self.fingerprint.hash_one(asset.data.as_slice());+ let key = (hash, asset.mime.clone(), asset.page, asset.link.clone());+ if let Some(id) = self.files.get(&key).and_then(|ids| ids.iter().copied().find(|id| assets.get(id).is_some_and(|a| a.same_file(&asset)))) {+ return id;+ }+ if !asset.data.is_empty() {+ let bucket = self.bytes.entry(hash).or_default();+ match bucket.iter().find_map(|id| assets.get(id).filter(|a| a.same_bytes(&asset))) {+ Some(a) => asset.data = Arc::clone(&a.data),+ None => bucket.push(asset.id),+ }+ }+ let id = asset.id;+ self.files.entry(key).or_default().push(id);+ assets.insert(id, Arc::new(asset));+ id+ }+}+/// Calls `f` with every asset reference in `it` and its children; false when nesting exceeded/// [`MAX_DEPTH`] (some references weren't visited).fn item_refs(it: &Item, f: &mut impl FnMut(AssetId), depth: usize) -> bool {@@ -463,6 +513,54 @@ mod tests {assert_ne!(m3, m4);}+ /// The index dedupes exactly as the scan does, and skips empty unlinked assets.+ #[test]+ fn indexed_insert_matches_scanning_insert() {+ let linked = |id, path: &str, data: &[u8]| Asset { link: Some(path.into()), ..asset(id, data) };+ let cases = vec![+ asset(1, b"same"),+ asset(2, b"same"),+ asset(3, b"diff"),+ Asset { page: 1, ..asset(4, b"same") },+ linked(5, "/p/a.png", b"same"),+ linked(6, "/q/b.png", b"same"),+ linked(7, "/p/a.png", b"same"),+ linked(8, "/m/1.png", b""),+ linked(9, "/m/1.png", b""),+ asset(10, b""),+ asset(11, b""),+ ];+ let mut scanned = BTreeMap::new();+ let mut indexed = BTreeMap::new();+ let mut index = AssetIndex::new();+ for a in cases {+ let s = insert_asset(&mut scanned, a.clone());+ let i = index.insert_asset(&mut indexed, a);+ assert_eq!(s, i);+ }+ assert_eq!(scanned.keys().collect::<Vec<_>>(), indexed.keys().collect::<Vec<_>>());+ assert!(Arc::ptr_eq(&indexed[&AssetId(5)].data, &indexed[&AssetId(6)].data), "identical bytes share one buffer");+ }++ /// Many empty unlinked placements (graphics with neither bytes nor a link) insert without+ /// comparing against each other.+ #[test]+ fn many_empty_unlinked_assets_insert_quickly() {+ let mut assets = BTreeMap::new();+ let mut index = AssetIndex::new();+ for id in 1..=200_000u64 {+ assert_eq!(insert_asset(&mut assets, asset(id, b"")), AssetId(id));+ }+ assert_eq!(assets.len(), 200_000);+ let mut indexed = BTreeMap::new();+ for id in 1..=20_000u64 {+ let mut bytes = vec![7; 64];+ bytes[56..].copy_from_slice(&id.to_le_bytes());+ assert_eq!(index.insert_asset(&mut indexed, asset(id, &bytes)), AssetId(id));+ }+ assert_eq!(indexed.len(), 20_000);+ }+#[test]fn files_at_different_paths_stay_separate_assets() {let mut d = Document::new(&Default::default());diff --git a/crates/doc/src/lib.rs b/crates/doc/src/lib.rsindex 8861dd924..cb0e4a020 100644--- a/crates/doc/src/lib.rs+++ b/crates/doc/src/lib.rs@@ -44,7 +44,7 @@ use std::collections::BTreeMap;use std::sync::Arc;pub use anchored::{AnchorPosition, AnchoredObject, OBJECT_MARK};-pub use assets::{find_same_asset, insert_asset, remap_story_assets, story_asset_refs};+pub use assets::{AssetIndex, find_same_asset, insert_asset, remap_story_assets, story_asset_refs};pub use attrs::*;pub use datamerge::{DataField, DataFieldKind, DataMerge, DataSource, Delimiter, Fingerprint, MergeOptions, Placeholder, PlaceholderAnchor, PlaceholderRole,diff --git a/crates/idml/src/import.rs b/crates/idml/src/import.rsindex 87f84a7db..3f708b7cd 100644--- a/crates/idml/src/import.rs+++ b/crates/idml/src/import.rs@@ -195,6 +195,8 @@ struct Importer<'r> {page_index: HashMap<String, usize>,item_ids: HashMap<String, ItemId>,assets: BTreeMap<AssetId, Arc<Asset>>,+ /// Content lookup for `assets`, so each placed graphic doesn't compare against every earlier one.+ asset_index: designcraft_doc::AssetIndex,/// Assets read from a link path at a page and crop, so later placements of that path, page/// and crop reuse them unread.linked_assets: HashMap<(String, u32, designcraft_doc::PdfCrop), AssetId>,@@ -277,6 +279,7 @@ impl<'r> Importer<'r> {page_index: HashMap::new(),item_ids: HashMap::new(),assets: BTreeMap::new(),+ asset_index: designcraft_doc::AssetIndex::new(),linked_assets: HashMap::new(),ctx: ItemCtx { threads: Vec::new() },sections: Vec::new(),diff --git a/crates/idml/src/import/item.rs b/crates/idml/src/import/item.rsindex d25044dc6..ac27a833e 100644--- a/crates/idml/src/import/item.rs+++ b/crates/idml/src/import/item.rs@@ -371,7 +371,7 @@ impl<'r> Importer<'r> {uri.as_deref().and_then(|p| p.rsplit(['/', '\\']).next()).filter(|n| !n.is_empty()).map(str::to_string).unwrap_or_else(|| "image".into());let fresh = AssetId(self.alloc());let link_path = uri.filter(|p| p.contains('/') || p.contains('\\'));- let id = designcraft_doc::insert_asset(+ let id = self.asset_index.insert_asset(&mut self.assets,Asset {page,