Skip to content

DesignCraft: An open-source page and layout design editor written by scratch in pure Rust

Assets: index placed files by content so importing many graphics stays linear (#2181)

Dylan M. Taylor74eab18d1676
diff --git a/crates/doc/src/assets.rs b/crates/doc/src/assets.rs
index 1533b8b4b..6ddd52b1e 100644
--- a/crates/doc/src/assets.rs
+++ b/crates/doc/src/assets.rs
@@ -112,8 +112,13 @@ pub fn find_same_asset(assets: &BTreeMap<AssetId, Arc<Asset>>, asset: &Asset) ->
/// Add `asset` to `assets` unless they already hold the same file; returns the id placements
/// should reference. A new asset with the bytes of an existing one shares its buffer.
+///
+/// This scans `assets`; an importer adding many files uses an [`AssetIndex`] instead.
pub fn insert_asset(assets: &mut BTreeMap<AssetId, Arc<Asset>>, mut asset: Asset) -> AssetId {
- if let Some(id) = find_same_asset(assets, &asset) {
+ // An empty, unlinked asset never holds the same file as another: nothing to search for.
+ if (!asset.data.is_empty() || asset.link.is_some())
+ && let Some(id) = find_same_asset(assets, &asset)
+ {
return id;
}
if !asset.data.is_empty()
@@ -126,6 +131,51 @@ pub fn insert_asset(assets: &mut BTreeMap<AssetId, Arc<Asset>>, mut asset: Asset
id
}
+/// A lookup of the assets added through it, by content, so that adding many files (an import)
+/// costs time in proportion to their number rather than its square. It only knows the assets
+/// it inserted: use one index per asset map, and add every asset through it.
+#[derive(Debug, Default)]
+pub struct AssetIndex {
+ fingerprint: std::collections::hash_map::RandomState,
+ /// Candidates for [`Asset::same_file`]: (bytes fingerprint, type, page, link).
+ files: HashMap<(u64, String, u32, Option<String>), Vec<AssetId>>,
+ /// One asset per distinct non-empty byte content, for buffer sharing.
+ bytes: HashMap<u64, Vec<AssetId>>,
+}
+
+impl AssetIndex {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ /// [`insert_asset`], looking up candidates by fingerprint instead of scanning `assets`.
+ /// A fingerprint only selects candidates; equality still compares the actual files.
+ pub fn insert_asset(&mut self, assets: &mut BTreeMap<AssetId, Arc<Asset>>, mut asset: Asset) -> AssetId {
+ // Empty, unlinked assets never compare equal: don't search for or index them.
+ if asset.data.is_empty() && asset.link.is_none() {
+ let id = asset.id;
+ assets.insert(id, Arc::new(asset));
+ return id;
+ }
+ let hash = self.fingerprint.hash_one(asset.data.as_slice());
+ let key = (hash, asset.mime.clone(), asset.page, asset.link.clone());
+ if let Some(id) = self.files.get(&key).and_then(|ids| ids.iter().copied().find(|id| assets.get(id).is_some_and(|a| a.same_file(&asset)))) {
+ return id;
+ }
+ if !asset.data.is_empty() {
+ let bucket = self.bytes.entry(hash).or_default();
+ match bucket.iter().find_map(|id| assets.get(id).filter(|a| a.same_bytes(&asset))) {
+ Some(a) => asset.data = Arc::clone(&a.data),
+ None => bucket.push(asset.id),
+ }
+ }
+ let id = asset.id;
+ self.files.entry(key).or_default().push(id);
+ assets.insert(id, Arc::new(asset));
+ id
+ }
+}
+
/// Calls `f` with every asset reference in `it` and its children; false when nesting exceeded
/// [`MAX_DEPTH`] (some references weren't visited).
fn item_refs(it: &Item, f: &mut impl FnMut(AssetId), depth: usize) -> bool {
@@ -463,6 +513,54 @@ mod tests {
assert_ne!(m3, m4);
}
+ /// The index dedupes exactly as the scan does, and skips empty unlinked assets.
+ #[test]
+ fn indexed_insert_matches_scanning_insert() {
+ let linked = |id, path: &str, data: &[u8]| Asset { link: Some(path.into()), ..asset(id, data) };
+ let cases = vec![
+ asset(1, b"same"),
+ asset(2, b"same"),
+ asset(3, b"diff"),
+ Asset { page: 1, ..asset(4, b"same") },
+ linked(5, "/p/a.png", b"same"),
+ linked(6, "/q/b.png", b"same"),
+ linked(7, "/p/a.png", b"same"),
+ linked(8, "/m/1.png", b""),
+ linked(9, "/m/1.png", b""),
+ asset(10, b""),
+ asset(11, b""),
+ ];
+ let mut scanned = BTreeMap::new();
+ let mut indexed = BTreeMap::new();
+ let mut index = AssetIndex::new();
+ for a in cases {
+ let s = insert_asset(&mut scanned, a.clone());
+ let i = index.insert_asset(&mut indexed, a);
+ assert_eq!(s, i);
+ }
+ assert_eq!(scanned.keys().collect::<Vec<_>>(), indexed.keys().collect::<Vec<_>>());
+ assert!(Arc::ptr_eq(&indexed[&AssetId(5)].data, &indexed[&AssetId(6)].data), "identical bytes share one buffer");
+ }
+
+ /// Many empty unlinked placements (graphics with neither bytes nor a link) insert without
+ /// comparing against each other.
+ #[test]
+ fn many_empty_unlinked_assets_insert_quickly() {
+ let mut assets = BTreeMap::new();
+ let mut index = AssetIndex::new();
+ for id in 1..=200_000u64 {
+ assert_eq!(insert_asset(&mut assets, asset(id, b"")), AssetId(id));
+ }
+ assert_eq!(assets.len(), 200_000);
+ let mut indexed = BTreeMap::new();
+ for id in 1..=20_000u64 {
+ let mut bytes = vec![7; 64];
+ bytes[56..].copy_from_slice(&id.to_le_bytes());
+ assert_eq!(index.insert_asset(&mut indexed, asset(id, &bytes)), AssetId(id));
+ }
+ assert_eq!(indexed.len(), 20_000);
+ }
+
#[test]
fn files_at_different_paths_stay_separate_assets() {
let mut d = Document::new(&Default::default());
diff --git a/crates/doc/src/lib.rs b/crates/doc/src/lib.rs
index 8861dd924..cb0e4a020 100644
--- a/crates/doc/src/lib.rs
+++ b/crates/doc/src/lib.rs
@@ -44,7 +44,7 @@ use std::collections::BTreeMap;
use std::sync::Arc;
pub use anchored::{AnchorPosition, AnchoredObject, OBJECT_MARK};
-pub use assets::{find_same_asset, insert_asset, remap_story_assets, story_asset_refs};
+pub use assets::{AssetIndex, find_same_asset, insert_asset, remap_story_assets, story_asset_refs};
pub use attrs::*;
pub use datamerge::{
DataField, DataFieldKind, DataMerge, DataSource, Delimiter, Fingerprint, MergeOptions, Placeholder, PlaceholderAnchor, PlaceholderRole,
diff --git a/crates/idml/src/import.rs b/crates/idml/src/import.rs
index 87f84a7db..3f708b7cd 100644
--- a/crates/idml/src/import.rs
+++ b/crates/idml/src/import.rs
@@ -195,6 +195,8 @@ struct Importer<'r> {
page_index: HashMap<String, usize>,
item_ids: HashMap<String, ItemId>,
assets: BTreeMap<AssetId, Arc<Asset>>,
+ /// Content lookup for `assets`, so each placed graphic doesn't compare against every earlier one.
+ asset_index: designcraft_doc::AssetIndex,
/// Assets read from a link path at a page and crop, so later placements of that path, page
/// and crop reuse them unread.
linked_assets: HashMap<(String, u32, designcraft_doc::PdfCrop), AssetId>,
@@ -277,6 +279,7 @@ impl<'r> Importer<'r> {
page_index: HashMap::new(),
item_ids: HashMap::new(),
assets: BTreeMap::new(),
+ asset_index: designcraft_doc::AssetIndex::new(),
linked_assets: HashMap::new(),
ctx: ItemCtx { threads: Vec::new() },
sections: Vec::new(),
diff --git a/crates/idml/src/import/item.rs b/crates/idml/src/import/item.rs
index d25044dc6..ac27a833e 100644
--- a/crates/idml/src/import/item.rs
+++ b/crates/idml/src/import/item.rs
@@ -371,7 +371,7 @@ impl<'r> Importer<'r> {
uri.as_deref().and_then(|p| p.rsplit(['/', '\\']).next()).filter(|n| !n.is_empty()).map(str::to_string).unwrap_or_else(|| "image".into());
let fresh = AssetId(self.alloc());
let link_path = uri.filter(|p| p.contains('/') || p.contains('\\'));
- let id = designcraft_doc::insert_asset(
+ let id = self.asset_index.insert_asset(
&mut self.assets,
Asset {
page,