DesignCraft: An open-source page and layout design editor written by scratch in pure Rust
Assets: one asset per image, and saves leave out unused ones
aionggdaf1a0ccb387
diff --git a/crates/doc/src/assets.rs b/crates/doc/src/assets.rsnew file mode 100644index 000000000..73f7ac98d--- /dev/null+++ b/crates/doc/src/assets.rs@@ -0,0 +1,342 @@+//! Placed files (assets): one asset per distinct file, the assets a document uses, and the+//! compaction a save applies (duplicates merged, unused assets left out).+//!+//! An asset holds what is shared by every placement of a file (bytes, link, type, the PDF page+//! shown); each placement's frame data (transform, crop, fitting) lives on its [`Graphic`].++use std::collections::{BTreeMap, BTreeSet, HashMap};+use std::sync::Arc;++use crate::story::Story;+use crate::{Asset, AssetId, Content, Document, Item};++/// Nesting limit for groups, anchored objects, table cells and notes. A document nested deeper+/// is treated as using every asset, so nothing it references is merged away or pruned.+const MAX_DEPTH: usize = 64;++impl Asset {+ /// Whether `other` holds the same file: the same type, page and bytes. Assets without bytes+ /// (a link that couldn't be read) are the same only when they link to the same path.+ pub fn same_file(&self, other: &Asset) -> bool {+ if self.mime != other.mime || self.page != other.page {+ return false;+ }+ match (self.data.is_empty(), other.data.is_empty()) {+ (false, false) => Arc::ptr_eq(&self.data, &other.data) || (self.data.len() == other.data.len() && *self.data == *other.data),+ (true, true) => self.link.is_some() && self.link == other.link,+ _ => false,+ }+ }+}++/// The asset in `assets` holding the same file as `asset`.+pub fn find_same_asset(assets: &BTreeMap<AssetId, Arc<Asset>>, asset: &Asset) -> Option<AssetId> {+ assets.values().find(|a| a.same_file(asset)).map(|a| a.id)+}++/// Add `asset` to `assets` unless they already hold the same file; returns the id placements+/// should reference. A reused asset without a link takes the new asset's link.+pub fn insert_asset(assets: &mut BTreeMap<AssetId, Arc<Asset>>, asset: Asset) -> AssetId {+ if let Some(id) = find_same_asset(assets, &asset) {+ if let Some(a) = assets.get_mut(&id)+ && a.link.is_none()+ && asset.link.is_some()+ {+ Arc::make_mut(a).link = asset.link;+ }+ return id;+ }+ let id = asset.id;+ assets.insert(id, Arc::new(asset));+ id+}++/// Calls `f` with every asset reference in `it` and its children; false when nesting exceeded+/// [`MAX_DEPTH`] (some references weren't visited).+fn item_refs(it: &Item, f: &mut impl FnMut(AssetId), depth: usize) -> bool {+ if depth > MAX_DEPTH {+ return false;+ }+ if let Content::Graphic(g) = &it.content {+ f(g.asset);+ }+ if let Some(p) = it.media.as_ref().and_then(|m| m.poster) {+ f(p);+ }+ it.children().iter().all(|c| item_refs(c, f, depth + 1))+}++/// Asset references in a story's anchored objects, graphic cells and notes (recursively).+fn story_refs(st: &Story, f: &mut impl FnMut(AssetId), depth: usize) -> bool {+ if depth > MAX_DEPTH {+ return false;+ }+ let mut complete = st.objects.iter().all(|o| item_refs(&o.item, f, depth + 1));+ for t in st.tables.values() {+ for c in &t.cells {+ if let Some(g) = &c.graphic {+ f(g.asset);+ }+ complete &= story_refs(&c.text, f, depth + 1);+ }+ }+ for n in st.notes.iter().chain(&st.endnotes) {+ complete &= story_refs(&n.text, f, depth + 1);+ }+ complete+}++/// Every asset reference in a story (its anchored objects, graphic cells and notes).+pub fn story_asset_refs(st: &Story) -> Vec<AssetId> {+ let mut out = Vec::new();+ story_refs(st, &mut |a| out.push(a), 0);+ out+}++/// Point a story's references to a key of `map` at its value.+pub fn remap_story_assets(st: &mut Story, map: &HashMap<AssetId, AssetId>) {+ if !map.is_empty() {+ remap_story(st, map, 0);+ }+}++fn item_uses(it: &Item, map: &HashMap<AssetId, AssetId>) -> bool {+ let mut hit = false;+ item_refs(it, &mut |a| hit |= map.contains_key(&a), 0);+ hit+}++fn story_uses(st: &Story, map: &HashMap<AssetId, AssetId>) -> bool {+ let mut hit = false;+ story_refs(st, &mut |a| hit |= map.contains_key(&a), 0);+ hit+}++fn remap_id(id: &mut AssetId, map: &HashMap<AssetId, AssetId>) {+ if let Some(n) = map.get(id) {+ *id = *n;+ }+}++/// Rewrites the references in `it`, copying only the shared items that change.+fn remap_item(it: &mut Arc<Item>, map: &HashMap<AssetId, AssetId>, depth: usize) {+ if depth > MAX_DEPTH || !item_uses(it, map) {+ return;+ }+ remap_item_in(Arc::make_mut(it), map, depth);+}++fn remap_item_in(it: &mut Item, map: &HashMap<AssetId, AssetId>, depth: usize) {+ if let Content::Graphic(g) = &mut it.content {+ remap_id(&mut g.asset, map);+ }+ if let Some(p) = it.media.as_mut().and_then(|m| m.poster.as_mut()) {+ remap_id(p, map);+ }+ if let Some(kids) = it.children_mut() {+ for k in kids {+ remap_item(k, map, depth + 1);+ }+ }+}++fn remap_story(st: &mut Story, map: &HashMap<AssetId, AssetId>, depth: usize) {+ if depth > MAX_DEPTH {+ return;+ }+ for o in &mut st.objects {+ if item_uses(&o.item, map) {+ remap_item_in(&mut Arc::make_mut(o).item, map, depth + 1);+ }+ }+ for t in st.tables.values_mut() {+ if !t.cells.iter().any(|c| c.graphic.as_ref().is_some_and(|g| map.contains_key(&g.asset)) || story_uses(&c.text, map)) {+ continue;+ }+ for c in &mut Arc::make_mut(t).cells {+ if let Some(g) = &mut c.graphic {+ remap_id(&mut g.asset, map);+ }+ remap_story(&mut c.text, map, depth + 1);+ }+ }+ for n in st.notes.iter_mut().chain(st.endnotes.iter_mut()) {+ if story_uses(&n.text, map) {+ remap_story(&mut Arc::make_mut(n).text, map, depth + 1);+ }+ }+}++impl Document {+ /// Add a placed file, reusing the asset of an identical file already in the document;+ /// returns the id to reference.+ pub fn add_asset(&mut self, asset: Asset) -> AssetId {+ insert_asset(&mut self.assets, asset)+ }++ /// Copy asset `id` of `src` into this document (or reuse the identical asset it already+ /// has); returns the id to reference here, `None` when `src` has no such asset.+ pub fn adopt_asset(&mut self, src: &Document, id: AssetId) -> Option<AssetId> {+ let a = src.assets.get(&id)?;+ if let Some(same) = find_same_asset(&self.assets, a) {+ return Some(same);+ }+ let fresh = AssetId(self.alloc());+ Some(self.add_asset(Asset { id: fresh, ..(**a).clone() }))+ }++ /// Calls `f` with every asset reference: graphics on document and parent spreads (in+ /// groups and multi-state objects too), media posters, anchored and inline objects, graphic+ /// table cells and notes. False when nesting exceeded the depth limit.+ fn asset_refs(&self, f: &mut impl FnMut(AssetId)) -> bool {+ let mut complete = true;+ for sp in self.spreads.iter().chain(&self.parents) {+ for it in &sp.items {+ complete &= item_refs(it, f, 0);+ }+ }+ for st in self.stories.values() {+ complete &= story_refs(st, f, 0);+ }+ complete+ }++ /// The assets something in the document references. When the document nests deeper than+ /// the traversal limit, every asset counts as used.+ pub fn used_assets(&self) -> BTreeSet<AssetId> {+ let mut used = BTreeSet::new();+ if !self.asset_refs(&mut |a| {+ used.insert(a);+ }) {+ return self.assets.keys().copied().collect();+ }+ used+ }++ /// Point every reference to a key of `map` at its value.+ pub fn remap_assets(&mut self, map: &HashMap<AssetId, AssetId>) {+ if map.is_empty() {+ return;+ }+ for sp in self.spreads.iter_mut().chain(self.parents.iter_mut()) {+ if sp.items.iter().any(|it| item_uses(it, map)) {+ for it in &mut Arc::make_mut(sp).items {+ remap_item(it, map, 0);+ }+ }+ }+ for st in self.stories.values_mut() {+ if story_uses(st, map) {+ remap_story(Arc::make_mut(st), map, 0);+ }+ }+ }++ /// Merge assets holding the same file into the first of them (lowest id), rewriting the+ /// references; returns how many assets were removed.+ pub fn merge_duplicate_assets(&mut self) -> usize {+ if !self.asset_refs(&mut |_| {}) {+ return 0;+ }+ let mut reps: HashMap<(usize, &str, u32), Vec<&Arc<Asset>>> = HashMap::new();+ let mut map = HashMap::new();+ let mut links = Vec::new();+ for a in self.assets.values() {+ let bucket = reps.entry((a.data.len(), a.mime.as_str(), a.page)).or_default();+ match bucket.iter().find(|r| r.same_file(a)) {+ Some(r) => {+ map.insert(a.id, r.id);+ if r.link.is_none()+ && let Some(l) = &a.link+ {+ links.push((r.id, l.clone()));+ }+ }+ None => bucket.push(a),+ }+ }+ for (id, link) in links {+ if let Some(a) = self.assets.get_mut(&id)+ && a.link.is_none()+ {+ Arc::make_mut(a).link = Some(link);+ }+ }+ self.remap_assets(&map);+ self.assets.retain(|id, _| !map.contains_key(id));+ map.len()+ }++ /// Merge duplicate assets and drop the ones nothing references (what a saved file holds).+ pub fn compact_assets(&mut self) {+ self.merge_duplicate_assets();+ let used = self.used_assets();+ self.assets.retain(|id, _| used.contains(id));+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::{Graphic, ItemId, LayerId, Shape, StoryId};+ use designcraft_geom::{Affine, Rect, shapes};++ fn asset(id: u64, data: &[u8]) -> Asset {+ Asset { id: AssetId(id), name: "p.png".into(), mime: "image/png".into(), link: None, data: Arc::new(data.to_vec()), pixels: None, page: 0 }+ }++ fn graphic_item(id: u64, asset: u64, crop: f64) -> Item {+ let mut it = Item::new(ItemId(id), LayerId(1), Shape::Rectangle, shapes::rectangle(Rect::new(0.0, 0.0, 10.0, 10.0)));+ it.content = Content::Graphic(Graphic {+ asset: AssetId(asset),+ size: (10.0, 10.0),+ xf: Affine::translate((crop, 0.0)),+ auto_fit: Default::default(),+ fit_align: 4,+ crop: [crop; 4],+ });+ it+ }++ #[test]+ fn identical_files_share_one_asset() {+ let mut d = Document::new(&Default::default());+ let a = d.add_asset(asset(1, b"same"));+ let b = d.add_asset(Asset { link: Some("/x/p.png".into()), ..asset(2, b"same") });+ let c = d.add_asset(asset(3, b"diff"));+ let p = d.add_asset(Asset { page: 1, ..asset(4, b"same") });+ assert_eq!(a, b);+ assert_ne!(a, c);+ assert_ne!(a, p);+ assert_eq!(d.assets.len(), 3);+ assert_eq!(d.assets[&a].link.as_deref(), Some("/x/p.png"));+ // Unreadable links match by path only.+ let m1 = d.add_asset(Asset { link: Some("/m/1.png".into()), ..asset(5, b"") });+ let m2 = d.add_asset(Asset { link: Some("/m/2.png".into()), ..asset(6, b"") });+ let m3 = d.add_asset(asset(7, b""));+ let m4 = d.add_asset(asset(8, b""));+ assert_ne!(m1, m2);+ assert_ne!(m3, m4);+ }++ #[test]+ fn compaction_merges_duplicates_and_drops_unused_assets() {+ let mut d = Document::new(&Default::default());+ for (id, data) in [(1, b"one"), (2, b"one"), (3, b"two"), (4, b"old")] {+ d.assets.insert(AssetId(id), Arc::new(asset(id, data)));+ }+ let sp = Arc::make_mut(d.spreads.first_mut().unwrap());+ sp.items.push(Arc::new(graphic_item(10, 1, 1.0)));+ sp.items.push(Arc::new(graphic_item(11, 2, 2.0)));+ // A graphic cell and an anchored object in a story.+ let mut st = Story::new(StoryId(20));+ st.objects.push(Arc::new(crate::AnchoredObject::new(graphic_item(12, 3, 3.0), Default::default())));+ d.stories.insert(StoryId(20), Arc::new(st));+ assert_eq!(d.used_assets(), [1, 2, 3].map(AssetId).into_iter().collect());++ d.compact_assets();+ assert_eq!(d.assets.keys().copied().collect::<Vec<_>>(), vec![AssetId(1), AssetId(3)]);+ let crops: Vec<(AssetId, f64)> = d.spreads[0].items.iter().filter_map(|i| i.graphic().map(|g| (g.asset, g.crop[0]))).collect();+ assert_eq!(crops, vec![(AssetId(1), 1.0), (AssetId(1), 2.0)]);+ }+}diff --git a/crates/doc/src/lib.rs b/crates/doc/src/lib.rsindex c4b69e86d..2c90d8b95 100644--- a/crates/doc/src/lib.rs+++ b/crates/doc/src/lib.rs@@ -12,6 +12,7 @@pub mod anchored;pub mod arabic;pub mod arrow;+pub mod assets;pub mod attrs;pub mod build;pub mod cjk;@@ -36,6 +37,7 @@ use std::collections::BTreeMap;use std::sync::Arc;pub use anchored::{AnchorPosition, AnchoredObject, OBJECT_MARK};+pub use assets::{find_same_asset, insert_asset, remap_story_assets, story_asset_refs};pub use attrs::*;pub use datamerge::{DataField, DataFieldKind, DataMerge, DataSource, Delimiter, Fingerprint, MergeOptions, Placeholder, PlaceholderAnchor, PlaceholderRole,diff --git a/crates/engine/src/cmd/datamerge/fill.rs b/crates/engine/src/cmd/datamerge/fill.rsindex deb0948bc..89dd0c74e 100644--- a/crates/engine/src/cmd/datamerge/fill.rs+++ b/crates/engine/src/cmd/datamerge/fill.rs@@ -75,6 +75,11 @@ fn renumber(dst: &mut designcraft_doc::Document, src: &designcraft_doc::Documentlet new_id = ItemId(dst.alloc());maps.items.insert(old, new_id);it.id = new_id;+ if let Some(m) = it.media.as_mut()+ && let Some(p) = m.poster+ {+ m.poster = dst.adopt_asset(src, p).or(Some(p));+ }match &mut it.content {Content::Text(tf) => {let new_sid = if let Some(n) = maps.stories.get(&tf.story) {@@ -83,6 +88,15 @@ fn renumber(dst: &mut designcraft_doc::Document, src: &designcraft_doc::Documentlet n = StoryId(dst.alloc());let mut st = src.story(tf.story).cloned().unwrap_or_else(|| Story::new(n));st.id = n;+ let mut assets = HashMap::new();+ for a in designcraft_doc::story_asset_refs(&st) {+ if let Some(na) = dst.adopt_asset(src, a)+ && na != a+ {+ assets.insert(a, na);+ }+ }+ designcraft_doc::remap_story_assets(&mut st, &assets);st.frames.clear();dst.stories.insert(n, Arc::new(st));maps.stories.insert(tf.story, n);@@ -94,8 +108,8 @@ fn renumber(dst: &mut designcraft_doc::Document, src: &designcraft_doc::Document}}Content::Graphic(g) => {- if let Some(a) = src.assets.get(&g.asset) {- dst.assets.entry(g.asset).or_insert_with(|| a.clone());+ if let Some(a) = dst.adopt_asset(src, g.asset) {+ g.asset = a;}}Content::Group { items } => {@@ -355,21 +369,18 @@ fn place_image(};let (nw, nh) = (f64::from(pw), f64::from(ph));let mode = fitting_of(&options.fitting);- let aid = AssetId(doc.alloc());+ let fresh = AssetId(doc.alloc());let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_else(|| field.to_string());let link = options.link_images.then(|| path.to_string_lossy().into_owned());- doc.assets.insert(- aid,- Arc::new(Asset {- id: aid,- name,- mime: designcraft_render::image_mime(&bytes).to_string(),- link,- data: Arc::new(bytes),- pixels: Some((pw, ph)),- page: 0,- }),- );+ let aid = doc.add_asset(Asset {+ id: fresh,+ name,+ mime: designcraft_render::image_mime(&bytes).to_string(),+ link,+ data: Arc::new(bytes),+ pixels: Some((pw, ph)),+ page: 0,+ });let mut g = Graphic { asset: aid, size: (nw, nh), xf: Affine::translate((inner.x0, inner.y0)), auto_fit: mode, fit_align: 0, crop: [0.0; 4] };if let Some(xf) = g.fitted(inner, mode) {g.xf = xf;diff --git a/crates/engine/src/cmd/file.rs b/crates/engine/src/cmd/file.rsindex f871a43e8..e7bc7e4ec 100644--- a/crates/engine/src/cmd/file.rs+++ b/crates/engine/src/cmd/file.rs@@ -336,9 +336,9 @@ fn file_place(s: &mut Session, p: &Value) -> Result<Value> {let py = p.get("y").and_then(Value::as_f64);let want_w = p.get("width").and_then(Value::as_f64);s.edit(|d, sel| {- let aid = AssetId(d.alloc());+ let fresh = AssetId(d.alloc());let mime = designcraft_render::image_mime(&bytes).to_string();- d.assets.insert(aid, Arc::new(Asset { page: pdf_page, id: aid, name, mime, link, data: Arc::new(bytes), pixels: Some((pw, ph)) }));+ let aid = d.add_asset(Asset { page: pdf_page, id: fresh, name, mime, link, data: Arc::new(bytes), pixels: Some((pw, ph)) });// The width the whole page (or image) gets; the height follows the proportions.let w = match want_w {Some(w) => w,@@ -441,13 +441,9 @@ fn place_load(s: &mut Session, p: &Value) -> Result<Value> {let (bytes, name, link) = read_source(p)?;let (pw, ph) = designcraft_render::image_size(&bytes).ok_or_else(|| bad("place.load", "unsupported or corrupt image"))?;let aid = s.edit(|d, _| {- let aid = AssetId(d.alloc());+ let fresh = AssetId(d.alloc());let mime = designcraft_render::image_mime(&bytes).to_string();- d.assets.insert(- aid,- Arc::new(Asset { page: 0, id: aid, name: name.clone(), mime, link: link.clone(), data: Arc::new(bytes), pixels: Some((pw, ph)) }),- );- Ok(aid)+ Ok(d.add_asset(Asset { page: 0, id: fresh, name: name.clone(), mime, link: link.clone(), data: Arc::new(bytes), pixels: Some((pw, ph)) }))})?;s.loaded = Some((aid, (pw as f64, ph as f64)));s.set_tool("placeGun");diff --git a/crates/engine/src/cmd/links.rs b/crates/engine/src/cmd/links.rsindex c05c6dad9..687cd7b1a 100644--- a/crates/engine/src/cmd/links.rs+++ b/crates/engine/src/cmd/links.rs@@ -112,8 +112,10 @@ pub fn status(a: &designcraft_doc::Asset) -> &'static str {}fn list(d: &Document) -> Vec<Value> {+ let used = d.used_assets();d.assets.values()+ .filter(|a| used.contains(&a.id)).map(|a| {let uses: Vec<Value> = uses(d, a.id).into_iter()@@ -311,3 +313,86 @@ mod relink_folder_tests {let _ = std::fs::remove_dir_all(&dir);}}++#[cfg(test)]+mod shared_asset_tests {+ use super::*;++ fn png(w: u32, h: u32) -> Vec<u8> {+ designcraft_render::Rendered { width: w, height: h, pixels: vec![200u8; (w * h * 4) as usize] }.to_png()+ }++ #[test]+ fn placements_of_one_file_share_an_asset_and_saves_drop_replaced_ones() {+ let mut s = Session::new();+ s.execute("file.new", &json!({})).unwrap();+ let a = super::super::file::base64_encode(&png(10, 10));+ s.execute("file.place", &json!({"base64": a, "name": "a.png", "x": 72, "y": 72})).unwrap();+ s.execute("edit.deselectAll", &json!({})).unwrap();+ s.execute("file.place", &json!({"base64": a, "name": "a.png", "x": 200, "y": 72})).unwrap();+ let l = s.execute("links.list", &json!({})).unwrap();+ assert_eq!(l.as_array().unwrap().len(), 1);+ assert_eq!(l[0]["uses"].as_array().unwrap().len(), 2);+ assert_eq!(s.doc().unwrap().doc.assets.len(), 1);++ // A new image into a frame: the old asset stays in memory (undo) but not in the file.+ let f = s.execute("frame.create", &json!({"rect": [100, 300, 200, 400]})).unwrap()["id"].as_u64().unwrap();+ let b = super::super::file::base64_encode(&png(20, 10));+ let c = super::super::file::base64_encode(&png(30, 10));+ s.execute("file.place", &json!({"base64": b, "name": "b.png", "frame": f})).unwrap();+ s.execute("file.place", &json!({"base64": c, "name": "c.png", "frame": f})).unwrap();+ let d = &s.doc().unwrap().doc;+ assert_eq!(d.assets.len(), 3);+ assert_eq!(s.execute("links.list", &json!({})).unwrap().as_array().unwrap().len(), 2);+ let back = designcraft_format::load(&designcraft_format::save(&s.doc().unwrap().doc).unwrap()).unwrap();+ let mut names: Vec<&str> = back.assets.values().map(|a| a.name.as_str()).collect();+ names.sort();+ assert_eq!(names, ["a.png", "c.png"]);+ }++ #[test]+ fn pasted_graphics_reuse_the_identical_asset_of_the_target() {+ use designcraft_doc::build::NewDocument;+ use designcraft_doc::{Asset, Graphic, Item, ItemId, Shape, SpreadRef};+ let asset = |id: u64, data: Vec<u8>| Asset {+ id: AssetId(id),+ name: "p.png".into(),+ mime: "image/png".into(),+ link: None,+ data: Arc::new(data),+ pixels: None,+ page: 0,+ };+ let mut src = Document::new(&NewDocument::default());+ let mut dst = Document::new(&NewDocument::default());+ let (pa, pb) = (png(4, 4), png(8, 4));+ // The source's asset id is taken in the target by a different file.+ let sid = AssetId(src.alloc());+ src.assets.insert(sid, Arc::new(asset(sid.0, pa.clone())));+ dst.assets.insert(sid, Arc::new(asset(sid.0, pb)));+ dst.alloc();+ let same = AssetId(dst.alloc());+ assert_ne!(same, sid);+ dst.assets.insert(same, Arc::new(asset(same.0, pa)));+ let iid = ItemId(src.alloc());+ let mut it = Item::new(+ iid,+ src.default_layer(),+ Shape::Rectangle,+ designcraft_geom::shapes::rectangle(designcraft_geom::Rect::new(0.0, 0.0, 9.0, 9.0)),+ );+ it.content = Content::Graphic(Graphic {+ asset: sid,+ size: (4.0, 4.0),+ xf: designcraft_geom::Affine::IDENTITY,+ auto_fit: Default::default(),+ fit_align: 4,+ crop: [1.0; 4],+ });+ src.insert_item(SpreadRef::Doc(0), it, None).unwrap();+ let ids = super::super::object::duplicate_from(&mut dst, &src, &[iid], SpreadRef::Doc(0), designcraft_geom::Vec2::ZERO).unwrap();+ let g = dst.item(ids[0]).and_then(|i| i.graphic()).unwrap();+ assert_eq!((g.asset, g.crop), (same, [1.0; 4]));+ assert_eq!(dst.assets.len(), 2);+ }+}diff --git a/crates/engine/src/cmd/mathexpr.rs b/crates/engine/src/cmd/mathexpr.rsindex 791a0d704..ec90d52e3 100644--- a/crates/engine/src/cmd/mathexpr.rs+++ b/crates/engine/src/cmd/mathexpr.rs@@ -13,11 +13,16 @@ use crate::{Result, Session};fn math_item(d: &mut designcraft_doc::Document, latex: &str, size: f64, lid: designcraft_doc::LayerId, at: (f64, f64)) -> Item {let (svg, (w, h)) = crate::math::to_svg(latex, size);- let aid = AssetId(d.alloc());- d.assets.insert(- aid,- Arc::new(Asset { page: 0, id: aid, name: "math.svg".into(), mime: "image/svg+xml".into(), link: None, data: Arc::new(svg), pixels: None }),- );+ let fresh = AssetId(d.alloc());+ let aid = d.add_asset(Asset {+ page: 0,+ id: fresh,+ name: "math.svg".into(),+ mime: "image/svg+xml".into(),+ link: None,+ data: Arc::new(svg),+ pixels: None,+ });let id = ItemId(d.alloc());let r = Rect::new(at.0, at.1, at.0 + w, at.1 + h);let mut it = Item::new(id, lid, Shape::Rectangle, shapes::rectangle(r));diff --git a/crates/engine/src/cmd/media.rs b/crates/engine/src/cmd/media.rsindex b80089585..b6e4cd4b8 100644--- a/crates/engine/src/cmd/media.rs+++ b/crates/engine/src/cmd/media.rs@@ -25,8 +25,8 @@ pub(crate) fn place_media(s: &mut Session, p: &Value, name: String, mime: &str,let (px, py) = (p.get("x").and_then(Value::as_f64), p.get("y").and_then(Value::as_f64));let mime = mime.to_string();s.edit(|d, sel| {- let aid = AssetId(d.alloc());- d.assets.insert(aid, Arc::new(Asset { page: 0, id: aid, name, mime, link, data: Arc::new(bytes), pixels: None }));+ let fresh = AssetId(d.alloc());+ let aid = d.add_asset(Asset { page: 0, id: fresh, name, mime, link, data: Arc::new(bytes), pixels: None });let id = match target {Some(fid) => {let it = d.item_mut(fid).ok_or(designcraft_doc::DocError::NoItem(fid))?;@@ -90,10 +90,9 @@ pub fn specs() -> Vec<CommandSpec> {s.edit(|d, _| {let poster_id = match poster {Some(Some((bytes, name, link, px))) => {- let aid = AssetId(d.alloc());+ let fresh = AssetId(d.alloc());let mime = designcraft_render::image_mime(&bytes).to_string();- d.assets.insert(aid, Arc::new(Asset { page: 0, id: aid, name, mime, link, data: Arc::new(bytes), pixels: Some(px) }));- Some(Some(aid))+ Some(Some(d.add_asset(Asset { page: 0, id: fresh, name, mime, link, data: Arc::new(bytes), pixels: Some(px) })))}Some(None) => Some(None),None => None,diff --git a/crates/engine/src/cmd/object.rs b/crates/engine/src/cmd/object.rsindex 17c44c27a..466c15914 100644--- a/crates/engine/src/cmd/object.rs+++ b/crates/engine/src/cmd/object.rs@@ -900,6 +900,11 @@ pub fn duplicate_from(dst: &mut Document, src: &Document, ids: &[ItemId], to: Spfn renumber(dst: &mut Document, src: &Document, it: &mut Item, stories: &mut HashMap<StoryId, StoryId>) {it.id = ItemId(dst.alloc());let my_id = it.id;+ if let Some(m) = it.media.as_mut()+ && let Some(p) = m.poster+ {+ m.poster = dst.adopt_asset(src, p).or(Some(p));+ }match &mut it.content {Content::Text(tf) => {let new_sid = match stories.get(&tf.story) {@@ -908,6 +913,15 @@ fn renumber(dst: &mut Document, src: &Document, it: &mut Item, stories: &mut Haslet n = StoryId(dst.alloc());let mut st = src.story(tf.story).cloned().unwrap_or_else(|| Story::new(n));st.id = n;+ let mut assets = HashMap::new();+ for a in designcraft_doc::story_asset_refs(&st) {+ if let Some(na) = dst.adopt_asset(src, a)+ && na != a+ {+ assets.insert(a, na);+ }+ }+ designcraft_doc::remap_story_assets(&mut st, &assets);st.frames.clear();dst.stories.insert(n, Arc::new(st));stories.insert(tf.story, n);@@ -920,10 +934,8 @@ fn renumber(dst: &mut Document, src: &Document, it: &mut Item, stories: &mut Has}}Content::Graphic(g) => {- if let Some(a) = src.assets.get(&g.asset)- && !dst.assets.contains_key(&g.asset)- {- dst.assets.insert(g.asset, a.clone());+ if let Some(a) = dst.adopt_asset(src, g.asset) {+ g.asset = a;}}Content::Group { items } => {diff --git a/crates/engine/src/cmd/package.rs b/crates/engine/src/cmd/package.rsindex 468667510..25f09537c 100644--- a/crates/engine/src/cmd/package.rs+++ b/crates/engine/src/cmd/package.rs@@ -285,13 +285,13 @@ fn write_links(d: &Document, root: &OutputRoot, dir: &Path, only: Option<&[Assetlet mut out = d.clone();let mut taken = Vec::new();let mut written = Vec::new();- let mut ids: Vec<AssetId> = d.assets.keys().copied().collect();- ids.sort_by_key(|i| i.0);+ // Only files something places; the saved document leaves the others out.+ let ids = d.used_assets();for id in ids {if only.is_some_and(|o| !o.contains(&id)) {continue;}- let a = &d.assets[&id];+ let Some(a) = d.assets.get(&id) else { continue };let name = link_name(a.link.as_deref().unwrap_or(&a.name), id, &mut taken);let path = dir.join(&name);root.write(&path, a.data.as_slice())?;diff --git a/crates/engine/src/cmd/table.rs b/crates/engine/src/cmd/table.rsindex 028971e97..fafece0df 100644--- a/crates/engine/src/cmd/table.rs+++ b/crates/engine/src/cmd/table.rs@@ -1402,20 +1402,10 @@ fn place_graphic(s: &mut Session, p: &Value) -> Result<Value> {let (pw, ph) = designcraft_render::image_size(&bytes).ok_or_else(|| bad(ID, "unsupported or corrupt image"))?;let fill = p.get("fit").and_then(Value::as_str) == Some("fill");s.edit(|d, sel| {- let aid = designcraft_doc::AssetId(d.alloc());+ let fresh = designcraft_doc::AssetId(d.alloc());let mime = designcraft_render::image_mime(&bytes).to_string();- d.assets.insert(- aid,- std::sync::Arc::new(designcraft_doc::Asset {- page: 0,- id: aid,- name,- mime,- link,- data: std::sync::Arc::new(bytes),- pixels: Some((pw, ph)),- }),- );+ let aid =+ d.add_asset(designcraft_doc::Asset { page: 0, id: fresh, name, mime, link, data: std::sync::Arc::new(bytes), pixels: Some((pw, ph)) });let st = d.story_mut(g.story).ok_or_else(|| bad(ID, "no story"))?;let t = st.table_mut(g.table).ok_or_else(|| bad(ID, "no table"))?;let (r, c) = t.owner(g.range.r0, g.range.c0);diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rsindex f80ad0a65..2de45402d 100644--- a/crates/format/src/lib.rs+++ b/crates/format/src/lib.rs@@ -2,14 +2,15 @@//!//! - `mimetype` (stored first, uncompressed): `application/vnd.designcraft+zip`//! - `document.json`: the [`Document`] (pretty JSON, documented by the serde model in `designcraft-doc`)-//! - `assets/<id>.<ext>`: embedded placed files, byte-for-byte+//! - `assets/<id>.<ext>`: embedded placed files, byte-for-byte: one per distinct file, and only+//! the ones the document references (see [`Document::compact_assets`])//! - `meta.json`: format version and generator//!//! Older single-file JSON documents (assets as base64 in `assetData`) still open.#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]#![forbid(unsafe_code)]-use std::collections::HashSet;+use std::collections::{HashMap, HashSet};use std::io::{Cursor, Read, Write};use std::sync::Arc;@@ -42,8 +43,12 @@ fn ext_for(mime: &str) -> &'static str {}}-/// Serialize a document to `.designcraft` bytes.+/// Serialize a document to `.designcraft` bytes. The file holds each distinct placed file once+/// and leaves out assets nothing references; `doc` itself keeps them (undo can bring them back).pub fn save(doc: &Document) -> Result<Vec<u8>, FormatError> {+ let mut compact = doc.clone();+ compact.compact_assets();+ let doc = &compact;let io = |e: zip::result::ZipError| FormatError::Io(e.to_string());let mut buf = Cursor::new(Vec::new());{@@ -100,13 +105,26 @@ fn load_zip(bytes: &[u8]) -> Result<Document, FormatError> {upgrade(&mut v);let mut doc: Document = serde_json::from_value(v).map_err(|e| FormatError::NotOurs(e.to_string()))?;let names: Vec<String> = z.file_names().map(str::to_string).collect();+ // Files saved with one copy per placement: identical copies share one buffer as they are+ // read, so memory holds each distinct file once.+ let mut loaded: HashMap<usize, Vec<Arc<Vec<u8>>>> = HashMap::new();for name in names {let Some(stem) = name.strip_prefix("assets/") else { continue };let Some(id) = stem.split('.').next().and_then(|s| s.parse::<u64>().ok()) else { continue };if let (Some(data), Some(a)) = (read(&mut z, &name), doc.assets.get_mut(&AssetId(id))) {- Arc::make_mut(a).data = Arc::new(data);+ let same = loaded.entry(data.len()).or_default();+ let data = match same.iter().find(|d| ***d == data) {+ Some(d) => d.clone(),+ None => {+ let d = Arc::new(data);+ same.push(d.clone());+ d+ }+ };+ Arc::make_mut(a).data = data;}}+ doc.merge_duplicate_assets();doc.check().map_err(|e| FormatError::NotOurs(e.to_string()))?;Ok(doc)}@@ -124,6 +142,7 @@ fn load_legacy_json(bytes: &[u8]) -> Result<Document, FormatError> {}}}+ doc.merge_duplicate_assets();doc.check().map_err(|e| FormatError::NotOurs(e.to_string()))?;Ok(doc)}@@ -205,24 +224,67 @@ mod tests {use designcraft_doc::geom::Rect;use designcraft_doc::{Asset, ParaFormat, SpreadRef};+ fn png_asset(d: &mut Document, data: &[u8]) -> AssetId {+ let aid = AssetId(d.alloc());+ let a = Asset {+ page: 0,+ id: aid,+ name: "x.png".into(),+ mime: "image/png".into(),+ link: None,+ data: Arc::new(data.to_vec()),+ pixels: Some((1, 1)),+ };+ d.assets.insert(aid, Arc::new(a));+ aid+ }++ fn place(d: &mut Document, asset: AssetId, crop: f64) -> designcraft_doc::ItemId {+ use designcraft_doc::geom::{Affine, shapes};+ use designcraft_doc::{Content, Graphic, Item, ItemId, Shape};+ let id = ItemId(d.alloc());+ let mut it = Item::new(id, d.default_layer(), Shape::Rectangle, shapes::rectangle(Rect::new(0.0, 0.0, 50.0, 50.0)));+ it.content = Content::Graphic(Graphic {+ asset,+ size: (1.0, 1.0),+ xf: Affine::translate((crop, crop)),+ auto_fit: Default::default(),+ fit_align: 4,+ crop: [crop; 4],+ });+ d.insert_item(SpreadRef::Doc(0), it, None).unwrap();+ id+ }++ fn asset_entries(bytes: &[u8]) -> usize {+ let z = zip::ZipArchive::new(Cursor::new(bytes)).unwrap();+ z.file_names().filter(|n| n.starts_with("assets/")).count()+ }++ /// Zip `d` the way files were saved with one asset per placement (nothing merged or pruned).+ fn save_uncompacted(d: &Document) -> Vec<u8> {+ let mut buf = Cursor::new(Vec::new());+ let mut z = zip::ZipWriter::new(&mut buf);+ let opts = zip::write::SimpleFileOptions::default();+ z.start_file("mimetype", opts).unwrap();+ z.write_all(MIME.as_bytes()).unwrap();+ z.start_file("document.json", opts).unwrap();+ z.write_all(&serde_json::to_vec(d).unwrap()).unwrap();+ for (id, a) in &d.assets {+ z.start_file(format!("assets/{}.png", id.0), opts).unwrap();+ z.write_all(&a.data).unwrap();+ }+ z.finish().unwrap();+ buf.into_inner()+ }+#[test]fn roundtrip_with_assets() {let mut d = Document::new(&NewDocument { pages: 3, ..Default::default() });let lid = d.default_layer();d.add_text_frame(SpreadRef::Doc(0), Rect::new(10.0, 10.0, 200.0, 200.0), lid, "héllo\nworld", ParaFormat::default()).unwrap();- let aid = AssetId(d.alloc());- d.assets.insert(- aid,- Arc::new(Asset {- page: 0,- id: aid,- name: "x.png".into(),- mime: "image/png".into(),- link: None,- data: Arc::new(vec![1, 2, 3, 4]),- pixels: Some((1, 1)),- }),- );+ let aid = png_asset(&mut d, &[1, 2, 3, 4]);+ place(&mut d, aid, 0.0);let bytes = save(&d).unwrap();assert!(bytes.starts_with(b"PK"));let back = load(&bytes).unwrap();@@ -231,6 +293,42 @@ mod tests {assert_eq!(back.stories, d.stories);}+ #[test]+ fn saves_hold_each_used_file_once() {+ let mut d = Document::new(&NewDocument::default());+ let first = png_asset(&mut d, &[7; 64]);+ let copy = png_asset(&mut d, &[7; 64]);+ let replaced = png_asset(&mut d, &[9; 64]);+ let a = place(&mut d, first, 1.0);+ let b = place(&mut d, copy, 2.0);+ assert_eq!(d.assets.len(), 3);++ // Duplicates merge into the first asset, the unreferenced one is left out, and every+ // placement keeps its own crop and transform.+ let bytes = save(&d).unwrap();+ assert_eq!(asset_entries(&bytes), 1);+ assert_eq!(d.assets.len(), 3, "the open document keeps its assets");+ let back = load(&bytes).unwrap();+ assert_eq!(back.assets.keys().copied().collect::<Vec<_>>(), vec![first]);+ assert!(!back.assets.contains_key(&replaced));+ for (id, crop) in [(a, 1.0), (b, 2.0)] {+ let g = back.item(id).and_then(|i| i.graphic()).unwrap();+ assert_eq!((g.asset, g.crop, g.xf), (first, [crop; 4], d.item(id).unwrap().graphic().unwrap().xf));+ }+ }++ #[test]+ fn older_files_with_one_asset_per_placement_open_with_one() {+ let mut d = Document::new(&NewDocument::default());+ let ids: Vec<AssetId> = (0..3).map(|_| png_asset(&mut d, &[5; 32])).collect();+ for (k, id) in ids.iter().enumerate() {+ place(&mut d, *id, k as f64);+ }+ let back = load(&save_uncompacted(&d)).unwrap();+ assert_eq!(back.assets.len(), 1);+ assert_eq!(asset_entries(&save(&back).unwrap()), 1);+ }+/// The object of the item `id` in a document's JSON.fn item_json(v: &mut Value, id: u64) -> Option<&mut Value> {if v.get("id").and_then(Value::as_u64) == Some(id) && v.get("content").is_some() {diff --git a/crates/idml/src/import.rs b/crates/idml/src/import.rsindex d85d9ee02..138775268 100644--- a/crates/idml/src/import.rs+++ b/crates/idml/src/import.rs@@ -132,6 +132,8 @@ struct Importer<'r> {page_index: HashMap<String, usize>,item_ids: HashMap<String, ItemId>,assets: BTreeMap<AssetId, Arc<Asset>>,+ /// Assets read from a link path, so later placements of that path reuse them unread.+ linked_assets: HashMap<String, AssetId>,ctx: ItemCtx,sections: Vec<Section>,footnote_options: designcraft_doc::FootnoteOptions,@@ -217,6 +219,7 @@ impl<'r> Importer<'r> {page_index: HashMap::new(),item_ids: HashMap::new(),assets: BTreeMap::new(),+ linked_assets: HashMap::new(),ctx: ItemCtx { threads: Vec::new() },sections: Vec::new(),footnote_options: Default::default(),@@ -2205,12 +2208,29 @@ impl<'r> Importer<'r> {let size = ((r - l).abs(), (b - t).abs());let link = g.find("Link");let uri = link.and_then(|k| k.get("LinkResourceURI")).map(uri_to_path);+ let graphic = |asset| {+ Content::Graphic(designcraft_doc::Graphic {+ asset,+ size,+ xf: gxf * Affine::translate((l, t)),+ auto_fit: Default::default(),+ fit_align: 4,+ crop: [0.0; 4],+ })+ };let mut data = g.prop_el("Contents").map(|c| base64_decode(&c.text_content())).unwrap_or_default();+ let mut read_from = None;if data.is_empty()&& let Some(p) = &uri- && let Some(d) = (self.read_link)(p){- data = d;+ // Every placement of a linked file shares one asset; read the file once.+ if let Some(id) = self.linked_assets.get(p) {+ return graphic(*id);+ }+ if let Some(d) = (self.read_link)(p) {+ data = d;+ read_from = Some(p.clone());+ }}let (mime, px) = sniff_image(&data);let mime = mime.map(str::to_string).unwrap_or_else(|| match link.and_then(|k| k.get("LinkResourceFormat")).unwrap_or("") {@@ -2226,17 +2246,14 @@ impl<'r> Importer<'r> {});let name =uri.as_deref().and_then(|p| p.rsplit(['/', '\\']).next()).filter(|n| !n.is_empty()).map(str::to_string).unwrap_or_else(|| "image".into());- let id = AssetId(self.alloc());+ let fresh = AssetId(self.alloc());let link_path = uri.filter(|p| p.contains('/') || p.contains('\\'));- self.assets.insert(id, Arc::new(Asset { page: 0, id, name, mime, link: link_path, data: Arc::new(data), pixels }));- Content::Graphic(designcraft_doc::Graphic {- asset: id,- size,- xf: gxf * Affine::translate((l, t)),- auto_fit: Default::default(),- fit_align: 4,- crop: [0.0; 4],- })+ let id =+ designcraft_doc::insert_asset(&mut self.assets, Asset { page: 0, id: fresh, name, mime, link: link_path, data: Arc::new(data), pixels });+ if let Some(p) = read_from {+ self.linked_assets.insert(p, id);+ }+ graphic(id)}fn link_threads(&mut self) {diff --git a/crates/idml/src/tests.rs b/crates/idml/src/tests.rsindex 7b747587a..4c7817009 100644--- a/crates/idml/src/tests.rs+++ b/crates/idml/src/tests.rs@@ -7,6 +7,7 @@ use designcraft_geom::{Rect, shapes};use super::*;+mod assets;mod decorations;fn zip_files(files: &[(&str, &str)]) -> Vec<u8> {diff --git a/crates/idml/src/tests/assets.rs b/crates/idml/src/tests/assets.rsnew file mode 100644index 000000000..a5cd90fb0--- /dev/null+++ b/crates/idml/src/tests/assets.rs@@ -0,0 +1,107 @@+//! Placed files: every placement of one file shares an asset.++use std::cell::Cell;++use designcraft_doc::{AssetId, Content};++use super::*;++const PHOTO: &str = "file:/photos/wedding%20day/couple.jpg";++/// A graphic frame at `x` showing `image` (an `<Image>` element) with `crop` on every side.+fn frame(n: usize, x: f64, crop: f64, image: &str) -> String {+ format!(+ r#"<Rectangle Self="r{n}" ItemLayer="L1" ItemTransform="1 0 0 1 {x} 0" ContentType="GraphicType">+ <Properties><PathGeometry><GeometryPathType PathOpen="false"><PathPointArray>+ <PathPointType Anchor="0 0"/><PathPointType Anchor="0 30"/><PathPointType Anchor="40 30"/><PathPointType Anchor="40 0"/>+ </PathPointArray></GeometryPathType></PathGeometry></Properties>+ <FrameFittingOption TopCrop="{crop}" LeftCrop="{crop}" BottomCrop="{crop}" RightCrop="{crop}"/>+ {image}+ </Rectangle>"#+ )+}++fn image(n: usize, uri: &str, contents: Option<&[u8]>) -> String {+ let contents = contents.map(|c| format!("<Contents><![CDATA[{}]]></Contents>", base64_encode(c))).unwrap_or_default();+ format!(+ r#"<Image Self="im{n}" ItemTransform="1 0 0 1 {n} 0">+ <Properties>{contents}<GraphicBounds Left="0" Top="0" Right="40" Bottom="30"/></Properties>+ <Link Self="lk{n}" LinkResourceURI="{uri}" LinkResourceFormat="$ID/JPEG" StoredState="Normal"/>+ </Image>"#+ )+}++fn package(frames: &[String]) -> Vec<u8> {+ let spread = format!(+ r#"<?xml version="1.0" encoding="UTF-8"?>+<idPkg:Spread xmlns:idPkg="http://ns.adobe.com/AdobeInDesign/idml/1.0/packaging" DOMVersion="16.0">+ <Spread Self="sp1" PageCount="1" ItemTransform="1 0 0 1 0 0">+ <Page Self="p1" AppliedMaster="m1" GeometricBounds="0 0 700 500" ItemTransform="1 0 0 1 0 0"/>+ {}+ </Spread>+</idPkg:Spread>"#,+ frames.join("\n")+ );+ zip_files(&[+ ("designmap.xml", DESIGNMAP),+ ("Resources/Graphic.xml", GRAPHIC),+ ("Resources/Styles.xml", STYLES),+ ("Resources/Preferences.xml", PREFS),+ ("MasterSpreads/MasterSpread_m1.xml", MASTER),+ ("Spreads/Spread_sp1.xml", &spread),+ ("Stories/Story_s1.xml", STORY),+ ])+}++/// (asset, crop, transform) of each graphic, left to right.+fn placements(d: &Document) -> Vec<(AssetId, [f64; 4], designcraft_geom::Affine)> {+ let mut out: Vec<(f64, AssetId, [f64; 4], designcraft_geom::Affine)> = d.spreads[0]+ .items+ .iter()+ .filter_map(|i| match &i.content {+ Content::Graphic(g) => Some((i.xf.translation().x, g.asset, g.crop, g.xf)),+ _ => None,+ })+ .collect();+ out.sort_by(|a, b| a.0.total_cmp(&b.0));+ out.into_iter().map(|(_, a, c, x)| (a, c, x)).collect()+}++#[test]+fn one_file_placed_several_times_imports_as_one_asset() {+ let bytes = vec![42u8; 300];+ let reads = Cell::new(0);+ let reader = |p: &str| {+ reads.set(reads.get() + 1);+ p.ends_with("couple.jpg").then(|| bytes.clone())+ };+ // Three placements linking the file, and one embedding the same bytes under another path.+ let frames = vec![+ frame(1, 0.0, 1.0, &image(1, PHOTO, None)),+ frame(2, 100.0, 2.0, &image(2, PHOTO, None)),+ frame(3, 200.0, 3.0, &image(3, PHOTO, None)),+ frame(4, 300.0, 4.0, &image(4, "file:/elsewhere/copy.jpg", Some(&bytes))),+ ];+ let d = import_idml_with(&package(&frames), &reader).unwrap();+ assert_eq!(reads.get(), 1, "the linked file is read once");+ assert_eq!(d.assets.len(), 1);+ let a = d.assets.values().next().unwrap();+ assert_eq!(a.link.as_deref(), Some("/photos/wedding day/couple.jpg"));+ let p = placements(&d);+ assert_eq!(p.iter().map(|(_, c, _)| c[0]).collect::<Vec<_>>(), vec![1.0, 2.0, 3.0, 4.0]);+ assert!(p.iter().all(|(id, _, _)| *id == a.id));++ // Exported linked or embedded, the round trip keeps one asset and each placement's crop and+ // transform.+ for embed_images in [false, true] {+ let out = export_idml_with(&d, &ExportOptions { embed_images });+ let back = import_idml_with(&out, &reader).unwrap();+ assert_eq!(back.assets.len(), 1, "embed_images: {embed_images}");+ let q = placements(&back);+ assert_eq!(q.len(), 4);+ for ((_, c0, x0), (_, c1, x1)) in p.iter().zip(&q) {+ assert_eq!(c0, c1);+ assert!(x0.as_coeffs().iter().zip(x1.as_coeffs()).all(|(a, b)| (a - b).abs() < 1e-6), "{x0:?} {x1:?}");+ }+ }+}