Skip to content

PdfCraft: An open-source PDF reader and editor written by scratch in pure Rust

perf(engine): export page images on worker threads

NetterSilver37bf4ee986ca
diff --git a/crates/automation/src/documents.rs b/crates/automation/src/documents.rs
index 26addfd80..3ce312f39 100644
--- a/crates/automation/src/documents.rs
+++ b/crates/automation/src/documents.rs
@@ -539,11 +539,23 @@ impl Automation {
};
let mut files = Vec::new();
let mut lowered = Vec::new();
- for p in pages {
- let img = ex.image(p, dpi, format).map_err(failed)?;
+ // A write error stops the export as it is, not as a page error.
+ let mut write_error = None;
+ let exported = ex.images(&pages, dpi, format, |p, img| {
let path = child(&folder, &format!("{stem}_page_{}.{}", p + 1, format.extension()));
- write_atomic(&path, &img)?;
+ write_atomic(&path, &img).map_err(|e| {
+ let message = e.to_string();
+ write_error = Some(e);
+ message
+ })?;
files.push(path.to_string_lossy().into_owned());
+ Ok(())
+ });
+ if let Some(e) = write_error {
+ return Err(e);
+ }
+ exported.map_err(failed)?;
+ for &p in &pages {
// A page too large for the renderer at `dpi` is drawn at the most it allows.
let used = ex.dpi_used(p, dpi);
if used < dpi.clamp(18.0, 1200.0) - 0.5 {
diff --git a/crates/engine/examples/export_bench.rs b/crates/engine/examples/export_bench.rs
new file mode 100644
index 000000000..41520d2ff
--- /dev/null
+++ b/crates/engine/examples/export_bench.rs
@@ -0,0 +1,61 @@
+//! Times Export ▸ Image over every page of a PDF, for before and after comparisons.
+//!
+//! Usage: `export_bench FILE [DPI] [png|jpeg|tiff] [RUNS]` (defaults: 150 dpi, PNG, 7 runs). Each
+//! run exports every page through [`Exporter::images`] from a fresh exporter, as Export ▸ Image
+//! does, and prints its wall time; the summary gives the median and the minimum, the total size
+//! and an FNV-1a hash of every file in page order. Equal hashes mean equal output. Build with
+//! `--release`; timings are diagnostic, never a CI deadline.
+
+use std::error::Error;
+use std::sync::Arc;
+use std::time::Instant;
+
+use pdfcraft_engine::Session;
+use pdfcraft_engine::export::{Exporter, ImageFormat};
+
+const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
+const FNV_PRIME: u64 = 0x0000_0100_0000_01b3;
+
+fn main() -> Result<(), Box<dyn Error>> {
+ let mut args = std::env::args().skip(1);
+ let path = args.next().ok_or("usage: export_bench FILE [DPI] [png|jpeg|tiff] [RUNS]")?;
+ let dpi: f64 = args.next().map_or(Ok(150.0), |s| s.parse())?;
+ let format = match args.next().as_deref() {
+ None | Some("png") => ImageFormat::Png,
+ Some("jpeg" | "jpg") => ImageFormat::Jpeg { quality: 85 },
+ Some("tiff" | "tif") => ImageFormat::Tiff,
+ Some(other) => return Err(format!("unknown format {other:?}").into()),
+ };
+ let runs: usize = args.next().map_or(Ok(7), |s| s.parse())?;
+ let mut session = Session::new();
+ let id = session.open("bench.pdf", None, Arc::new(std::fs::read(&path)?), None)?;
+ let src = session.get(id).ok_or("the document did not open")?.export_source();
+ let pages: Vec<usize> = (0..src.pages).collect();
+ let mut times = Vec::new();
+ let mut summary = None;
+ for run in 0..runs {
+ let start = Instant::now();
+ let mut exporter = Exporter::from_source(src.clone());
+ let (mut hash, mut bytes) = (FNV_OFFSET, 0usize);
+ exporter.images(&pages, dpi, format, |_, file| {
+ bytes += file.len();
+ for b in file {
+ hash = (hash ^ u64::from(b)).wrapping_mul(FNV_PRIME);
+ }
+ Ok(())
+ })?;
+ let ms = start.elapsed().as_secs_f64() * 1000.0;
+ println!("run {}: {ms:.1} ms", run + 1);
+ times.push(ms);
+ summary = Some((bytes, hash));
+ }
+ times.sort_by(f64::total_cmp);
+ if let (Some(median), Some(min), Some((bytes, hash))) = (times.get(times.len() / 2), times.first(), summary) {
+ println!(
+ "{} pages, {} at {dpi} dpi: median {median:.1} ms, min {min:.1} ms ({runs} runs), {bytes} bytes, hash {hash:016x}",
+ pages.len(),
+ format.label()
+ );
+ }
+ Ok(())
+}
diff --git a/crates/engine/src/export.rs b/crates/engine/src/export.rs
index 01159df80..1330faf27 100644
--- a/crates/engine/src/export.rs
+++ b/crates/engine/src/export.rs
@@ -4,13 +4,52 @@
//! text, from a
//! document's working file (so unsaved edits and hidden layers are respected, as on screen).
+use std::collections::BTreeMap;
+use std::panic::{AssertUnwindSafe, catch_unwind};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::{Condvar, Mutex, PoisonError, mpsc};
+
use pdfcraft_render::{PageRenderer, RenderRequest, RequestKind};
use crate::Document;
+/// At most this many pages are drawn at once by [`Exporter::images`].
+const MAX_IMAGE_WORKERS: usize = 8;
+/// Bytes of rasters [`Exporter::images`] may hold at once: 1 GiB, about 16 Letter pages at 300 dpi.
+/// Each worker holds one page (the raster, its RGB copy and the encoded file), so large pages run
+/// on fewer workers; a page over the cap still exports, on one worker, as it did before.
+const IMAGE_MEMORY: f64 = 1024.0 * 1024.0 * 1024.0;
+
+/// Page `page` drawn at `dpi` and encoded as `format`, saying it was drawn at `used` dpi.
+fn draw_image(renderer: &mut PageRenderer, page: usize, dpi: f64, format: ImageFormat, used: f64) -> Result<Vec<u8>, String> {
+ let r = renderer.render(RenderRequest { page, scale: (dpi.clamp(18.0, 1200.0) / 72.0) as f32, ..Default::default() });
+ if let Some(e) = r.error {
+ return Err(format!("page {}: {e}", page + 1));
+ }
+ encode_image(r.width, r.height, &r.rgba, format, used)
+}
+
+/// Tells the workers of [`Exporter::images`] to stop when dropped.
+struct StopWorkers<'a> {
+ stop: &'a AtomicBool,
+ handed_over: &'a (Mutex<usize>, Condvar),
+}
+
+impl Drop for StopWorkers<'_> {
+ fn drop(&mut self) {
+ self.stop.store(true, Ordering::Relaxed);
+ // Taken under the lock, so a worker between its check and its wait still sees it.
+ drop(self.handed_over.0.lock().unwrap_or_else(PoisonError::into_inner));
+ self.handed_over.1.notify_all();
+ }
+}
+
/// Renders and extracts pages of one document state.
pub struct Exporter {
renderer: PageRenderer,
+ /// What the renderer draws, for the extra renderers [`Exporter::images`] starts.
+ bytes: std::sync::Arc<Vec<u8>>,
+ config: pdfcraft_render::RenderConfig,
pages: usize,
/// Each page's displayed size in points, for [`Exporter::dpi_used`].
sizes: Vec<(f32, f32)>,
@@ -65,7 +104,13 @@ impl Exporter {
/// resolution they allow, as the interactive Export does; [`Exporter::dpi_used`] says at what.
/// A source whose `config.reject_oversize` is set refuses such pages instead.
pub fn from_source(src: ExportSource) -> Self {
- Self { renderer: PageRenderer::new(src.bytes, src.config), pages: src.pages, sizes: src.sizes }
+ Self {
+ renderer: PageRenderer::new(src.bytes.clone(), src.config.clone()),
+ bytes: src.bytes,
+ config: src.config,
+ pages: src.pages,
+ sizes: src.sizes,
+ }
}
/// The resolution page `page` is actually exported at when asked for `dpi`: lower than asked
@@ -96,11 +141,146 @@ impl Exporter {
/// Page `page` as an image file of `format`, which says the resolution it was drawn at.
pub fn image(&mut self, page: usize, dpi: f64, format: ImageFormat) -> Result<Vec<u8>, String> {
self.check(page)?;
- let r = self.renderer.render(RenderRequest { page, scale: (dpi.clamp(18.0, 1200.0) / 72.0) as f32, ..Default::default() });
- if let Some(e) = r.error {
- return Err(format!("page {}: {e}", page + 1));
+ let used = self.dpi_used(page, dpi);
+ draw_image(&mut self.renderer, page, dpi, format, used)
+ }
+
+ /// Pages `pages` as image files of `format`, handed to `sink` in order with their page
+ /// number: the same files as [`Exporter::image`] for each page in turn. Encoding a page costs
+ /// several times what drawing it does, so several pages are drawn and encoded at once on
+ /// worker threads, each with its own renderer, while `sink` runs on the calling thread.
+ /// Workers draw at most one page each ahead of `sink`, and fewer run when the pages are
+ /// large, so memory stays bounded. Stops at the first page that fails or the first error `sink`
+ /// returns, and returns it; every page before it has been handed to `sink`.
+ pub fn images(
+ &mut self,
+ pages: &[usize],
+ dpi: f64,
+ format: ImageFormat,
+ sink: impl FnMut(usize, Vec<u8>) -> Result<(), String>,
+ ) -> Result<(), String> {
+ let workers = self.image_workers(pages, dpi);
+ self.images_on(workers, pages, dpi, format, sink)
+ }
+
+ /// [`Exporter::images`] with `workers` pages drawn at once (one: in turn, on this thread).
+ pub(crate) fn images_on(
+ &mut self,
+ workers: usize,
+ pages: &[usize],
+ dpi: f64,
+ format: ImageFormat,
+ mut sink: impl FnMut(usize, Vec<u8>) -> Result<(), String>,
+ ) -> Result<(), String> {
+ if workers <= 1 {
+ for &p in pages {
+ let img = self.image(p, dpi, format)?;
+ sink(p, img)?;
+ }
+ return Ok(());
+ }
+ let used: Vec<f64> = pages.iter().map(|p| self.dpi_used(*p, dpi)).collect();
+ let count = self.pages;
+ let (bytes, config) = (&self.bytes, &self.config);
+ let next = AtomicUsize::new(0);
+ let stop = AtomicBool::new(false);
+ // How many pages `sink` has taken; a worker waits while its page is `workers` past it.
+ let handed_over = (Mutex::new(0usize), Condvar::new());
+ let (tx, rx) = mpsc::channel::<(usize, Result<Vec<u8>, String>)>();
+ let mut first = Some(&mut self.renderer);
+ std::thread::scope(|s| {
+ let handles: Vec<_> = (0..workers)
+ .map(|_| {
+ let tx = tx.clone();
+ let (first, used, next, stop, handed_over) = (first.take(), &used, &next, &stop, &handed_over);
+ s.spawn(move || {
+ let mut own = None;
+ let renderer = match first {
+ Some(r) => r,
+ None => own.insert(PageRenderer::new(bytes.clone(), config.clone())),
+ };
+ loop {
+ let i = next.fetch_add(1, Ordering::Relaxed);
+ let (Some(&page), Some(&used)) = (pages.get(i), used.get(i)) else { break };
+ let mut taken = handed_over.0.lock().unwrap_or_else(PoisonError::into_inner);
+ while i >= taken.saturating_add(workers) && !stop.load(Ordering::Relaxed) {
+ taken = handed_over.1.wait(taken).unwrap_or_else(PoisonError::into_inner);
+ }
+ drop(taken);
+ if stop.load(Ordering::Relaxed) {
+ break;
+ }
+ let img = if page < count {
+ catch_unwind(AssertUnwindSafe(|| draw_image(renderer, page, dpi, format, used)))
+ .unwrap_or_else(|_| Err(format!("page {}: the export crashed", page + 1)))
+ } else {
+ Err(format!("page {} does not exist", page + 1))
+ };
+ if tx.send((i, img)).is_err() {
+ break;
+ }
+ }
+ })
+ })
+ .collect();
+ drop(tx);
+ // Stops the workers however this ends, a panicking `sink` included: waiting ones
+ // would otherwise keep the scope from returning.
+ let stopper = StopWorkers { stop: &stop, handed_over: &handed_over };
+ let mut ready = BTreeMap::new();
+ let mut result = Ok(());
+ let mut taken = 0;
+ while let Some(&page) = pages.get(taken) {
+ let Some(img) = ready.remove(&taken) else {
+ match rx.recv() {
+ Ok((i, img)) => {
+ ready.insert(i, img);
+ }
+ Err(_) => {
+ result = Err(format!("page {}: the export stopped unexpectedly", page + 1));
+ break;
+ }
+ }
+ continue;
+ };
+ if let Err(e) = img.and_then(|img| sink(page, img)) {
+ result = Err(e);
+ break;
+ }
+ taken += 1;
+ *handed_over.0.lock().unwrap_or_else(PoisonError::into_inner) = taken;
+ handed_over.1.notify_all();
+ }
+ drop(stopper);
+ // Joined here, so a worker that panicked outside a page is not re-raised by the scope.
+ for h in handles {
+ let _ = h.join();
+ }
+ result
+ })
+ }
+
+ /// How many pages [`Exporter::images`] draws at once: one per core but one, up to
+ /// [`MAX_IMAGE_WORKERS`], as many as [`IMAGE_MEMORY`] holds of the largest page, and one
+ /// when there is nothing to share or no threads (wasm).
+ fn image_workers(&self, pages: &[usize], dpi: f64) -> usize {
+ if cfg!(target_arch = "wasm32") || pages.len() < 2 {
+ return 1;
}
- encode_image(r.width, r.height, &r.rgba, format, self.dpi_used(page, dpi))
+ // One core is left for the interface, which keeps drawing while an export runs.
+ let cores = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get).saturating_sub(1).min(MAX_IMAGE_WORKERS);
+ // The raster (RGBA), the RGB copy the encoders take, and room for the encoded file.
+ let largest = pages
+ .iter()
+ .filter_map(|p| {
+ let (w, h) = *self.sizes.get(*p)?;
+ let k = self.dpi_used(*p, dpi) / 72.0;
+ Some((f64::from(w) * k).ceil() * (f64::from(h) * k).ceil() * 8.0)
+ })
+ .fold(0.0, f64::max);
+ // A float-to-integer cast saturates (and NaN is 0).
+ let by_memory = if largest > 0.0 { (IMAGE_MEMORY / largest) as usize } else { cores };
+ cores.min(by_memory).min(pages.len()).max(1)
}
/// The reading-order text of a page.
diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs
index c622af21b..e7e68a758 100644
--- a/crates/engine/src/tests.rs
+++ b/crates/engine/src/tests.rs
@@ -3410,6 +3410,70 @@ fn pages_export_as_png_jpeg_and_tiff() {
assert_eq!((compression, predictor), (5, 1), "LZW, without a predictor");
}
+/// Export ▸ Image draws several pages at once: each file is the one exporting that page alone
+/// writes, handed over in the order asked for, whatever the number of pages drawn at once.
+#[test]
+fn pages_exported_at_once_are_the_files_each_page_exports_alone_in_order() {
+ use crate::export::{Exporter, ImageFormat};
+ let (s, id) = session_with(12);
+ let pages = [3, 0, 11, 5, 5, 7, 1, 2, 10, 9, 8, 6, 4];
+ for format in [ImageFormat::Png, ImageFormat::Jpeg { quality: 85 }, ImageFormat::Tiff] {
+ let alone: Vec<(usize, Vec<u8>)> = pages.iter().map(|&p| (p, Exporter::new(s.get(id).unwrap()).image(p, 96.0, format).unwrap())).collect();
+ for workers in [1, 2, 4, 8] {
+ let mut got = Vec::new();
+ let mut ex = Exporter::new(s.get(id).unwrap());
+ ex.images_on(workers, &pages, 96.0, format, |p, img| {
+ got.push((p, img));
+ Ok(())
+ })
+ .unwrap();
+ assert!(got == alone, "{} with {workers} workers", format.label());
+ }
+ let mut got = Vec::new();
+ Exporter::new(s.get(id).unwrap())
+ .images(&pages, 96.0, format, |p, img| {
+ got.push((p, img));
+ Ok(())
+ })
+ .unwrap();
+ assert!(got == alone, "{}", format.label());
+ }
+}
+
+/// A page that can't be exported, or a file that can't be written, stops the export there with
+/// its error: every page before it is handed over, none after it.
+#[test]
+fn exporting_pages_at_once_stops_at_the_first_error() {
+ use crate::export::{Exporter, ImageFormat};
+ let (s, id) = session_with(6);
+ for workers in [1, 4] {
+ let mut ex = Exporter::new(s.get(id).unwrap());
+ let mut got = Vec::new();
+ let result = ex.images_on(workers, &[0, 1, 99, 2, 3], 72.0, ImageFormat::Png, |p, _| {
+ got.push(p);
+ Ok(())
+ });
+ assert_eq!(result, Err("page 100 does not exist".to_string()), "{workers} workers");
+ assert_eq!(got, [0, 1], "{workers} workers");
+
+ let mut got = Vec::new();
+ let result = ex.images_on(workers, &[0, 1, 2, 3, 4, 5], 72.0, ImageFormat::Png, |p, _| {
+ got.push(p);
+ if p == 2 { Err("disk full".to_string()) } else { Ok(()) }
+ });
+ assert_eq!(result, Err("disk full".to_string()), "{workers} workers");
+ assert_eq!(got, [0, 1, 2], "{workers} workers");
+ // The exporter is still usable afterwards.
+ assert!(ex.image(5, 72.0, ImageFormat::Png).is_ok());
+ }
+ // A panicking sink stops the workers rather than leaving them waiting for it.
+ let mut ex = Exporter::new(s.get(id).unwrap());
+ let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+ ex.images_on(4, &[0, 1, 2, 3, 4, 5], 72.0, ImageFormat::Png, |p, _| if p == 1 { panic!("sink") } else { Ok(()) })
+ }));
+ assert!(panicked.is_err());
+}
+
/// A TIFF's resolution unit (2 = inch) and its horizontal and vertical resolution.
fn tiff_resolution(bytes: &[u8]) -> (u32, f64, f64) {
use tiff::{decoder::ifd::Value, tags::Tag};
diff --git a/crates/ui-egui/src/workflows/export_ui.rs b/crates/ui-egui/src/workflows/export_ui.rs
index fae075e30..bcda63c57 100644
--- a/crates/ui-egui/src/workflows/export_ui.rs
+++ b/crates/ui-egui/src/workflows/export_ui.rs
@@ -176,12 +176,18 @@ fn run(
// Handled above.
ExportKind::AllImages => String::new(),
ExportKind::Image => {
- for (k, p) in pages.iter().enumerate() {
- set(k, None);
- let result = ex.image(*p, dpi, format).and_then(|img| sink(&format!("{stem}_page_{}.{}", p + 1, format.extension()), img));
- if let Err(e) = result {
- return crate::i18n::fmt(tl!("Export stopped: {e}"), &[("e", &e.to_string())]);
- }
+ set(0, None);
+ // Pages are drawn several at once and handed over in order: the progress counts the
+ // pages written so far.
+ let mut done = 0;
+ let result = ex.images(&pages, dpi, format, |p, img| {
+ sink(&format!("{stem}_page_{}.{}", p + 1, format.extension()), img)?;
+ done += 1;
+ set(done, None);
+ Ok(())
+ });
+ if let Err(e) = result {
+ return crate::i18n::fmt(tl!("Export stopped: {e}"), &[("e", &e.to_string())]);
}
let mut msg = if total == 1 {
crate::i18n::fmt(tl!("Exported 1 image"), &[])