PdfCraft: An open-source PDF reader and editor written by scratch in pure Rust
perf(engine): export page images on worker threads
NetterSilver37bf4ee986ca
diff --git a/crates/automation/src/documents.rs b/crates/automation/src/documents.rsindex 26addfd80..3ce312f39 100644--- a/crates/automation/src/documents.rs+++ b/crates/automation/src/documents.rs@@ -539,11 +539,23 @@ impl Automation {};let mut files = Vec::new();let mut lowered = Vec::new();- for p in pages {- let img = ex.image(p, dpi, format).map_err(failed)?;+ // A write error stops the export as it is, not as a page error.+ let mut write_error = None;+ let exported = ex.images(&pages, dpi, format, |p, img| {let path = child(&folder, &format!("{stem}_page_{}.{}", p + 1, format.extension()));- write_atomic(&path, &img)?;+ write_atomic(&path, &img).map_err(|e| {+ let message = e.to_string();+ write_error = Some(e);+ message+ })?;files.push(path.to_string_lossy().into_owned());+ Ok(())+ });+ if let Some(e) = write_error {+ return Err(e);+ }+ exported.map_err(failed)?;+ for &p in &pages {// A page too large for the renderer at `dpi` is drawn at the most it allows.let used = ex.dpi_used(p, dpi);if used < dpi.clamp(18.0, 1200.0) - 0.5 {diff --git a/crates/engine/examples/export_bench.rs b/crates/engine/examples/export_bench.rsnew file mode 100644index 000000000..41520d2ff--- /dev/null+++ b/crates/engine/examples/export_bench.rs@@ -0,0 +1,61 @@+//! Times Export ▸ Image over every page of a PDF, for before and after comparisons.+//!+//! Usage: `export_bench FILE [DPI] [png|jpeg|tiff] [RUNS]` (defaults: 150 dpi, PNG, 7 runs). Each+//! run exports every page through [`Exporter::images`] from a fresh exporter, as Export ▸ Image+//! does, and prints its wall time; the summary gives the median and the minimum, the total size+//! and an FNV-1a hash of every file in page order. Equal hashes mean equal output. Build with+//! `--release`; timings are diagnostic, never a CI deadline.++use std::error::Error;+use std::sync::Arc;+use std::time::Instant;++use pdfcraft_engine::Session;+use pdfcraft_engine::export::{Exporter, ImageFormat};++const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;+const FNV_PRIME: u64 = 0x0000_0100_0000_01b3;++fn main() -> Result<(), Box<dyn Error>> {+ let mut args = std::env::args().skip(1);+ let path = args.next().ok_or("usage: export_bench FILE [DPI] [png|jpeg|tiff] [RUNS]")?;+ let dpi: f64 = args.next().map_or(Ok(150.0), |s| s.parse())?;+ let format = match args.next().as_deref() {+ None | Some("png") => ImageFormat::Png,+ Some("jpeg" | "jpg") => ImageFormat::Jpeg { quality: 85 },+ Some("tiff" | "tif") => ImageFormat::Tiff,+ Some(other) => return Err(format!("unknown format {other:?}").into()),+ };+ let runs: usize = args.next().map_or(Ok(7), |s| s.parse())?;+ let mut session = Session::new();+ let id = session.open("bench.pdf", None, Arc::new(std::fs::read(&path)?), None)?;+ let src = session.get(id).ok_or("the document did not open")?.export_source();+ let pages: Vec<usize> = (0..src.pages).collect();+ let mut times = Vec::new();+ let mut summary = None;+ for run in 0..runs {+ let start = Instant::now();+ let mut exporter = Exporter::from_source(src.clone());+ let (mut hash, mut bytes) = (FNV_OFFSET, 0usize);+ exporter.images(&pages, dpi, format, |_, file| {+ bytes += file.len();+ for b in file {+ hash = (hash ^ u64::from(b)).wrapping_mul(FNV_PRIME);+ }+ Ok(())+ })?;+ let ms = start.elapsed().as_secs_f64() * 1000.0;+ println!("run {}: {ms:.1} ms", run + 1);+ times.push(ms);+ summary = Some((bytes, hash));+ }+ times.sort_by(f64::total_cmp);+ if let (Some(median), Some(min), Some((bytes, hash))) = (times.get(times.len() / 2), times.first(), summary) {+ println!(+ "{} pages, {} at {dpi} dpi: median {median:.1} ms, min {min:.1} ms ({runs} runs), {bytes} bytes, hash {hash:016x}",+ pages.len(),+ format.label()+ );+ }+ Ok(())+}diff --git a/crates/engine/src/export.rs b/crates/engine/src/export.rsindex 01159df80..1330faf27 100644--- a/crates/engine/src/export.rs+++ b/crates/engine/src/export.rs@@ -4,13 +4,52 @@//! text, from a//! document's working file (so unsaved edits and hidden layers are respected, as on screen).+use std::collections::BTreeMap;+use std::panic::{AssertUnwindSafe, catch_unwind};+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};+use std::sync::{Condvar, Mutex, PoisonError, mpsc};+use pdfcraft_render::{PageRenderer, RenderRequest, RequestKind};use crate::Document;+/// At most this many pages are drawn at once by [`Exporter::images`].+const MAX_IMAGE_WORKERS: usize = 8;+/// Bytes of rasters [`Exporter::images`] may hold at once: 1 GiB, about 16 Letter pages at 300 dpi.+/// Each worker holds one page (the raster, its RGB copy and the encoded file), so large pages run+/// on fewer workers; a page over the cap still exports, on one worker, as it did before.+const IMAGE_MEMORY: f64 = 1024.0 * 1024.0 * 1024.0;++/// Page `page` drawn at `dpi` and encoded as `format`, saying it was drawn at `used` dpi.+fn draw_image(renderer: &mut PageRenderer, page: usize, dpi: f64, format: ImageFormat, used: f64) -> Result<Vec<u8>, String> {+ let r = renderer.render(RenderRequest { page, scale: (dpi.clamp(18.0, 1200.0) / 72.0) as f32, ..Default::default() });+ if let Some(e) = r.error {+ return Err(format!("page {}: {e}", page + 1));+ }+ encode_image(r.width, r.height, &r.rgba, format, used)+}++/// Tells the workers of [`Exporter::images`] to stop when dropped.+struct StopWorkers<'a> {+ stop: &'a AtomicBool,+ handed_over: &'a (Mutex<usize>, Condvar),+}++impl Drop for StopWorkers<'_> {+ fn drop(&mut self) {+ self.stop.store(true, Ordering::Relaxed);+ // Taken under the lock, so a worker between its check and its wait still sees it.+ drop(self.handed_over.0.lock().unwrap_or_else(PoisonError::into_inner));+ self.handed_over.1.notify_all();+ }+}+/// Renders and extracts pages of one document state.pub struct Exporter {renderer: PageRenderer,+ /// What the renderer draws, for the extra renderers [`Exporter::images`] starts.+ bytes: std::sync::Arc<Vec<u8>>,+ config: pdfcraft_render::RenderConfig,pages: usize,/// Each page's displayed size in points, for [`Exporter::dpi_used`].sizes: Vec<(f32, f32)>,@@ -65,7 +104,13 @@ impl Exporter {/// resolution they allow, as the interactive Export does; [`Exporter::dpi_used`] says at what./// A source whose `config.reject_oversize` is set refuses such pages instead.pub fn from_source(src: ExportSource) -> Self {- Self { renderer: PageRenderer::new(src.bytes, src.config), pages: src.pages, sizes: src.sizes }+ Self {+ renderer: PageRenderer::new(src.bytes.clone(), src.config.clone()),+ bytes: src.bytes,+ config: src.config,+ pages: src.pages,+ sizes: src.sizes,+ }}/// The resolution page `page` is actually exported at when asked for `dpi`: lower than asked@@ -96,11 +141,146 @@ impl Exporter {/// Page `page` as an image file of `format`, which says the resolution it was drawn at.pub fn image(&mut self, page: usize, dpi: f64, format: ImageFormat) -> Result<Vec<u8>, String> {self.check(page)?;- let r = self.renderer.render(RenderRequest { page, scale: (dpi.clamp(18.0, 1200.0) / 72.0) as f32, ..Default::default() });- if let Some(e) = r.error {- return Err(format!("page {}: {e}", page + 1));+ let used = self.dpi_used(page, dpi);+ draw_image(&mut self.renderer, page, dpi, format, used)+ }++ /// Pages `pages` as image files of `format`, handed to `sink` in order with their page+ /// number: the same files as [`Exporter::image`] for each page in turn. Encoding a page costs+ /// several times what drawing it does, so several pages are drawn and encoded at once on+ /// worker threads, each with its own renderer, while `sink` runs on the calling thread.+ /// Workers draw at most one page each ahead of `sink`, and fewer run when the pages are+ /// large, so memory stays bounded. Stops at the first page that fails or the first error `sink`+ /// returns, and returns it; every page before it has been handed to `sink`.+ pub fn images(+ &mut self,+ pages: &[usize],+ dpi: f64,+ format: ImageFormat,+ sink: impl FnMut(usize, Vec<u8>) -> Result<(), String>,+ ) -> Result<(), String> {+ let workers = self.image_workers(pages, dpi);+ self.images_on(workers, pages, dpi, format, sink)+ }++ /// [`Exporter::images`] with `workers` pages drawn at once (one: in turn, on this thread).+ pub(crate) fn images_on(+ &mut self,+ workers: usize,+ pages: &[usize],+ dpi: f64,+ format: ImageFormat,+ mut sink: impl FnMut(usize, Vec<u8>) -> Result<(), String>,+ ) -> Result<(), String> {+ if workers <= 1 {+ for &p in pages {+ let img = self.image(p, dpi, format)?;+ sink(p, img)?;+ }+ return Ok(());+ }+ let used: Vec<f64> = pages.iter().map(|p| self.dpi_used(*p, dpi)).collect();+ let count = self.pages;+ let (bytes, config) = (&self.bytes, &self.config);+ let next = AtomicUsize::new(0);+ let stop = AtomicBool::new(false);+ // How many pages `sink` has taken; a worker waits while its page is `workers` past it.+ let handed_over = (Mutex::new(0usize), Condvar::new());+ let (tx, rx) = mpsc::channel::<(usize, Result<Vec<u8>, String>)>();+ let mut first = Some(&mut self.renderer);+ std::thread::scope(|s| {+ let handles: Vec<_> = (0..workers)+ .map(|_| {+ let tx = tx.clone();+ let (first, used, next, stop, handed_over) = (first.take(), &used, &next, &stop, &handed_over);+ s.spawn(move || {+ let mut own = None;+ let renderer = match first {+ Some(r) => r,+ None => own.insert(PageRenderer::new(bytes.clone(), config.clone())),+ };+ loop {+ let i = next.fetch_add(1, Ordering::Relaxed);+ let (Some(&page), Some(&used)) = (pages.get(i), used.get(i)) else { break };+ let mut taken = handed_over.0.lock().unwrap_or_else(PoisonError::into_inner);+ while i >= taken.saturating_add(workers) && !stop.load(Ordering::Relaxed) {+ taken = handed_over.1.wait(taken).unwrap_or_else(PoisonError::into_inner);+ }+ drop(taken);+ if stop.load(Ordering::Relaxed) {+ break;+ }+ let img = if page < count {+ catch_unwind(AssertUnwindSafe(|| draw_image(renderer, page, dpi, format, used)))+ .unwrap_or_else(|_| Err(format!("page {}: the export crashed", page + 1)))+ } else {+ Err(format!("page {} does not exist", page + 1))+ };+ if tx.send((i, img)).is_err() {+ break;+ }+ }+ })+ })+ .collect();+ drop(tx);+ // Stops the workers however this ends, a panicking `sink` included: waiting ones+ // would otherwise keep the scope from returning.+ let stopper = StopWorkers { stop: &stop, handed_over: &handed_over };+ let mut ready = BTreeMap::new();+ let mut result = Ok(());+ let mut taken = 0;+ while let Some(&page) = pages.get(taken) {+ let Some(img) = ready.remove(&taken) else {+ match rx.recv() {+ Ok((i, img)) => {+ ready.insert(i, img);+ }+ Err(_) => {+ result = Err(format!("page {}: the export stopped unexpectedly", page + 1));+ break;+ }+ }+ continue;+ };+ if let Err(e) = img.and_then(|img| sink(page, img)) {+ result = Err(e);+ break;+ }+ taken += 1;+ *handed_over.0.lock().unwrap_or_else(PoisonError::into_inner) = taken;+ handed_over.1.notify_all();+ }+ drop(stopper);+ // Joined here, so a worker that panicked outside a page is not re-raised by the scope.+ for h in handles {+ let _ = h.join();+ }+ result+ })+ }++ /// How many pages [`Exporter::images`] draws at once: one per core but one, up to+ /// [`MAX_IMAGE_WORKERS`], as many as [`IMAGE_MEMORY`] holds of the largest page, and one+ /// when there is nothing to share or no threads (wasm).+ fn image_workers(&self, pages: &[usize], dpi: f64) -> usize {+ if cfg!(target_arch = "wasm32") || pages.len() < 2 {+ return 1;}- encode_image(r.width, r.height, &r.rgba, format, self.dpi_used(page, dpi))+ // One core is left for the interface, which keeps drawing while an export runs.+ let cores = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get).saturating_sub(1).min(MAX_IMAGE_WORKERS);+ // The raster (RGBA), the RGB copy the encoders take, and room for the encoded file.+ let largest = pages+ .iter()+ .filter_map(|p| {+ let (w, h) = *self.sizes.get(*p)?;+ let k = self.dpi_used(*p, dpi) / 72.0;+ Some((f64::from(w) * k).ceil() * (f64::from(h) * k).ceil() * 8.0)+ })+ .fold(0.0, f64::max);+ // A float-to-integer cast saturates (and NaN is 0).+ let by_memory = if largest > 0.0 { (IMAGE_MEMORY / largest) as usize } else { cores };+ cores.min(by_memory).min(pages.len()).max(1)}/// The reading-order text of a page.diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rsindex c622af21b..e7e68a758 100644--- a/crates/engine/src/tests.rs+++ b/crates/engine/src/tests.rs@@ -3410,6 +3410,70 @@ fn pages_export_as_png_jpeg_and_tiff() {assert_eq!((compression, predictor), (5, 1), "LZW, without a predictor");}+/// Export ▸ Image draws several pages at once: each file is the one exporting that page alone+/// writes, handed over in the order asked for, whatever the number of pages drawn at once.+#[test]+fn pages_exported_at_once_are_the_files_each_page_exports_alone_in_order() {+ use crate::export::{Exporter, ImageFormat};+ let (s, id) = session_with(12);+ let pages = [3, 0, 11, 5, 5, 7, 1, 2, 10, 9, 8, 6, 4];+ for format in [ImageFormat::Png, ImageFormat::Jpeg { quality: 85 }, ImageFormat::Tiff] {+ let alone: Vec<(usize, Vec<u8>)> = pages.iter().map(|&p| (p, Exporter::new(s.get(id).unwrap()).image(p, 96.0, format).unwrap())).collect();+ for workers in [1, 2, 4, 8] {+ let mut got = Vec::new();+ let mut ex = Exporter::new(s.get(id).unwrap());+ ex.images_on(workers, &pages, 96.0, format, |p, img| {+ got.push((p, img));+ Ok(())+ })+ .unwrap();+ assert!(got == alone, "{} with {workers} workers", format.label());+ }+ let mut got = Vec::new();+ Exporter::new(s.get(id).unwrap())+ .images(&pages, 96.0, format, |p, img| {+ got.push((p, img));+ Ok(())+ })+ .unwrap();+ assert!(got == alone, "{}", format.label());+ }+}++/// A page that can't be exported, or a file that can't be written, stops the export there with+/// its error: every page before it is handed over, none after it.+#[test]+fn exporting_pages_at_once_stops_at_the_first_error() {+ use crate::export::{Exporter, ImageFormat};+ let (s, id) = session_with(6);+ for workers in [1, 4] {+ let mut ex = Exporter::new(s.get(id).unwrap());+ let mut got = Vec::new();+ let result = ex.images_on(workers, &[0, 1, 99, 2, 3], 72.0, ImageFormat::Png, |p, _| {+ got.push(p);+ Ok(())+ });+ assert_eq!(result, Err("page 100 does not exist".to_string()), "{workers} workers");+ assert_eq!(got, [0, 1], "{workers} workers");++ let mut got = Vec::new();+ let result = ex.images_on(workers, &[0, 1, 2, 3, 4, 5], 72.0, ImageFormat::Png, |p, _| {+ got.push(p);+ if p == 2 { Err("disk full".to_string()) } else { Ok(()) }+ });+ assert_eq!(result, Err("disk full".to_string()), "{workers} workers");+ assert_eq!(got, [0, 1, 2], "{workers} workers");+ // The exporter is still usable afterwards.+ assert!(ex.image(5, 72.0, ImageFormat::Png).is_ok());+ }+ // A panicking sink stops the workers rather than leaving them waiting for it.+ let mut ex = Exporter::new(s.get(id).unwrap());+ let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {+ ex.images_on(4, &[0, 1, 2, 3, 4, 5], 72.0, ImageFormat::Png, |p, _| if p == 1 { panic!("sink") } else { Ok(()) })+ }));+ assert!(panicked.is_err());+}+/// A TIFF's resolution unit (2 = inch) and its horizontal and vertical resolution.fn tiff_resolution(bytes: &[u8]) -> (u32, f64, f64) {use tiff::{decoder::ifd::Value, tags::Tag};diff --git a/crates/ui-egui/src/workflows/export_ui.rs b/crates/ui-egui/src/workflows/export_ui.rsindex fae075e30..bcda63c57 100644--- a/crates/ui-egui/src/workflows/export_ui.rs+++ b/crates/ui-egui/src/workflows/export_ui.rs@@ -176,12 +176,18 @@ fn run(// Handled above.ExportKind::AllImages => String::new(),ExportKind::Image => {- for (k, p) in pages.iter().enumerate() {- set(k, None);- let result = ex.image(*p, dpi, format).and_then(|img| sink(&format!("{stem}_page_{}.{}", p + 1, format.extension()), img));- if let Err(e) = result {- return crate::i18n::fmt(tl!("Export stopped: {e}"), &[("e", &e.to_string())]);- }+ set(0, None);+ // Pages are drawn several at once and handed over in order: the progress counts the+ // pages written so far.+ let mut done = 0;+ let result = ex.images(&pages, dpi, format, |p, img| {+ sink(&format!("{stem}_page_{}.{}", p + 1, format.extension()), img)?;+ done += 1;+ set(done, None);+ Ok(())+ });+ if let Err(e) = result {+ return crate::i18n::fmt(tl!("Export stopped: {e}"), &[("e", &e.to_string())]);}let mut msg = if total == 1 {crate::i18n::fmt(tl!("Exported 1 image"), &[])