PdfCraft: An open-source PDF reader and editor written by scratch in pure Rust
M8: a cleared black-and-white scan is stored as Group 4, or JBIG2 again, when smaller
Grzegorz Martens7f9554f29b70
diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.tomlindex 8d1515d8a..fe21d13b3 100644--- a/crates/redact/Cargo.toml+++ b/crates/redact/Cargo.toml@@ -17,8 +17,9 @@ pdfcraft-forms = { workspace = true }pdfcraft-edit = { workspace = true }pdfcraft-annot = { workspace = true }pdfcraft-xfa = { workspace = true }+# CCITT Group 4 and JBIG2 encoding for cleared one-bit images.+pdfcraft-filters = { workspace = true }thiserror = { workspace = true }[dev-dependencies]-pdfcraft-filters = { workspace = true }pdfcraft-testkit = { workspace = true }diff --git a/crates/redact/README.md b/crates/redact/README.mdindex af43e958d..e52c1d8b9 100644--- a/crates/redact/README.md+++ b/crates/redact/README.md@@ -21,7 +21,8 @@ clear_marks(&mut doc, None)?; // remove marks without app(8/16-bit and 1/2/4-bit images, image masks; Flate/LZW/RL/A85/AHx, and CCITT fax or JBIG2 asthe last filter, decoded through `Document::decoded_image_within`, a JBIG2 image with its`/JBIG2Globals`; `/DecodeParms` given by reference, numbers inside included, count). The copy- is Flate, and refers to no globals: when no other image uses them, they leave the file with the+ is Flate, or for a one-bit image of one component (a scan as a rule) CCITT Group 4, or JBIG2+ when it was JBIG2, whichever is smallest as stored, coded from the cleared samples; it refers to no globals: when no other image uses them, they leave the file with theoriginal on the full rewrite; when one does, they stay as they are, and may hold the shapes ofmarks under the area (not where they were). Codecs redaction doesn't decode (DCT, JPX) areremoved whole (fail-closed), and so is a JBIG2 image whose page isn't its size (the renderer@@ -109,5 +110,4 @@ Patterns are heuristic: dates, protocol identifiers and longer number runs are rCustom code sets aren't supported yet.Not yet: pattern locales, struct-tree/alt-text cleanup, metadata and hidden-information-sanitising (M8.4), re-encoding of DCT images (they are removed instead), and storing a cleared black-and-white-image as CCITT Group 4 or JBIG2 again (it is Flate, larger; the optimizer makes it smaller).+sanitising (M8.4), re-encoding of DCT images (they are removed instead).diff --git a/crates/redact/src/image.rs b/crates/redact/src/image.rsindex 6c7a6bf93..e9606c5e3 100644--- a/crates/redact/src/image.rs+++ b/crates/redact/src/image.rs@@ -1,13 +1,14 @@//! Clearing the pixels of an image under redaction regions. The image is decoded, every pixel//! whose footprint intersects a region is set to zero (no ink for image masks), and the result is-//! written as a new Flate image; the original object is untouched (other pages may use it).+//! written as a new image, Flate, or for a one-bit image of one component Group 4, or JBIG2 when it+//! was JBIG2, whichever is smallest; the original object is untouched (other pages may use it).//! CCITT fax and JBIG2 images decode through `Document::decoded_image_within`, but for Group 3 on//! bytes, which PdfCraft's renderer may draw as other rows, and a JBIG2 page of other rows than the//! image's. Images whose codec PdfCraft can't decode (DCT, JPX), and those, return `None`, and the//! caller removes the whole image instead; that is fail-closed.use pdfcraft_content::{Matrix, overlaps};-use pdfcraft_cos::{Document, Object, Stream};+use pdfcraft_cos::{Dict, Document, Object, Stream};/// Most pixels an image may have to be cleared in place (larger ones are removed).const MAX_PIXELS: u64 = 64 << 20;@@ -165,5 +166,39 @@ fn decode_and_clear(doc: &Document, s: &Stream, ctm: &Matrix, rects: &[[f64; 4]]dict.remove(b"Length");dict.remove(b"DecodeParms");dict.remove(b"Alternates");- Some(Some(Stream::flate(dict, &data)))+ Some(Some(smallest(dict, &data, w, h, ncomp == 1 && bpc == 1, is_jbig2)))+}++/// The cleared samples `data` of a `w` × `h` image as a stream with `dict`: Flate, or, for one bit+/// of one component (`bilevel`, a scan as a rule, which Flate stores several times larger), CCITT+/// Group 4, or JBIG2 when the image was JBIG2 (`jbig2`; JBIG2 is PDF 1.4, which such a file already+/// is), whichever is smallest as stored, dictionary included (a tiny image can lose by Group 4's+/// parameters). Both decode to `data`, cleared before it is coded, so nothing of the original's+/// code is carried over; Group 4 or JBIG2 that can't code the rows (too wide) is left out.+fn smallest(dict: Dict, data: &[u8], w: usize, h: usize, bilevel: bool, jbig2: bool) -> Stream {+ let flate = Stream::flate(dict.clone(), data);+ let Some((columns, rows)) = u32::try_from(w).ok().zip(u32::try_from(h).ok()).filter(|_| bilevel) else { return flate };+ let g4 = pdfcraft_filters::encode_g4(data, columns, rows).ok().map(|coded| {+ let mut dict = dict.clone();+ let mut parms = Dict::new();+ parms.set(b"K".to_vec(), Object::Int(-1));+ parms.set(b"Columns".to_vec(), Object::Int(i64::from(columns)));+ parms.set(b"Rows".to_vec(), Object::Int(i64::from(rows)));+ dict.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode"));+ dict.set(b"DecodeParms".to_vec(), Object::Dict(parms));+ Stream::from_raw(dict, coded)+ });+ let jbig = jbig2.then(|| pdfcraft_filters::encode_jbig2(data, columns, rows).ok()).flatten().map(|coded| {+ let mut dict = dict.clone();+ dict.set(b"Filter".to_vec(), Object::name("JBIG2Decode"));+ Stream::from_raw(dict, coded)+ });+ [g4, jbig].into_iter().flatten().fold(flate, |best, s| if stored_len(&s) < stored_len(&best) { s } else { best })+}++/// The bytes stream `s` takes in a file: its data and its dictionary as written.+fn stored_len(s: &Stream) -> usize {+ let mut dict = Vec::new();+ pdfcraft_cos::serialize(&Object::Dict(s.dict.clone()), &mut dict);+ dict.len().saturating_add(s.raw.len())}diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rsindex 08faef69a..2e76bb0ba 100644--- a/crates/redact/src/tests.rs+++ b/crates/redact/src/tests.rs@@ -519,13 +519,19 @@ fn two_pages_one_resource_dictionary(inherited: bool, again: bool) -> Document {])}-/// The samples of the image a page's (possibly inherited) resources name `name`.+/// The samples of the image a page's (possibly inherited) resources name `name`, decoded through+/// its image codec too (`Document::decoded_image`).fn named_image(doc: &Document, page: usize, name: &[u8]) -> Option<Vec<u8>> {+ named_stream(doc, page, name).and_then(|s| doc.decoded_image(&s).ok())+}++/// The image a page's (possibly inherited) resources name `name`.+fn named_stream(doc: &Document, page: usize, name: &[u8]) -> Option<Stream> {let p = pdfcraft_model::pages(doc).swap_remove(page);let res = doc.resolve(p.dict.get(b"Resources")?).as_dict().cloned()?;let xo = doc.resolve(res.get(b"XObject")?).as_dict().cloned()?;match &*doc.get(xo.get(name)?.as_ref()?) {- Object::Stream(s) => s.decoded().ok(),+ Object::Stream(s) => Some(s.clone()),_ => None,}}@@ -833,7 +839,8 @@ fn a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole() {/// A scan stored as JBIG2 symbols, with globals, keeps every pixel outside the area once it is/// redacted and saved, and the pixels under it are cleared; neither its JBIG2 code nor its globals,-/// which no other image uses, stay in the file.+/// which no other image uses, stay in the file. The cleared copy is coded afresh as JBIG2, smaller+/// than Flate, and refers to no globals.#[test]fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() {const LETTERS: [[&str; 7]; 3] = [@@ -854,6 +861,7 @@ fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() {}}let coded = pdfcraft_filters::encode_jbig2(&bits, w as u32, h as u32).unwrap();+ let original = coded.clone();assert!(coded.len() < pdfcraft_filters::encode_jbig2_generic(&bits, w as u32, h as u32).unwrap().len(), "symbol coded");let dict = format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /JBIG2Decode /DecodeParms << /JBIG2Globals 8 0 R >>"@@ -877,10 +885,108 @@ fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() {let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap();assert_eq!(named_image(&saved, 0, b"PCRedacted1"), Some(cleared));assert_no_secret(&doc, b"LEAK_GLOBALS");+ let copy = named_stream(&saved, 0, b"PCRedacted1").unwrap();+ assert_eq!(copy.dict.name(b"Filter"), Some(&b"JBIG2Decode"[..]));+ assert!(!copy.dict.contains(b"DecodeParms"), "the copy refers to no globals");+ let flate = pdfcraft_filters::encode_flate(&named_image(&saved, 0, b"PCRedacted1").unwrap());+ assert!(copy.raw.len() < flate.len(), "JBIG2 {} bytes, Flate {}", copy.raw.len(), flate.len());for (r, object) in saved.scan_objects() {if let Object::Stream(s) = &*object {- assert_ne!(s.dict.name(b"Filter"), Some(&b"JBIG2Decode"[..]), "the JBIG2 image stays as {r:?}");+ assert_ne!(&s.raw[..], &original[..], "the original JBIG2 code stays as {r:?}");+ }+ }+}++/// A cleared one-bit image of one component, a scan as a rule, is stored as CCITT Group 4 when+/// that is smaller than Flate, whatever it was stored as (Flate here), and never as JBIG2 unless it+/// was JBIG2; an eight-bit image stays Flate. The copy decodes to the cleared samples.+#[test]+fn a_cleared_black_and_white_scan_is_stored_as_group_4() {+ // 800 × 200: lines of word-sized dark blocks 14 rows tall, their edges ragged by a sample from+ // row to row, as a scan's are; Group 4 codes such rows from the row above.+ let (w, h) = (800usize, 200usize);+ let stride = w / 8;+ let mut seed = 0x9E37_79B9_7F4A_7C15u64;+ let mut next = move || {+ seed ^= seed << 13;+ seed ^= seed >> 7;+ seed ^= seed << 17;+ seed+ };+ let mut bits = vec![0xFFu8; stride * h];+ for top in (3..h - 14).step_by(20) {+ let mut x = 4;+ while x + 70 < w {+ let len = 20 + (next() % 40) as usize;+ for y in top..top + 14 {+ let (left, right) = (x + (next() % 2) as usize, x + len - (next() % 2) as usize);+ for xx in left..right {+ bits[y * stride + xx / 8] &= !(0x80 >> (xx % 8));+ }+ }+ x += len + 6 + (next() % 6) as usize;+ }+ }+ let g4 = pdfcraft_filters::encode_g4(&bits, w as u32, h as u32).unwrap();+ assert!(g4.len() < pdfcraft_filters::encode_flate(&bits).len(), "Group 4 is smaller than Flate here");+ let grey: Vec<u8> = (0..w * h).map(|i| (i % 251) as u8).collect();+ for (dict, data, filter) in [+ (+ format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /FlateDecode"),+ pdfcraft_filters::encode_flate(&bits),+ &b"CCITTFaxDecode"[..],+ ),+ (+ format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 8 /ColorSpace /DeviceGray /Filter /FlateDecode"),+ pdfcraft_filters::encode_flate(&grey),+ &b"FlateDecode"[..],+ ),+ ] {+ let mut doc = one_page(b"q 200 0 0 50 0 100 cm /Im Do Q", "/XObject << /Im 7 0 R >>", vec![stream(&dict, data)]);+ // The left quarter of the top half.+ mark(&mut doc, 0, &[[0.0, 125.0, 50.0, 150.0]], "");+ assert_eq!(apply(&mut doc, None).unwrap().images_cleared, 1, "{dict}");+ let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap();+ let copy = named_stream(&saved, 0, b"PCRedacted1").unwrap();+ assert_eq!(copy.dict.name(b"Filter"), Some(filter), "{dict}");+ // Covered: rows 0 to 99 (y 150 down to 125, a quarter point a row), samples 0 to 199.+ let mut expected = if filter == b"FlateDecode" { grey.clone() } else { bits.clone() };+ for y in 0..100 {+ for x in 0..200 {+ if filter == b"FlateDecode" {+ expected[y * w + x] = 0;+ } else {+ expected[y * stride + x / 8] &= !(0x80 >> (x % 8));+ }+ }}+ assert_eq!(named_image(&saved, 0, b"PCRedacted1").unwrap(), expected, "{dict}");+ }+}++/// A cleared one-bit image is stored as the smallest stream, its dictionary counted: a tiny one,+/// whose Group 4 code is shorter than Flate's but whose `/DecodeParms` aren't, stays Flate. Rows+/// wider than Group 4 codes (`MAX_G4_COLUMNS`) stay Flate too, and still clear.+#[test]+fn a_cleared_image_group_4_wouldnt_shrink_stays_flate() {+ let wide = pdfcraft_filters::MAX_G4_COLUMNS as usize + 1;+ for (w, h) in [(3usize, 2usize), (wide, 2)] {+ let stride = w.div_ceil(8);+ let dict = format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /FlateDecode");+ let mut doc = one_page(+ b"q 200 0 0 20 0 100 cm /Im Do Q",+ "/XObject << /Im 7 0 R >>",+ vec![stream(&dict, pdfcraft_filters::encode_flate(&vec![0xFF; stride * h]))],+ );+ // The top row's left half.+ mark(&mut doc, 0, &[[0.0, 112.0, 99.0, 118.0]], "");+ assert_eq!(apply(&mut doc, None).unwrap().images_cleared, 1, "{w} × {h}");+ let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap();+ assert_eq!(named_stream(&saved, 0, b"PCRedacted1").unwrap().dict.name(b"Filter"), Some(&b"FlateDecode"[..]), "{w} × {h}");+ let samples = named_image(&saved, 0, b"PCRedacted1").unwrap();+ let sample = |x: usize| samples.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0);+ assert!(!sample(0) && sample(w - 1), "{w} × {h}: the top row's first sample is cleared and its last kept");+ assert!(samples.get(stride..).is_some_and(|row| row.iter().all(|b| *b == 0xFF)), "{w} × {h}: the bottom row is kept");}}@@ -2525,7 +2631,8 @@ fn assert_no_secret(doc: &Document, secret: &[u8]) {pdfcraft_cos::serialize(&object, &mut bytes);assert!(!bytes.windows(secret.len()).any(|w| w == secret), "secret remains in object {r:?}");if let Object::Stream(s) = &*object {- let data = s.decoded().unwrap();+ // Image codecs, a cleared scan coded as Group 4 or JBIG2, decode through the document.+ let data = s.decoded().or_else(|_| saved.decoded_image(s)).unwrap();assert!(!data.windows(secret.len()).any(|w| w == secret), "secret remains in stream {r:?}");}}diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.tomlindex 4878eac62..de1d9d9e3 100644--- a/parity/acrobat-features.toml+++ b/parity/acrobat-features.toml@@ -5804,8 +5804,8 @@ milestone = "M8"status = "shipped"commands = ["redact.apply"]tools = ["redact_apply"]-evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared", "crates/redact/src/tests.rs::a_jbig2_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole", "crates/redact/src/tests.rs::a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals"]-notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate (larger than a Group 4 or JBIG2 scan) and refers to no globals; globals another image still uses stay, and may hold the shapes of marks under the area (not where they were)"+evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared", "crates/redact/src/tests.rs::a_jbig2_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole", "crates/redact/src/tests.rs::a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals", "crates/redact/src/tests.rs::a_cleared_black_and_white_scan_is_stored_as_group_4"]+notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate, or for a one-bit image of one component Group 4, or JBIG2 when it was JBIG2, whichever is smallest as stored, and refers to no globals; globals another image still uses stay, and may hold the shapes of marks under the area (not where they were)"[[feature]]id = "protect.redact-vectors"