Skip to content

A fast, helpful, and open-source document parser

wip

Logan Markewichc68460133943
diff --git a/Cargo.lock b/Cargo.lock
index e3e6c4d..2492640 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2101,6 +2101,7 @@ dependencies = [
"serde",
"serde_json",
"skrifa",
+ "ssf-rs",
"thiserror 2.0.19",
"tiny-skia",
"unicode-normalization",
@@ -4147,6 +4148,15 @@ dependencies = [
"unicode-segmentation",
]
+[[package]]
+name = "ssf-rs"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b553b4f0a04a9d3c330973c51f285f25315f56931522ebf101bd5c5ebc613f9"
+dependencies = [
+ "regex",
+]
+
[[package]]
name = "stable_deref_trait"
version = "1.2.1"
diff --git a/crates/liteparse-ooxml/Cargo.toml b/crates/liteparse-ooxml/Cargo.toml
index a1b33c6..31fadca 100644
--- a/crates/liteparse-ooxml/Cargo.toml
+++ b/crates/liteparse-ooxml/Cargo.toml
@@ -13,6 +13,11 @@ quick-xml = { version = "0.39", features = ["serialize"] }
serde = { version = "1", features = ["derive"] }
zip = { version = "2", default-features = false, features = ["deflate"] }
thiserror = "2"
+# XLSX number formatting (§18.8.31). A faithful port of SheetJS's `ssf`, which
+# is what makes it worth taking as a dependency: the reference implementation is
+# JS, so a port can be diffed against its origin rather than only against a
+# spec reading. Apache-2.0, one transitive dep (`regex`).
+ssf-rs = "0.1"
log = "0.4"
rustc-hash = "2"
# C′ measurer stack (spikes 6–8): resolution via fontdb, metrics/advances via
diff --git a/crates/liteparse-ooxml/src/xlsx/mod.rs b/crates/liteparse-ooxml/src/xlsx/mod.rs
index eb1b9c4..e45a73e 100644
--- a/crates/liteparse-ooxml/src/xlsx/mod.rs
+++ b/crates/liteparse-ooxml/src/xlsx/mod.rs
@@ -24,7 +24,10 @@
//! for sheet in &wb.sheets {
//! for row in &sheet.rows {
//! for cell in &row.cells {
+//! // The raw value and its format code, kept apart …
//! let text = wb.cell_text(cell);
+//! // … and joined on demand: `0.155` + `0.0%` → `15.5%`.
+//! let shown = wb.display_text(cell);
//! }
//! }
//! }
@@ -33,12 +36,6 @@
//!
//! # What this layer does *not* do
//!
-//! * **Number formatting.** A cell keeps its raw `f64` and its format code;
-//! turning `0.155` into `15.5%` is an interpreter, and the corpus says it is
-//! a real one — 665 distinct format codes, 50.0% of documents needing
-//! multi-section `pos;neg;zero;text` dispatch, 34.9% needing `_` skip-width
-//! padding. That is a vendored component (`ssf-rs` / `ironcalc`), landing
-//! separately, and [`Workbook::format_code`] is the seam it plugs into.
//! * **Geometry.** Column widths stay in Excel's character unit and row
//! heights in points; converting to a page needs the Normal font's digit
//! width. That is the geometry pass.
@@ -46,6 +43,7 @@
//! keeping it out of here is what lets this module be tested against a
//! corpus with no renderer in the loop.
+pub mod numfmt;
pub mod package;
pub mod refs;
pub mod sheet;
@@ -272,6 +270,18 @@ mod tests {
assert_eq!(pct.value, CellValue::Number(0.155));
assert_eq!(wb.format_code(pct), "0.0%");
assert!(wb.cell_text(pct).is_none());
+ // …and joining them is `display_text`, not the reader's business.
+ assert_eq!(wb.display_text(pct).unwrap(), "15.5%");
+ assert_eq!(wb.display_text(plain).unwrap(), "1234.5");
+ }
+
+ /// A shared string reaches the formatter through the table, so the text
+ /// path cannot be exercised by [`numfmt::render`] alone.
+ #[test]
+ fn display_text_resolves_a_shared_string_before_formatting_it() {
+ let wb = workbook();
+ let cell = &wb.sheets[0].rows[0].cells[0];
+ assert_eq!(wb.display_text(cell).unwrap(), "Total assets");
}
/// A chartsheet is declared in `<sheets>` exactly like a worksheet. Parsing
diff --git a/crates/liteparse-ooxml/src/xlsx/numfmt.rs b/crates/liteparse-ooxml/src/xlsx/numfmt.rs
new file mode 100644
index 0000000..b48d27c
--- /dev/null
+++ b/crates/liteparse-ooxml/src/xlsx/numfmt.rs
@@ -0,0 +1,432 @@
+//! Number formatting — the seam [`Workbook::format_code`] left open.
+//!
+//! A cell stores `0.155`; the sheet shows `15.5%`. The gap between the two is
+//! ECMA-376 §18.8.31's format grammar, and the finance-slice census said it is
+//! not a small one: **50.0% of documents need multi-section `pos;neg;zero;text`
+//! dispatch**, 36.0% need quoted literals, 34.9% need `_` skip-width padding,
+//! 23.2% `?` placeholders, 23.0% `*` fill, 14.4% `[Red]`. Excel's Accounting
+//! code — `_("$"* #,##0.00_);…` in 99 documents — exercises five of those at
+//! once.
+//!
+//! So the interpreter is [`ssf_rs`], a port of SheetJS's `ssf`, and this module
+//! is only the adapter: which of [`CellValue`]'s six shapes gets fed to it,
+//! what happens when it refuses, and what the caller is handed back.
+//!
+//! # Padding is preserved, not trimmed
+//!
+//! `_(* #,##0_)` renders `1234` as `" 1,235 "`. Those spaces are the format
+//! doing its job — aligning a column — and stripping them here would be this
+//! layer deciding how the value gets *presented*. A markdown emitter should
+//! trim; a fixed-width one should not. The reader keeps what the format says.
+//!
+//! Colour and condition brackets (`[Red]`, `[<100]`) are consumed by the
+//! interpreter and never reach the string, which is correct for a text target
+//! and lossy for a styled one. Cell colour is out of scope for the reader
+//! entirely (see the module docs for [`super`]).
+
+use ssf_rs::Value;
+
+use super::sheet::{Cell, CellValue};
+use super::styles::GENERAL;
+use super::{RichText, Workbook};
+
+/// Render a value with a format code, exactly as the interpreter sees it.
+///
+/// Returns `Err` when the code is one `ssf` will not evaluate — a fifth
+/// section, a malformed bracket. Callers that must produce *something* want
+/// [`Workbook::display_text`], which applies the fallback; this one exists so a
+/// corpus run can count the refusals instead of hiding them.
+pub fn render(code: &str, value: &CellValue, date1904: bool) -> Result<String, String> {
+ match value {
+ CellValue::Number(n) => {
+ let out = ssf_rs::format(code, &Value::Num(*n), date1904)?;
+ // Deliberate divergence from the oracle — see `is_spliced_exponent`.
+ if is_spliced_exponent(*n, code, &out) {
+ return ssf_rs::format(GENERAL, &Value::Num(*n), date1904);
+ }
+ // Nothing to protect: a number has no source text of its own, so
+ // any `NaN` in the output came from the interpreter.
+ Ok(strip_nan(code, out, None))
+ }
+ CellValue::Bool(b) => Ok(bool_text(*b).to_string()),
+ // An error is what the sheet displays verbatim; there is no format
+ // section for it. Excel shows `#REF!` under any code.
+ CellValue::Error(e) => Ok(e.clone()),
+ // ECMA-376 Strict's literal ISO date. It is already a display string,
+ // and re-deriving a serial to re-format it would only lose precision.
+ CellValue::Date(d) => Ok(d.clone()),
+ CellValue::Text(t) => Ok(render_text(code, &t.plain())),
+ // A shared-string index means nothing without the table; the resolving
+ // path is on `Workbook`.
+ CellValue::SharedString(_) => Err("shared string needs the workbook table".into()),
+ }
+}
+
+/// Whether the interpreter spliced digits into an exponent string and produced
+/// something that reads as a different number entirely.
+///
+/// SheetJS formats by placing digits into a string produced by JS's
+/// number-to-string, which switches to exponent notation outside
+/// `[1e-6, 1e21)`. Neither implementation handles the switch: `1e-7` under
+/// `0.00000000` is `"1e-7.00000000"` in SheetJS and `"1.2e-70000"` here; `1e21`
+/// under `#,##0.00` is `"1e,+21.00"` in both.
+///
+/// This is the one place the port is deliberately not followed, because the
+/// two failures are not equally bad — `"1.2e-70000"` reads as a nineteen-digit
+/// number. `General` renders the value truthfully (`1.23E-10`), and a truthful
+/// number under the wrong presentation beats a presentable number that is off
+/// by orders of magnitude.
+///
+/// The detection is on the **output**, not the input, and the difference is
+/// load-bearing: `1.23e-7` under `0.00` is `"0.00"` in both implementations,
+/// because the rounding discards the mantissa before the splice can go wrong.
+/// Rejecting every out-of-range magnitude would have broken those, which the
+/// oracle diff duly reported.
+///
+/// The corpus reaches an out-of-range magnitude in **1,436 cells across 9 of
+/// 1,178 workbooks**, so the guard is not hypothetical and is also not the
+/// common path.
+fn is_spliced_exponent(n: f64, code: &str, out: &str) -> bool {
+ let mag = n.abs();
+ if n == 0.0 || !n.is_finite() || (1e-6..1e21).contains(&mag) {
+ return false;
+ }
+ has_exponent_marker(out) && !code_writes_exponents(code)
+}
+
+/// `e`/`E` followed by a sign or digit — an exponent, not a letter that happens
+/// to sit next to a number.
+///
+/// Group separators and decimal points are skipped over, because the splice
+/// puts them *inside* the exponent: `1e21` under `#,##0.00` comes out as
+/// `"1e,+21.00"`, with the thousands comma landing between the `e` and its
+/// sign. Requiring the sign to be adjacent misses precisely the case that
+/// looks most like a real number.
+fn has_exponent_marker(s: &str) -> bool {
+ let bytes = s.as_bytes();
+ for (i, c) in bytes.iter().enumerate() {
+ if !matches!(c, b'e' | b'E') {
+ continue;
+ }
+ let next = bytes[i + 1..]
+ .iter()
+ .find(|b| !matches!(b, b',' | b'.' | b' '));
+ if next.is_some_and(|n| matches!(n, b'+' | b'-') || n.is_ascii_digit()) {
+ return true;
+ }
+ }
+ false
+}
+
+/// Whether the code itself asks for scientific notation (`0.00E+00`).
+///
+/// Only unquoted `E` is a placeholder: `"Revenue"` and `\e` are literals, and a
+/// code carrying one must not be mistaken for a scientific format — that would
+/// disable the guard for exactly the currency and label formats it exists for.
+fn code_writes_exponents(code: &str) -> bool {
+ let mut in_quotes = false;
+ let mut in_bracket = false;
+ let mut chars = code.chars();
+ while let Some(c) = chars.next() {
+ match c {
+ '\\' => {
+ chars.next();
+ }
+ '"' => in_quotes = !in_quotes,
+ '[' if !in_quotes => in_bracket = true,
+ ']' if !in_quotes => in_bracket = false,
+ 'e' | 'E' if !in_quotes && !in_bracket => return true,
+ _ => {}
+ }
+ }
+ false
+}
+
+/// Excel writes booleans as `0`/`1` and displays them uppercased, under any
+/// format code — the numeric sections do not apply to `t="b"`.
+fn bool_text(b: bool) -> &'static str {
+ if b { "TRUE" } else { "FALSE" }
+}
+
+/// Text only meets the format when the code has something to say about it.
+///
+/// A code with no sections and no `@` — `0.00`, `#,##0` — leaves a string
+/// alone, which is both Excel's behaviour and the reason a shared string used
+/// 40,000 times is not run through the interpreter 40,000 times. Anything with
+/// a `;` in it goes to the interpreter, because the *fourth* section is the
+/// text section and an **empty** one blanks the cell: `#,##0.00;;;` displays
+/// nothing at all, and a `@`-check alone would have rendered the string. The
+/// oracle diff caught exactly that, on three corpus codes.
+fn render_text(code: &str, text: &str) -> String {
+ if code == GENERAL || (!code.contains('@') && !code.contains(';')) {
+ return text.to_string();
+ }
+ let out = ssf_rs::format(code, &Value::Text(text.to_string()), false)
+ .unwrap_or_else(|_| text.to_string());
+ strip_nan(code, out, Some(text))
+}
+
+/// Remove a `NaN` the interpreter leaked into a cell.
+///
+/// `*` repeats the next character until the column is full, so with no column
+/// width the repeat count is unknown; SheetJS repeats zero times and the port
+/// stringifies the count instead — `@*.` renders `"Vareforbrug"` as
+/// `"VareforbrugNaN."`. Removing the token reproduces the origin exactly, which
+/// is why this is a repair and not a second opinion.
+///
+/// Gated on the `*` that leaks it, and on the code not writing `NaN` as a
+/// literal — and then applied **only outside the value's own text**, because a
+/// blind `replace` is a data-loss bug waiting for the right cell. The Zenodo
+/// corpus supplied that cell: a conference named `NaNA`, in a paper title, and
+/// an ungated strip shortens it to `A`. Skipping the repair whenever the source
+/// contains `NaN` would be the opposite error — it leaves the leak in the one
+/// title that has both.
+///
+/// `protect` is the source text a text cell renders from; a number has none,
+/// and only renders `NaN` when the value itself is one.
+///
+/// **4 cells in 24M** hit the real leak; the guard is here because a cell
+/// reading `NaN` is not a presentation defect, it is a wrong answer.
+fn strip_nan(code: &str, out: String, protect: Option<&str>) -> String {
+ if !out.contains("NaN") || code.contains("NaN") || !code.contains('*') {
+ return out;
+ }
+ let Some(text) = protect.filter(|t| t.contains("NaN")) else {
+ return out.replace("NaN", "");
+ };
+ // The format wraps the value in literals, so the value's own text is a
+ // contiguous span of the output. Only what surrounds it is the interpreter
+ // speaking.
+ match out.find(text) {
+ Some(at) => {
+ let (before, rest) = out.split_at(at);
+ let (mid, after) = rest.split_at(text.len());
+ format!(
+ "{}{mid}{}",
+ before.replace("NaN", ""),
+ after.replace("NaN", "")
+ )
+ }
+ // The value did not survive verbatim, so there is no span to protect
+ // and no safe strip. Leaving the leak is the recoverable failure.
+ None => out,
+ }
+}
+
+impl Workbook {
+ /// The string a reader of the sheet sees in this cell.
+ ///
+ /// Infallible by construction: a format code the interpreter refuses falls
+ /// back to `General`, and a `General` that somehow fails falls back to
+ /// Rust's own float rendering. A cell never renders as nothing because its
+ /// *style* was unusable — losing the number would be a strictly worse
+ /// outcome than losing its presentation.
+ ///
+ /// Returns `None` only for a shared-string index the table cannot resolve,
+ /// which is corruption and already warned about by
+ /// [`Workbook::cell_text`].
+ pub fn display_text(&self, cell: &Cell) -> Option<String> {
+ let code = self.format_code(cell);
+ if let CellValue::SharedString(_) = cell.value {
+ let text = self.cell_text(cell)?;
+ return Some(render_text(code, &text.plain()));
+ }
+ Some(match render(code, &cell.value, self.date1904) {
+ Ok(s) => s,
+ Err(e) => {
+ log::debug!("format code {code:?} refused ({e}); falling back to General");
+ self.fallback(&cell.value)
+ }
+ })
+ }
+
+ fn fallback(&self, value: &CellValue) -> String {
+ match render(GENERAL, value, self.date1904) {
+ Ok(s) => s,
+ Err(_) => match value {
+ CellValue::Number(n) => n.to_string(),
+ _ => String::new(),
+ },
+ }
+ }
+
+ /// Whether a cell's format code makes it a date rather than a number.
+ ///
+ /// The serial `45000.25` is a number until its format says `m/d/yyyy`, so
+ /// this is the only way to tell — nothing in the cell itself distinguishes
+ /// a date from a count.
+ pub fn is_date_cell(&self, cell: &Cell) -> bool {
+ matches!(cell.value, CellValue::Number(_)) && ssf_rs::is_date(self.format_code(cell))
+ }
+}
+
+impl RichText {
+ /// The plain text of this run sequence, formatted by `code`.
+ ///
+ /// Convenience for a caller holding text it already resolved.
+ pub fn display_with(&self, code: &str) -> String {
+ render_text(code, &self.plain())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::xlsx::text::{RichText, RunProps, TextRun};
+
+ fn plain(text: &str) -> RichText {
+ RichText {
+ runs: vec![TextRun {
+ text: text.to_string(),
+ props: RunProps::default(),
+ }],
+ }
+ }
+
+ fn num(code: &str, n: f64) -> String {
+ render(code, &CellValue::Number(n), false).unwrap()
+ }
+
+ /// The six mechanisms the finance-slice census named, each at the value
+ /// that exercises it. Failing any one of these is a whole class of the
+ /// corpus rendering wrong, not a rounding difference.
+ #[test]
+ fn the_censused_mechanisms_render() {
+ // multi-section dispatch: 50.0% of documents
+ let acct = r#"_("$"* #,##0.00_);_("$"* \("$"* #,##0.00\);_("$"* "-"??_);_(@_)"#;
+ assert_eq!(num(acct, 1234.5678), " $1,234.57 ");
+ assert_eq!(num(acct, -1234.5678), " $($1,234.57)");
+ // zero section: `"-"??` is the accounting dash, padded to two digits
+ assert_eq!(num(acct, 0.0), " $- ");
+ // quoted literal: 36.0%
+ assert_eq!(num(r#"0.00"kg""#, 1234.5678), "1234.57kg");
+ // skip-width `_)`: 34.9% — one space, not a dropped character
+ assert_eq!(num("0.00_);(0.00)", 1234.5678), "1234.57 ");
+ // `[Red]` is consumed, never emitted: 14.4%
+ assert_eq!(num(r"#,##0.0;[Red]\-#,##0.0;0", -1234.5678), "-1,234.6");
+ // comma scaling: 0.2% of docs, but silently 1000x wrong if unhandled
+ assert_eq!(num("#,##0.00,,", 45000.25), "0.05");
+ // suppressed sections render empty rather than falling back
+ assert_eq!(num("0;;", -1234.5678), "");
+ assert_eq!(num("0;;", 0.0), "");
+ }
+
+ /// The guard's whole point is that the un-guarded output is not merely
+ /// unformatted but *misread*: `1.2e-70000` looks like a huge number.
+ #[test]
+ fn out_of_range_magnitudes_render_truthfully_instead_of_corrupt() {
+ assert_eq!(num("0.00000000", 1.23e-7), "0.000000123");
+ assert_eq!(num("#,##0.00", 1e21), "1E+21");
+ // …but a code whose rounding never reaches the mantissa is left alone,
+ // because it agrees with the oracle and `0.00` is the better answer.
+ assert_eq!(num("0.00", 1.23e-7), "0.00");
+ assert_eq!(num("#,##0.00", -1.23e-10), "-0.00");
+ // Just inside the bounds the format always applies.
+ assert_eq!(num("0.00000000", 1e-6), "0.00000100");
+ assert_eq!(num("0.00", 0.0), "0.00");
+ assert_eq!(num("#,##0.00", 1e20), "100,000,000,000,000,000,000.00");
+ }
+
+ /// A scientific code produces an exponent on purpose; treating it as
+ /// corruption would route every such cell to `General`. A literal `E` in
+ /// quotes is not a placeholder and must not grant the same exemption.
+ #[test]
+ fn a_scientific_code_keeps_its_exponent() {
+ assert_eq!(num("0.00E+00", 1.23e-7), "1.23E-07");
+ assert!(code_writes_exponents("0.00E+00"));
+ assert!(!code_writes_exponents(r#""Revenue"0.00"#));
+ assert!(!code_writes_exponents(r"\e0.00"));
+ assert!(!code_writes_exponents("[Red]0.00"));
+ assert!(has_exponent_marker("1.2e-70000"));
+ assert!(!has_exponent_marker("Revenue"));
+ }
+
+ #[test]
+ fn a_serial_becomes_a_date_only_because_the_code_says_so() {
+ assert_eq!(num("m/d/yyyy", 45000.25), "3/15/2023");
+ assert_eq!(num("General", 45000.25), "45000.25");
+ assert!(ssf_rs::is_date("[$-409]mmmm d, yyyy;@"));
+ assert!(!ssf_rs::is_date("#,##0.00"));
+ }
+
+ /// The 1904 epoch is 1,462 days off the 1900 one. A workbook-level flag
+ /// that the renderer ignores shifts every date in the file by four years.
+ #[test]
+ fn the_1904_epoch_is_carried_not_assumed() {
+ let v = CellValue::Number(45000.0);
+ assert_eq!(render("m/d/yyyy", &v, false).unwrap(), "3/15/2023");
+ assert_eq!(render("m/d/yyyy", &v, true).unwrap(), "3/16/2027");
+ }
+
+ #[test]
+ fn text_meets_the_format_only_through_an_at_section() {
+ // No `@` anywhere: the numeric code does not touch a string.
+ assert_eq!(render_text("0.00", "hello"), "hello");
+ assert_eq!(render_text(GENERAL, "hello"), "hello");
+ // The fourth section of Accounting pads text like it pads numbers.
+ assert_eq!(render_text(r#"_(* #,##0_);;;_(@_)"#, "hello"), " hello ");
+ assert_eq!(render_text(r#""Note: "@"#, "x"), "Note: x");
+ }
+
+ /// The fourth section governs text even when it is empty, and empty means
+ /// the cell displays nothing. Skipping the interpreter for codes without
+ /// `@` renders the string instead — three corpus codes, found by the
+ /// SheetJS diff rather than by reading the spec.
+ #[test]
+ fn an_empty_fourth_section_blanks_text() {
+ assert_eq!(render_text(";;;", "NO"), "");
+ assert_eq!(render_text("#,##0.00;;;", "0"), "");
+ assert_eq!(
+ render_text(r#"0.000_ ;[Red]\-0.000_ ;"inactive";"#, "x"),
+ ""
+ );
+ }
+
+ /// The port leaks its `*`-repeat count into the string when there is no
+ /// column width to fill. Four corpus cells, found by the oracle diff.
+ #[test]
+ fn a_leaked_nan_is_removed_but_a_literal_one_is_kept() {
+ assert_eq!(render_text("@*.", "Vareforbrug"), "Vareforbrug.");
+ assert_eq!(render_text(r#"@" NaN""#, "x"), "x NaN");
+ // The Zenodo counterexample: a conference called NaNA, under a fill
+ // code. An ungated strip shortens the title.
+ assert_eq!(
+ render_text("@*.", "Networking and Network Applications (NaNA)"),
+ "Networking and Network Applications (NaNA)."
+ );
+ }
+
+ #[test]
+ fn booleans_and_errors_ignore_the_format_code() {
+ assert_eq!(
+ render("0.00", &CellValue::Bool(true), false).unwrap(),
+ "TRUE"
+ );
+ assert_eq!(
+ render("0.00", &CellValue::Bool(false), false).unwrap(),
+ "FALSE"
+ );
+ let err = CellValue::Error("#REF!".into());
+ assert_eq!(render("0.00%", &err, false).unwrap(), "#REF!");
+ }
+
+ #[test]
+ fn a_strict_iso_date_is_already_a_display_string() {
+ let d = CellValue::Date("2023-03-15T00:00:00".into());
+ assert_eq!(
+ render("m/d/yyyy", &d, false).unwrap(),
+ "2023-03-15T00:00:00"
+ );
+ }
+
+ #[test]
+ fn a_shared_string_needs_the_workbook() {
+ assert!(render("General", &CellValue::SharedString(0), false).is_err());
+ }
+
+ #[test]
+ fn rich_text_formats_through_its_plain_text() {
+ let rt = plain("hello");
+ assert_eq!(rt.display_with(r#""Note: "@"#), "Note: hello");
+ }
+}