Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): replace custom document parser with anydoc (#4225)

tomsideguideaebeba7f9adf
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.toml
index 4fbe8afb0..f5ee9a40c 100644
--- a/apps/api/native/Cargo.toml
+++ b/apps/api/native/Cargo.toml
@@ -9,11 +9,11 @@ version = "0.1.0"
crate-type = ["cdylib"]
[dependencies]
+anydoc = "0.1.2"
chrono = { version = "0.4", features = ["serde"] }
kuchikiki = "0.8.2"
lol_html = "2.6.0"
pdf-inspector = "0.1.0"
-maud = "0.27.0"
napi = { version = "3.0.0", features = ["serde-json", "tokio_rt"] }
napi-derive = "3.0.0"
nodesig = { git = "https://github.com/firecrawl/nodesig" }
@@ -25,11 +25,6 @@ serde_json = "1.0.143"
strsim = "0.11"
texting_robots = "0.2.2"
url = "2.5.7"
-zip = "5.0.0"
-calamine = { git = "https://github.com/firecrawl/calamine", branch = "fc-prod" }
-cfb = "0.10"
-codepage = "0.1"
-encoding_rs = "0.8"
tokio = "1.48.0"
tracing = "0.1"
tracing-subscriber = { version = "0.3", default-features = false, features = ["registry"] }
diff --git a/apps/api/native/src/document.rs b/apps/api/native/src/document.rs
new file mode 100644
index 000000000..f019113d2
--- /dev/null
+++ b/apps/api/native/src/document.rs
@@ -0,0 +1,22 @@
+use napi::bindgen_prelude::*;
+use napi_derive::napi;
+
+/// Convert a document (doc, docx, odt/ods/odp, rtf, xls/xlsx, ppt/pptx, epub,
+/// csv, ...) to GitHub-Flavored Markdown using anydoc. The format is detected
+/// from the file content; `extension_hint` (with or without a leading dot) is
+/// only consulted for signature-less formats like CSV.
+#[napi]
+pub fn convert_document_to_markdown(
+ data: &[u8],
+ extension_hint: Option<String>,
+) -> Result<String> {
+ let format = anydoc::Format::from_bytes(data).or_else(|| {
+ extension_hint
+ .as_deref()
+ .map(|ext| ext.trim_start_matches('.'))
+ .and_then(anydoc::Format::from_extension)
+ });
+
+ anydoc::to_markdown_bytes(data, format)
+ .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))
+}
diff --git a/apps/api/native/src/document/encoding.rs b/apps/api/native/src/document/encoding.rs
deleted file mode 100644
index 2b107feaa..000000000
--- a/apps/api/native/src/document/encoding.rs
+++ /dev/null
@@ -1,38 +0,0 @@
-use encoding_rs::{Encoding, WINDOWS_1252};
-
-/// Encoding for a Windows code page, excluding the Unicode pseudo code
-/// pages.
-pub fn encoding_for_codepage(codepage: u16) -> Option<&'static Encoding> {
- match codepage {
- 0 | 1200 | 1201 | 65000 | 65001 => None,
- cp => codepage::to_encoding(cp),
- }
-}
-
-/// Default ANSI code page encoding for a Windows language ID.
-pub fn encoding_for_lid(lid: u16) -> &'static Encoding {
- let codepage = match lid {
- // sublanguage decides the script for these
- 0x0404 | 0x0C04 | 0x1404 | 0x7C04 => 950, // Chinese (Taiwan, Hong Kong, Macau, Traditional)
- 0x0C1A | 0x1C1A => 1251, // Serbian (Cyrillic)
- 0x082C | 0x0843 => 1251, // Azerbaijani, Uzbek (Cyrillic)
- _ => match lid & 0x3FF {
- 0x04 => 936, // Chinese (Simplified unless a Traditional sublanguage above)
- // Central European
- 0x05 | 0x0E | 0x15 | 0x18 | 0x1A | 0x1B | 0x1C | 0x24 => 1250,
- // Cyrillic
- 0x02 | 0x19 | 0x22 | 0x23 | 0x2F | 0x3F | 0x40 | 0x44 | 0x50 => 1251,
- 0x08 => 1253, // Greek
- 0x1F | 0x2C | 0x43 => 1254, // Turkish, Azerbaijani, Uzbek (Latin)
- 0x0D => 1255, // Hebrew
- 0x01 | 0x20 | 0x29 => 1256, // Arabic, Urdu, Persian
- 0x25 | 0x26 | 0x27 => 1257, // Baltic
- 0x2A => 1258, // Vietnamese
- 0x1E => 874, // Thai
- 0x11 => 932, // Japanese
- 0x12 => 949, // Korean
- _ => 1252,
- },
- };
- encoding_for_codepage(codepage).unwrap_or(WINDOWS_1252)
-}
diff --git a/apps/api/native/src/document/mod.rs b/apps/api/native/src/document/mod.rs
deleted file mode 100644
index 278c8c3c4..000000000
--- a/apps/api/native/src/document/mod.rs
+++ /dev/null
@@ -1,53 +0,0 @@
-pub mod encoding;
-pub mod model;
-pub mod oleps;
-pub mod providers;
-pub mod renderers;
-pub mod xml;
-
-pub use providers::factory::DocumentType;
-
-use crate::document::model::Document;
-use crate::document::providers::factory::ProviderFactory;
-use crate::document::renderers::html::HtmlRenderer;
-use napi::bindgen_prelude::*;
-use napi_derive::napi;
-
-#[napi]
-pub struct DocumentConverter {
- factory: ProviderFactory,
- html_renderer: HtmlRenderer,
-}
-
-impl Default for DocumentConverter {
- fn default() -> Self {
- Self::new()
- }
-}
-
-#[napi]
-impl DocumentConverter {
- #[napi(constructor)]
- pub fn new() -> Self {
- Self {
- factory: ProviderFactory::new(),
- html_renderer: HtmlRenderer::new(),
- }
- }
-
- #[napi]
- pub fn convert_buffer_to_html(
- &self,
- data: &[u8],
- doc_type: DocumentType,
- ) -> napi::Result<String> {
- let provider = self.factory.get_provider(doc_type);
-
- let document: Document = provider
- .parse_buffer(data)
- .map_err(|e| Error::new(Status::GenericFailure, format!("Provider error: {e}")))?;
-
- let html = self.html_renderer.render(&document);
- Ok(html)
- }
-}
diff --git a/apps/api/native/src/document/model/mod.rs b/apps/api/native/src/document/model/mod.rs
deleted file mode 100644
index 444acb88e..000000000
--- a/apps/api/native/src/document/model/mod.rs
+++ /dev/null
@@ -1,150 +0,0 @@
-use chrono::{DateTime, Utc};
-use std::num::NonZeroU32;
-
-#[derive(Debug, Clone)]
-pub struct Document {
- pub blocks: Vec<Block>,
- pub metadata: DocumentMetadata,
- pub notes: Vec<Note>,
- pub comments: Vec<Comment>,
-}
-
-#[derive(Debug, Clone, Default)]
-pub struct DocumentMetadata {
- pub title: Option<String>,
- pub author: Option<String>,
- pub created: Option<DateTime<Utc>>,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct NoteId(pub String);
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct CommentId(pub String);
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct BookmarkId(pub String);
-
-#[derive(Debug, Clone)]
-pub enum Block {
- Paragraph(Paragraph),
- Table(Table),
- List(List),
- Image(Image),
-}
-
-#[derive(Debug, Clone)]
-pub struct Paragraph {
- pub kind: ParagraphKind,
- pub inlines: Vec<Inline>,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum ParagraphKind {
- Normal,
- Heading(u8), // 1..=6 will render as <h1>.. <h6>
- Blockquote,
-}
-
-#[derive(Debug, Clone)]
-pub enum Inline {
- Text(String),
- LineBreak,
- Link { href: String, children: Vec<Inline> },
-
- Strong(Vec<Inline>),
- Em(Vec<Inline>),
- Del(Vec<Inline>),
- Code(String),
- Sup(Vec<Inline>),
- Sub(Vec<Inline>),
-
- FootnoteRef(NoteId),
- EndnoteRef(NoteId),
- CommentRef(CommentId),
- Bookmark(BookmarkId),
-}
-
-/// True when the inlines would render something the reader can see.
-pub fn has_visible_content(inlines: &[Inline]) -> bool {
- inlines.iter().any(|inline| match inline {
- Inline::Text(t) => !t.trim().is_empty(),
- Inline::LineBreak => false,
- Inline::Link { children, .. } => has_visible_content(children),
- Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {
- has_visible_content(c)
- }
- Inline::Code(t) => !t.trim().is_empty(),
- Inline::FootnoteRef(_) | Inline::EndnoteRef(_) | Inline::CommentRef(_) => true,
- Inline::Bookmark(_) => false,
- })
-}
-
-#[derive(Debug, Clone)]
-pub struct Image {
- pub src: String,
- pub alt: Option<String>,
-}
-
-#[derive(Debug, Clone)]
-pub struct Table {
- pub rows: Vec<TableRow>,
-}
-
-#[derive(Debug, Clone)]
-pub struct TableRow {
- pub cells: Vec<TableCell>,
- pub kind: TableRowKind,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum TableRowKind {
- Header,
- Body,
- Footer,
-}
-
-#[derive(Debug, Clone)]
-pub struct TableCell {
- pub blocks: Vec<Block>,
- pub colspan: NonZeroU32,
- pub rowspan: NonZeroU32,
-}
-
-#[derive(Debug, Clone)]
-pub struct List {
- pub items: Vec<ListItem>,
- pub list_type: ListType,
-}
-
-#[derive(Debug, Clone)]
-pub struct ListItem {
- pub blocks: Vec<Block>,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum ListType {
- Ordered,
- Unordered,
-}
-
-#[derive(Debug, Clone)]
-pub struct Note {
- pub id: NoteId,
- pub kind: NoteKind,
- pub blocks: Vec<Block>,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum NoteKind {
- Footnote,
- Endnote,
-}
-
-#[derive(Debug, Clone)]
-pub struct Comment {
- pub id: CommentId,
- pub author_name: Option<String>,
- pub author_initials: Option<String>,
- pub blocks: Vec<Block>,
-}
diff --git a/apps/api/native/src/document/oleps.rs b/apps/api/native/src/document/oleps.rs
deleted file mode 100644
index 7c2f09ce3..000000000
--- a/apps/api/native/src/document/oleps.rs
+++ /dev/null
@@ -1,136 +0,0 @@
-//! Minimal MS-OLEPS parser for the `\x05SummaryInformation` stream of OLE
-//! compound files.
-
-use crate::document::encoding::encoding_for_codepage;
-use chrono::{DateTime, Utc};
-use encoding_rs::WINDOWS_1252;
-
-const PID_CODEPAGE: u32 = 0x0001;
-const PIDSI_TITLE: u32 = 0x0002;
-const PIDSI_AUTHOR: u32 = 0x0004;
-const PIDSI_CREATE_DTM: u32 = 0x000C;
-
-const VT_I2: u32 = 0x0002;
-const VT_LPSTR: u32 = 0x001E;
-const VT_LPWSTR: u32 = 0x001F;
-const VT_FILETIME: u32 = 0x0040;
-
-/// Seconds between the FILETIME epoch (1601) and the Unix epoch (1970).
-const FILETIME_UNIX_EPOCH_DIFF_SECS: i64 = 11_644_473_600;
-
-#[derive(Debug, Default)]
-pub struct SummaryInformation {
- pub codepage: Option<u16>,
- pub title: Option<String>,
- pub author: Option<String>,
- pub created: Option<DateTime<Utc>>,
-}
-
-pub fn parse_summary_information(data: &[u8]) -> Option<SummaryInformation> {
- // byte order mark at 0, property set count at 0x18, first section offset at 0x2C
- if read_u16(data, 0)? != 0xFFFE {
- return None;
- }
- if read_u32(data, 0x18)? == 0 {
- return None;
- }
- let section_start = read_u32(data, 0x2C)? as usize;
-
- let num_properties = (read_u32(data, section_start + 4)? as usize).min(1024);
- let mut properties = Vec::with_capacity(num_properties);
- for i in 0..num_properties {
- let entry = section_start + 8 + i * 8;
- let prop_id = read_u32(data, entry)?;
- let prop_offset = section_start + read_u32(data, entry + 4)? as usize;
- properties.push((prop_id, prop_offset));
- }
-
- let mut info = SummaryInformation::default();
-
- // code page first, it governs VT_LPSTR decoding
- info.codepage = properties
- .iter()
- .find(|(id, _)| *id == PID_CODEPAGE)
- .and_then(|(_, offset)| {
- if read_u32(data, *offset)? != VT_I2 {
- return None;
- }
- read_u16(data, offset + 4)
- });
- let string_encoding = info
- .codepage
- .and_then(encoding_for_codepage)
- .unwrap_or(WINDOWS_1252);
-
- for (prop_id, offset) in properties {
- match prop_id {
- PIDSI_TITLE => info.title = read_string(data, offset, string_encoding),
- PIDSI_AUTHOR => info.author = read_string(data, offset, string_encoding),
- PIDSI_CREATE_DTM => info.created = read_filetime(data, offset),
- _ => {}
- }
- }
-
- Some(info)
-}
-
-fn read_string(
- data: &[u8],
- offset: usize,
- encoding: &'static encoding_rs::Encoding,
-) -> Option<String> {
- let vt = read_u32(data, offset)?;
- let len = read_u32(data, offset + 4)? as usize;
- let text = match vt {
- VT_LPSTR => {
- let bytes = data.get(offset + 8..offset + 8 + len)?;
- let (decoded, _, _) = encoding.decode(bytes);
- decoded.into_owned()
- }
- VT_LPWSTR => {
- let bytes = data.get(offset + 8..offset + 8 + len.checked_mul(2)?)?;
- let units: Vec<u16> = bytes
- .chunks_exact(2)
- .map(|c| u16::from_le_bytes([c[0], c[1]]))
- .collect();
- String::from_utf16_lossy(&units)
- }
- _ => return None,
- };
- let text = text.trim_end_matches('\0').trim();
- if text.is_empty() {
- None
- } else {
- Some(text.to_string())
- }
-}
-
-fn read_filetime(data: &[u8], offset: usize) -> Option<DateTime<Utc>> {
- if read_u32(data, offset)? != VT_FILETIME {
- return None;
- }
- let filetime = read_u64(data, offset + 4)?;
- if filetime == 0 {
- return None;
- }
- let secs = (filetime / 10_000_000) as i64 - FILETIME_UNIX_EPOCH_DIFF_SECS;
- let nanos = (filetime % 10_000_000) as u32 * 100;
- DateTime::from_timestamp(secs, nanos)
-}
-
-fn read_u16(data: &[u8], offset: usize) -> Option<u16> {
- let bytes = data.get(offset..offset + 2)?;
- Some(u16::from_le_bytes([bytes[0], bytes[1]]))
-}
-
-fn read_u32(data: &[u8], offset: usize) -> Option<u32> {
- let bytes = data.get(offset..offset + 4)?;
- Some(u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]))
-}
-
-fn read_u64(data: &[u8], offset: usize) -> Option<u64> {
- let bytes = data.get(offset..offset + 8)?;
- Some(u64::from_le_bytes([
- bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7],
- ]))
-}
diff --git a/apps/api/native/src/document/providers/doc.rs b/apps/api/native/src/document/providers/doc.rs
deleted file mode 100644
index 7fd10a85e..000000000
--- a/apps/api/native/src/document/providers/doc.rs
+++ /dev/null
@@ -1,1501 +0,0 @@
-//! Legacy Word binary (.doc) provider, following the MS-DOC piece table and
-//! formatting bin table (FKP) formats.
-
-use crate::document::encoding::{encoding_for_codepage, encoding_for_lid};
-use crate::document::model::*;
-use crate::document::oleps::parse_summary_information;
-use crate::document::providers::DocumentProvider;
-use cfb::CompoundFile;
-use encoding_rs::Encoding;
-use std::error::Error;
-use std::io::{Cursor, Read, Seek};
-use std::num::NonZeroU32;
-
-const MAGIC_WORD_97: u16 = 0xA5EC;
-const MAGIC_WORD_6_95: u16 = 0xA5DC;
-
-/// Upper bound on text extracted from one document.
-const MAX_TEXT_CHARS: usize = 10_000_000;
-
-const FKP_PAGE_SIZE: usize = 512;
-const MAX_FIELD_INSTRUCTION_CHARS: usize = 2048;
-
-pub struct DocProvider;
-
-impl DocProvider {
- pub fn new() -> Self {
- Self
- }
-}
-
-impl DocumentProvider for DocProvider {
- fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {
- let mut cfb = CompoundFile::open(Cursor::new(data))?;
-
- let summary = read_stream(&mut cfb, "\x05SummaryInformation")
- .ok()
- .and_then(|bytes| parse_summary_information(&bytes));
-
- let word_doc = read_stream(&mut cfb, "WordDocument")?;
- let fib = Fib::parse(&word_doc)?;
-
- let (primary, secondary) = if fib.table_stream_1 {
- ("1Table", "0Table")
- } else {
- ("0Table", "1Table")
- };
- let table = read_stream(&mut cfb, primary)
- .or_else(|_| read_stream(&mut cfb, secondary))
- .ok();
-
- let ansi = summary
- .as_ref()
- .and_then(|s| s.codepage)
- .and_then(encoding_for_codepage)
- .unwrap_or_else(|| encoding_for_lid(fib.lid));
-
- let segments = extract_segments(&word_doc, table.as_deref(), &fib, ansi);
- let formatting = if fib.legacy {
- Formatting::default()
- } else {
- table
- .as_deref()
- .map(|table| Formatting::parse(&word_doc, table, &fib))
- .unwrap_or_default()
- };
-
- let mut metadata = DocumentMetadata::default();
- if let Some(summary) = summary {
- metadata.title = summary.title;
- metadata.author = summary.author;
- metadata.created = summary.created;
- }
-
- Ok(Document {
- blocks: build_blocks(&segments, &formatting),
- metadata,
- notes: Vec::new(),
- comments: Vec::new(),
- })
- }
-
- fn name(&self) -> &'static str {
- "doc"
- }
-}
-
-fn read_stream<R: Read + Seek>(
- cfb: &mut CompoundFile<R>,
- name: &str,
-) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
- let mut stream = cfb.open_stream(name)?;
- let mut buf = Vec::new();
- stream.read_to_end(&mut buf)?;
- Ok(buf)
-}
-
-struct Fib {
- legacy: bool,
- lid: u16,
- table_stream_1: bool,
- ext_char: bool,
- fc_min: usize,
- fc_mac: usize,
- ccp_text: usize,
- fc_clx: usize,
- lcb_clx: usize,
- fc_plcf_bte_chpx: usize,
- lcb_plcf_bte_chpx: usize,
- fc_plcf_bte_papx: usize,
- lcb_plcf_bte_papx: usize,
-}
-
-impl Fib {
- fn parse(word_doc: &[u8]) -> Result<Self, Box<dyn Error + Send + Sync>> {
- if word_doc.len() < 0x20 {
- return Err("WordDocument stream too short".into());
- }
- let magic = read_u16(word_doc, 0x00);
- let legacy = match magic {
- MAGIC_WORD_97 => false,
- MAGIC_WORD_6_95 => true,
- _ => return Err(format!("not a Word document (magic {magic:#06X})").into()),
- };
-
- let flags = read_u16(word_doc, 0x0A);
- if flags & 0x0100 != 0 {
- return Err("encrypted .doc files are not supported".into());
- }
-
- let u32_at = |offset: usize| {
- if word_doc.len() >= offset + 4 {
- read_u32(word_doc, offset) as usize
- } else {
- 0
- }
- };
-
- Ok(Fib {
- legacy,
- lid: read_u16(word_doc, 0x06),
- table_stream_1: flags & 0x0200 != 0,
- ext_char: flags & 0x1000 != 0,
- fc_min: read_u32(word_doc, 0x18) as usize,
- fc_mac: read_u32(word_doc, 0x1C) as usize,
- ccp_text: u32_at(0x4C),
- fc_clx: u32_at(0x1A2),
- lcb_clx: u32_at(0x1A6),
- fc_plcf_bte_chpx: u32_at(0xFA),
- lcb_plcf_bte_chpx: u32_at(0xFE),
- fc_plcf_bte_papx: u32_at(0x102),
- lcb_plcf_bte_papx: u32_at(0x106),
- })
- }
-}
-
-/// Decoded text run and the stream offset its bytes came from.
-struct TextSegment {
- text: String,
- fc_start: u32,
- bytes_per_char: u8,
-}
-
-fn extract_segments(
- word_doc: &[u8],
- table: Option<&[u8]>,
- fib: &Fib,
- ansi: &'static Encoding,
-) -> Vec<TextSegment> {
- if !fib.legacy {
- if let Some(table) = table {
- if let Some(pieces) = parse_piece_table(table, fib) {
- return decode_pieces(word_doc, &pieces, fib.ccp_text, ansi);
- }
- }
- }
-
- // Word 6/95 files and corrupt piece tables: text is at fcMin..fcMac.
- let start = fib.fc_min.min(word_doc.len());
- let end = fib.fc_mac.clamp(start, word_doc.len());
- let bytes = &word_doc[start..end];
- let (text, bytes_per_char) = if fib.ext_char {
- (decode_utf16le(bytes), 2)
- } else {
- (ansi.decode_without_bom_handling(bytes).0.into_owned(), 1)
- };
- vec![TextSegment {
- text,
- fc_start: start as u32,
- bytes_per_char,
- }]
-}
-
-struct Piece {
- start_cp: u32,
- end_cp: u32,
- offset: usize,
- compressed: bool,
-}
-
-fn parse_piece_table(table: &[u8], fib: &Fib) -> Option<Vec<Piece>> {
- if fib.lcb_clx == 0 {
- return None;
- }
- let clx = table.get(fib.fc_clx..fib.fc_clx.checked_add(fib.lcb_clx)?)?;
-
- // Clx: zero or more Prc (0x01) blocks, then the Pcdt (0x02) piece table.
- let mut i = 0;
- while i < clx.len() {
- match clx[i] {
- 0x01 => {
- let cb = read_u16(clx.get(i + 1..i + 3)?, 0) as usize;
- i += 3 + cb;
- }
- 0x02 => {
- let lcb = read_u32(clx.get(i + 1..i + 5)?, 0) as usize;
- let plc = clx.get(i + 5..(i + 5).checked_add(lcb)?)?;
- return parse_plc_pcd(plc);
- }
- _ => return None,
- }
- }
- None
-}
-
-fn parse_plc_pcd(plc: &[u8]) -> Option<Vec<Piece>> {
- // n+1 CPs (4 bytes each) followed by n PCDs (8 bytes each)
- if plc.len() < 16 || (plc.len() - 4) % 12 != 0 {
- return None;
- }
- let n = (plc.len() - 4) / 12;
-
- let mut pieces = Vec::with_capacity(n);
- for k in 0..n {
- let start_cp = read_u32(plc, k * 4);
- let end_cp = read_u32(plc, (k + 1) * 4);
- if end_cp < start_cp {
- return None;
- }
- let fc = read_u32(plc, 4 * (n + 1) + 8 * k + 2);
- let compressed = fc & 0x4000_0000 != 0;
- let offset = if compressed {
- ((fc & 0x3FFF_FFFF) / 2) as usize
- } else {
- (fc & 0x3FFF_FFFF) as usize
- };
- pieces.push(Piece {
- start_cp,
- end_cp,
- offset,
- compressed,
- });
- }
- Some(pieces)
-}
-
-fn decode_pieces(
- word_doc: &[u8],
- pieces: &[Piece],
- ccp_text: usize,
- ansi: &'static Encoding,
-) -> Vec<TextSegment> {
- // piece cp ranges are contiguous, so this clamp bounds total output
- let ccp_text = ccp_text.min(MAX_TEXT_CHARS);
- let mut segments = Vec::new();
- for piece in pieces {
- // body text only; cps past ccpText are headers, footnotes etc.
- if piece.start_cp as usize >= ccp_text {
- continue;
- }
- let count = (piece.end_cp as usize).min(ccp_text) - piece.start_cp as usize;
- let available = word_doc.len().saturating_sub(piece.offset);
- let start = piece.offset.min(word_doc.len());
- let (text, bytes_per_char) = if piece.compressed {
- let bytes = &word_doc[start..][..count.min(available)];
- (ansi.decode_without_bom_handling(bytes).0.into_owned(), 1)
- } else {
- let bytes = &word_doc[start..][..(count * 2).min(available)];
- (decode_utf16le(bytes), 2)
- };
- segments.push(TextSegment {
- text,
- fc_start: start as u32,
- bytes_per_char,
- });
- }
- segments
-}
-
-fn decode_utf16le(bytes: &[u8]) -> String {
- let units: Vec<u16> = bytes
- .chunks_exact(2)
- .map(|c| u16::from_le_bytes([c[0], c[1]]))
- .collect();
- String::from_utf16_lossy(&units)
-}
-
-#[derive(Clone, Copy, Default, PartialEq, Eq)]
-struct CharProps {
- bold: bool,
- italic: bool,
- strike: bool,
-}
-
-#[derive(Clone, Copy, Default)]
-struct ParaProps {
- in_table: bool,
- row_end: bool,
- heading: Option<u8>,
-}
-
-struct FcRun<T> {
- fc_start: u32,
- fc_end: u32,
- props: T,
-}
-
-#[derive(Default)]
-struct Formatting {
- char_runs: Vec<FcRun<CharProps>>,
- para_runs: Vec<FcRun<ParaProps>>,
-}
-
-impl Formatting {
- fn parse(word_doc: &[u8], table: &[u8], fib: &Fib) -> Self {
- Self {
- char_runs: parse_bin_table(
- word_doc,
- table,
- fib.fc_plcf_bte_chpx,
- fib.lcb_plcf_bte_chpx,
- parse_chpx_fkp,
- ),
- para_runs: parse_bin_table(
- word_doc,
- table,
- fib.fc_plcf_bte_papx,
- fib.lcb_plcf_bte_papx,
- parse_papx_fkp,
- ),
- }
- }
-}
-
-/// Walks a bin table: a PLC of page numbers of FKPs in the WordDocument
-/// stream.
-fn parse_bin_table<T>(
- word_doc: &[u8],
- table: &[u8],
- fc: usize,
- lcb: usize,
- parse_fkp: impl Fn(&[u8], &mut Vec<FcRun<T>>),
-) -> Vec<FcRun<T>> {
- let mut runs = Vec::new();
- let Some(end) = fc.checked_add(lcb) else {
- return runs;
- };
- let Some(plc) = table.get(fc..end) else {
- return runs;
- };
- if lcb < 12 || (lcb - 4) % 8 != 0 {
- return runs;
- }
- let n = (lcb - 4) / 8;
- for k in 0..n {
- let pn = read_u32(plc, 4 * (n + 1) + 4 * k) as usize;
- let Some(page_start) = pn.checked_mul(FKP_PAGE_SIZE) else {
- continue;
- };
- if let Some(page) = word_doc.get(page_start..page_start + FKP_PAGE_SIZE) {
- parse_fkp(page, &mut runs);
- }
- }
- runs.sort_by_key(|run| run.fc_start);
- runs
-}
-
-fn parse_chpx_fkp(page: &[u8], runs: &mut Vec<FcRun<CharProps>>) {
- let crun = page[FKP_PAGE_SIZE - 1] as usize;
- if crun == 0 || 4 * (crun + 1) + crun >= FKP_PAGE_SIZE {
- return;
- }
- for i in 0..crun {
- let fc_start = read_u32(page, 4 * i);
- let fc_end = read_u32(page, 4 * (i + 1));
- if fc_end <= fc_start {
- continue;
- }
- // rgb entry: word offset of the Chpx within the page; 0 means default
- let rgb = page[4 * (crun + 1) + i] as usize;
- let mut props = CharProps::default();
- if rgb != 0 {
- let offset = rgb * 2;
- if let Some(&cb) = page.get(offset) {
- if let Some(grpprl) = page.get(offset + 1..offset + 1 + cb as usize) {
- props = char_props_from_grpprl(grpprl);
- }
- }
- }
- runs.push(FcRun {
- fc_start,
- fc_end,
- props,
- });
- }
-}
-
-fn parse_papx_fkp(page: &[u8], runs: &mut Vec<FcRun<ParaProps>>) {
- let cpara = page[FKP_PAGE_SIZE - 1] as usize;
- if cpara == 0 || 4 * (cpara + 1) + 13 * cpara >= FKP_PAGE_SIZE {
- return;
- }
- for i in 0..cpara {
- let fc_start = read_u32(page, 4 * i);
- let fc_end = read_u32(page, 4 * (i + 1));
- if fc_end <= fc_start {
- continue;
- }
- // BxPap: first byte is the word offset of the PapxInFkp; 0 means default
- let bx = page[4 * (cpara + 1) + 13 * i] as usize;
- let mut props = ParaProps::default();
- if bx != 0 {
- let offset = bx * 2;
- // PapxInFkp: grpprl length is cb*2 - 1, or cb2*2 when cb is 0
- if let Some(&cb) = page.get(offset) {
- let (start, len) = if cb == 0 {
- match page.get(offset + 1) {
- Some(&cb2) => (offset + 2, cb2 as usize * 2),
- None => (offset, 0),
- }
- } else {
- (offset + 1, (cb as usize) * 2 - 1)
- };
- if let Some(grpprl) = page.get(start..start + len) {
- props = para_props_from_grpprl(grpprl);
- }
- }
- }
- runs.push(FcRun {
- fc_start,
- fc_end,
- props,
- });
- }
-}
-
-fn char_props_from_grpprl(grpprl: &[u8]) -> CharProps {
- let mut props = CharProps::default();
- walk_sprms(grpprl, |sprm, operand| {
- // toggle operand: 1 = on, 0x81 = invert style (assume on)
- let on = matches!(operand.first(), Some(1) | Some(0x81));
- match sprm {
- 0x0835 => props.bold = on,
- 0x0836 => props.italic = on,
- 0x0837 => props.strike = on,
- _ => {}
- }
- });
- props
-}
-
-fn para_props_from_grpprl(grpprl: &[u8]) -> ParaProps {
- let mut props = ParaProps::default();
- if grpprl.len() < 2 {
- return props;
- }
- props.heading = heading_from_istd(read_u16(grpprl, 0));
- walk_sprms(&grpprl[2..], |sprm, operand| match sprm {
- // sprmPFInTable, sprmPFTtp, sprmPOutLvl, sprmPIstd
- 0x2416 => props.in_table = operand.first().is_some_and(|&v| v != 0),
- 0x2417 => props.row_end = operand.first().is_some_and(|&v| v != 0),
- 0x2640 => {
- if let Some(&level) = operand.first() {
- if level < 9 {
- props.heading = Some((level + 1).min(6));
- }
- }
- }
- 0x4600 => {
- if operand.len() >= 2 {
- if let Some(level) = heading_from_istd(read_u16(operand, 0)) {
- props.heading = Some(level);
- }
- }
- }
- _ => {}
- });
- props
-}
-
-/// Built-in style indexes 1..=9 are "heading 1".."heading 9".
-fn heading_from_istd(istd: u16) -> Option<u8> {
- if (1..=9).contains(&istd) {
- Some((istd as u8).min(6))
- } else {
- None
- }
-}
-
-/// Visits each sprm in a grpprl; operand size comes from the top 3 bits.
-fn walk_sprms(grpprl: &[u8], mut visit: impl FnMut(u16, &[u8])) {
- let mut i = 0;
- while i + 2 <= grpprl.len() {
- let sprm = read_u16(grpprl, i);
- i += 2;
- let size = match sprm >> 13 {
- 0 | 1 => 1,
- 2 | 4 | 5 => 2,
- 7 => 3,
- 3 => 4,
- _ => {
- // variable size: length byte first (u16 + 1 for sprmTDefTable)
- if sprm == 0xD608 {
- if i + 2 > grpprl.len() {
- return;
- }
- read_u16(grpprl, i) as usize + 1
- } else {
- match grpprl.get(i) {
- Some(&cb) => cb as usize + 1,
- None => return,
- }
- }
- }
- };
- let Some(operand) = grpprl.get(i..i + size) else {
- return;
- };
- visit(sprm, operand);
- i += size;
- }
-}
-
-/// Props for the run containing `fc`, with a cursor for sequential lookups.
-fn run_lookup<T: Copy + Default>(runs: &[FcRun<T>], fc: u32, cursor: &mut usize) -> T {
- if runs.is_empty() {
- return T::default();
- }
- if *cursor >= runs.len() || fc < runs[*cursor].fc_start {
- *cursor = runs.partition_point(|run| run.fc_end <= fc);
- } else {
- while *cursor < runs.len() && runs[*cursor].fc_end <= fc {
- *cursor += 1;
- }
- }
- match runs.get(*cursor) {
- Some(run) if fc >= run.fc_start && fc < run.fc_end => run.props,
- _ => T::default(),
- }
-}
-
-fn build_blocks(segments: &[TextSegment], formatting: &Formatting) -> Vec<Block> {
- let mut builder = BlockBuilder {
- formatting,
- blocks: Vec::new(),
- inlines: Vec::new(),
- run_text: String::new(),
- run_props: CharProps::default(),
- char_cursor: 0,
- para_cursor: 0,
- fields: Vec::new(),
- table_rows: Vec::new(),
- row_cells: Vec::new(),
- cell_blocks: Vec::new(),
- };
-
- for segment in segments {
- let mut fc = segment.fc_start;
- for ch in segment.text.chars() {
- builder.handle_char(ch, fc);
- fc = fc.saturating_add(match segment.bytes_per_char {
- 1 => 1,
- _ => 2 * ch.len_utf16() as u32,
- });
- }
- }
- builder.finish()
-}
-
-struct FieldFrame {
- in_instruction: bool,
- instruction: String,
- result_inline_start: usize,
-}
-
-struct BlockBuilder<'a> {
- formatting: &'a Formatting,
- blocks: Vec<Block>,
- inlines: Vec<Inline>,
- run_text: String,
- run_props: CharProps,
- char_cursor: usize,
- para_cursor: usize,
- fields: Vec<FieldFrame>,
- table_rows: Vec<TableRow>,
- row_cells: Vec<TableCell>,
- cell_blocks: Vec<Block>,
-}
-
-impl BlockBuilder<'_> {
- fn handle_char(&mut self, ch: char, fc: u32) {
- match ch {
- '\u{13}' => {
- self.fields.push(FieldFrame {
- in_instruction: true,
- instruction: String::new(),
- result_inline_start: usize::MAX,
- });
- return;
- }
- '\u{14}' => {
- self.flush_run();
- let start = self.inlines.len();
- if let Some(frame) = self.fields.last_mut() {
- frame.in_instruction = false;
- frame.result_inline_start = start;
- }
- return;
- }
- '\u{15}' => {
- self.flush_run();
- if let Some(frame) = self.fields.pop() {
- self.close_field(frame);
- }
- return;
- }
- _ => {}
- }
-
- if let Some(frame) = self.fields.iter_mut().rev().find(|f| f.in_instruction) {
- if frame.instruction.len() < MAX_FIELD_INSTRUCTION_CHARS {
- frame.instruction.push(ch);
- }
- return;
- }
-
- match ch {
- '\r' | '\n' | '\u{c}' | '\u{e}' => {
- let props = self.para_props(fc);
- self.end_paragraph(props);
- }
- '\u{7}' => {
- let props = self.para_props(fc);
- if props.row_end {
- self.end_row();
- } else if props.in_table {
- self.end_cell(props);
- } else {
- // no table context: behave like a paragraph mark
- self.end_paragraph(props);
- }
- }
- '\u{b}' => {
- self.flush_run();
- if !self.inlines.is_empty() {
- self.inlines.push(Inline::LineBreak);
- }
- }
- '\t' => self.push_char('\t', fc),
- '\u{1e}' => self.push_char('-', fc),
- c if c.is_control() || c == '\u{1f}' => {}
- c => self.push_char(c, fc),
- }
- }
-
- fn push_char(&mut self, ch: char, fc: u32) {
- let props = run_lookup(&self.formatting.char_runs, fc, &mut self.char_cursor);
- if props != self.run_props {
- self.flush_run();
- self.run_props = props;
- }
- self.run_text.push(ch);
- }
-
- fn para_props(&mut self, fc: u32) -> ParaProps {
- run_lookup(&self.formatting.para_runs, fc, &mut self.para_cursor)
- }
-
- fn flush_run(&mut self) {
- if self.run_text.is_empty() {
- return;
- }
- let mut node = Inline::Text(std::mem::take(&mut self.run_text));
- if self.run_props.strike {
- node = Inline::Del(vec![node]);
- }
- if self.run_props.italic {
- node = Inline::Em(vec![node]);
- }
- if self.run_props.bold {
- node = Inline::Strong(vec![node]);
- }
- self.inlines.push(node);
- }
-
- fn close_field(&mut self, frame: FieldFrame) {
- let Some(href) = hyperlink_target(&frame.instruction) else {
- return;
- };
- let start = frame.result_inline_start.min(self.inlines.len());
- let children = self.inlines.split_off(start);
- if has_visible_content(&children) {
- self.inlines.push(Inline::Link { href, children });
- } else {
- self.inlines.extend(children);
- }
- }
-
- /// Takes the pending inlines as a paragraph, or `None` when nothing
- /// visible accumulated.
- fn take_paragraph(&mut self, props: ParaProps) -> Option<Block> {
- self.flush_run();
- // links cannot span paragraphs
- for frame in &mut self.fields {
- frame.result_inline_start = 0;
- }
- while matches!(self.inlines.last(), Some(Inline::LineBreak)) {
- self.inlines.pop();
- }
- if !has_visible_content(&self.inlines) {
- self.inlines.clear();
- return None;
- }
- if let Some(Inline::Text(first)) = self.inlines.first_mut() {
- *first = first.trim_start().to_string();
- }
- if let Some(Inline::Text(last)) = self.inlines.last_mut() {
- *last = last.trim_end().to_string();
- }
- let kind = match props.heading {
- Some(level) => ParagraphKind::Heading(level),
- None => ParagraphKind::Normal,
- };
- Some(Block::Paragraph(Paragraph {
- kind,
- inlines: std::mem::take(&mut self.inlines),
- }))
- }
-
- fn end_paragraph(&mut self, props: ParaProps) {
- let paragraph = self.take_paragraph(props);
- if props.in_table {
- if let Some(paragraph) = paragraph {
- self.cell_blocks.push(paragraph);
- }
- } else {
- self.flush_table();
- if let Some(paragraph) = paragraph {
- self.blocks.push(paragraph);
- }
- }
- }
-
- fn end_cell(&mut self, props: ParaProps) {
- if let Some(paragraph) = self.take_paragraph(props) {
- self.cell_blocks.push(paragraph);
- }
- self.row_cells.push(TableCell {
- blocks: std::mem::take(&mut self.cell_blocks),
- colspan: NonZeroU32::MIN,
- rowspan: NonZeroU32::MIN,
- });
- }
-
- /// Row terminator: the TTP mark's own paragraph carries no cell content.
- fn end_row(&mut self) {
- self.flush_run();
- self.inlines.clear();
- if !self.cell_blocks.is_empty() {
- self.row_cells.push(TableCell {
- blocks: std::mem::take(&mut self.cell_blocks),
- colspan: NonZeroU32::MIN,
- rowspan: NonZeroU32::MIN,
- });
- }
- if !self.row_cells.is_empty() {
- self.table_rows.push(TableRow {
- cells: std::mem::take(&mut self.row_cells),
- kind: TableRowKind::Body,
- });
- }
- }
-
- fn flush_table(&mut self) {
- if !self.cell_blocks.is_empty() || !self.row_cells.is_empty() {
- self.end_row();
- }
- if !self.table_rows.is_empty() {
- self.blocks.push(Block::Table(Table {
- rows: std::mem::take(&mut self.table_rows),
- }));
- }
- }
-
- fn finish(mut self) -> Vec<Block> {
- self.end_paragraph(ParaProps::default());
- self.flush_table();
- self.blocks
- }
-}
-
-/// Target of a HYPERLINK field instruction.
-fn hyperlink_target(instruction: &str) -> Option<String> {
- let rest = instruction.trim_start();
- if !rest
- .get(..9)
- .is_some_and(|p| p.eq_ignore_ascii_case("HYPERLINK"))
- {
- return None;
- }
- let mut tokens = field_tokens(&rest[9..]);
- let mut anchor = false;
- while let Some(token) = tokens.next() {
- match token.as_str() {
- "\\l" => anchor = true,
- // switches whose argument is not the target
- "\\o" | "\\t" => {
- tokens.next();
- }
- t if t.starts_with('\\') => {}
- t => {
- return Some(if anchor {
- format!("#{t}")
- } else {
- t.to_string()
- });
- }
- }
- }
- None
-}
-
-/// Whitespace-separated tokens; quoted strings stay together, unquoted.
-fn field_tokens(input: &str) -> impl Iterator<Item = String> + '_ {
- let mut chars = input.chars().peekable();
- std::iter::from_fn(move || {
- while chars.peek().is_some_and(|c| c.is_whitespace()) {
- chars.next();
- }
- match chars.peek() {
- None => None,
- Some('"') => {
- chars.next();
- let mut token = String::new();
- for c in chars.by_ref() {
- if c == '"' {
- break;
- }
- token.push(c);
- }
- Some(token)
- }
- Some(_) => {
- let mut token = String::new();
- while let Some(&c) = chars.peek() {
- if c.is_whitespace() {
- break;
- }
- token.push(c);
- chars.next();
- }
- Some(token)
- }
- }
- })
-}
-
-fn read_u16(data: &[u8], offset: usize) -> u16 {
- u16::from_le_bytes([data[offset], data[offset + 1]])
-}
-
-fn read_u32(data: &[u8], offset: usize) -> u32 {
- u32::from_le_bytes([
- data[offset],
- data[offset + 1],
- data[offset + 2],
- data[offset + 3],
- ])
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use std::io::Write;
-
- enum TestPiece {
- Compressed(Vec<u8>),
- Unicode(&'static str),
- }
-
- /// Contiguous formatting runs: (fc_start, fc_end, grpprl). An empty grpprl
- /// becomes a default (offset 0) FKP entry.
- #[derive(Default)]
- struct TestFmt {
- char_runs: Vec<(u32, u32, Vec<u8>)>,
- para_runs: Vec<(u32, u32, Vec<u8>)>,
- }
-
- fn utf16_bytes(text: &str) -> Vec<u8> {
- text.encode_utf16().flat_map(u16::to_le_bytes).collect()
- }
-
- const TEXT_START: usize = 0x200;
-
- /// FC of the i-th UTF-16 unit of a single Unicode test piece.
- fn ufc(i: usize) -> u32 {
- (TEXT_START + 2 * i) as u32
- }
-
- fn sprm1(code: u16, value: u8) -> Vec<u8> {
- let mut out = code.to_le_bytes().to_vec();
- out.push(value);
- out
- }
-
- fn papx(istd: u16, sprms: &[Vec<u8>]) -> Vec<u8> {
- let mut out = istd.to_le_bytes().to_vec();
- for sprm in sprms {
- out.extend_from_slice(sprm);
- }
- out
- }
-
- fn build_chpx_fkp(runs: &[(u32, u32, Vec<u8>)]) -> Vec<u8> {
- let mut page = vec![0u8; FKP_PAGE_SIZE];
- let crun = runs.len();
- for (i, (fc_start, fc_end, _)) in runs.iter().enumerate() {
- page[4 * i..4 * i + 4].copy_from_slice(&fc_start.to_le_bytes());
- page[4 * (i + 1)..4 * (i + 1) + 4].copy_from_slice(&fc_end.to_le_bytes());
- }
- let mut blob_end = FKP_PAGE_SIZE - 1;
- for (i, (_, _, grpprl)) in runs.iter().enumerate() {
- if grpprl.is_empty() {
- continue;
- }
- let blob_len = 1 + grpprl.len();
- let mut pos = blob_end - blob_len;
- pos &= !1;
- page[pos] = grpprl.len() as u8;
- page[pos + 1..pos + 1 + grpprl.len()].copy_from_slice(grpprl);
- page[4 * (crun + 1) + i] = (pos / 2) as u8;
- blob_end = pos;
- }
- page[FKP_PAGE_SIZE - 1] = crun as u8;
- page
- }
-
- fn build_papx_fkp(runs: &[(u32, u32, Vec<u8>)]) -> Vec<u8> {
- let mut page = vec![0u8; FKP_PAGE_SIZE];
- let cpara = runs.len();
- for (i, (fc_start, fc_end, _)) in runs.iter().enumerate() {
- page[4 * i..4 * i + 4].copy_from_slice(&fc_start.to_le_bytes());
- page[4 * (i + 1)..4 * (i + 1) + 4].copy_from_slice(&fc_end.to_le_bytes());
- }
- let mut blob_end = FKP_PAGE_SIZE - 1;
- for (i, (_, _, grpprl)) in runs.iter().enumerate() {
- if grpprl.is_empty() {
- continue;
- }
- let blob = if grpprl.len() % 2 == 1 {
- let mut b = vec![((grpprl.len() + 1) / 2) as u8];
- b.extend_from_slice(grpprl);
- b
- } else {
- let mut b = vec![0, (grpprl.len() / 2) as u8];
- b.extend_from_slice(grpprl);
- b
- };
- let mut pos = blob_end - blob.len();
- pos &= !1;
- page[pos..pos + blob.len()].copy_from_slice(&blob);
- page[4 * (cpara + 1) + 13 * i] = (pos / 2) as u8;
- blob_end = pos;
- }
- page[FKP_PAGE_SIZE - 1] = cpara as u8;
- page
- }
-
- fn build_word_doc_streams(pieces: &[TestPiece], lid: u16) -> (Vec<u8>, Vec<u8>) {
- build_word_doc_streams_fmt(pieces, lid, &TestFmt::default())
- }
-
- fn build_word_doc_streams_fmt(
- pieces: &[TestPiece],
- lid: u16,
- fmt: &TestFmt,
- ) -> (Vec<u8>, Vec<u8>) {
- let mut word_doc = vec![0u8; TEXT_START];
- let mut cps = vec![0u32];
- let mut pcds = Vec::new();
-
- for piece in pieces {
- let offset = word_doc.len();
- let (count, fc) = match piece {
- TestPiece::Compressed(bytes) => {
- word_doc.extend_from_slice(bytes);
- (bytes.len() as u32, (offset as u32 * 2) | 0x4000_0000)
- }
- TestPiece::Unicode(text) => {
- let bytes = utf16_bytes(text);
- word_doc.extend_from_slice(&bytes);
- (text.encode_utf16().count() as u32, offset as u32)
- }
- };
- cps.push(cps.last().unwrap() + count);
- pcds.push(fc);
- }
-
- let ccp_text = *cps.last().unwrap();
- let mut plc = Vec::new();
- for cp in &cps {
- plc.extend_from_slice(&cp.to_le_bytes());
- }
- for fc in &pcds {
- plc.extend_from_slice(&0u16.to_le_bytes());
- plc.extend_from_slice(&fc.to_le_bytes());
- plc.extend_from_slice(&0u16.to_le_bytes());
- }
- let mut table = vec![0x02];
- table.extend_from_slice(&(plc.len() as u32).to_le_bytes());
- table.extend_from_slice(&plc);
- let lcb_clx = table.len() as u32;
-
- let append_fkp =
- |word_doc: &mut Vec<u8>, table: &mut Vec<u8>, page: Vec<u8>, runs: &[(u32, u32, Vec<u8>)]| {
- while word_doc.len() % FKP_PAGE_SIZE != 0 {
- word_doc.push(0);
- }
- let pn = (word_doc.len() / FKP_PAGE_SIZE) as u32;
- word_doc.extend_from_slice(&page);
- let bin_table_offset = table.len() as u32;
- table.extend_from_slice(&runs[0].0.to_le_bytes());
- table.extend_from_slice(&runs.last().unwrap().1.to_le_bytes());
- table.extend_from_slice(&pn.to_le_bytes());
- (bin_table_offset, 12u32)
- };
-
- let chpx_bt = (!fmt.char_runs.is_empty()).then(|| {
- let page = build_chpx_fkp(&fmt.char_runs);
- append_fkp(&mut word_doc, &mut table, page, &fmt.char_runs)
- });
- let papx_bt = (!fmt.para_runs.is_empty()).then(|| {
- let page = build_papx_fkp(&fmt.para_runs);
- append_fkp(&mut word_doc, &mut table, page, &fmt.para_runs)
- });
-
- word_doc[0x00..0x02].copy_from_slice(&MAGIC_WORD_97.to_le_bytes());
- word_doc[0x02..0x04].copy_from_slice(&0x00C1u16.to_le_bytes());
- word_doc[0x06..0x08].copy_from_slice(&lid.to_le_bytes());
- word_doc[0x0A..0x0C].copy_from_slice(&0x0200u16.to_le_bytes());
- word_doc[0x4C..0x50].copy_from_slice(&ccp_text.to_le_bytes());
- word_doc[0x1A2..0x1A6].copy_from_slice(&0u32.to_le_bytes());
- word_doc[0x1A6..0x1AA].copy_from_slice(&lcb_clx.to_le_bytes());
- if let Some((fc, lcb)) = chpx_bt {
- word_doc[0xFA..0xFE].copy_from_slice(&fc.to_le_bytes());
- word_doc[0xFE..0x102].copy_from_slice(&lcb.to_le_bytes());
- }
- if let Some((fc, lcb)) = papx_bt {
- word_doc[0x102..0x106].copy_from_slice(&fc.to_le_bytes());
- word_doc[0x106..0x10A].copy_from_slice(&lcb.to_le_bytes());
- }
-
- (word_doc, table)
- }
-
- fn build_cfb(streams: &[(&str, &[u8])]) -> Vec<u8> {
- let mut comp = CompoundFile::create(Cursor::new(Vec::new())).unwrap();
- for (name, data) in streams {
- let mut stream = comp.create_stream(name).unwrap();
- stream.write_all(data).unwrap();
- }
- comp.into_inner().into_inner()
- }
-
- fn build_doc(pieces: &[TestPiece], lid: u16, summary: Option<&[u8]>) -> Vec<u8> {
- build_doc_fmt(pieces, lid, summary, &TestFmt::default())
- }
-
- fn build_doc_fmt(
- pieces: &[TestPiece],
- lid: u16,
- summary: Option<&[u8]>,
- fmt: &TestFmt,
- ) -> Vec<u8> {
- let (word_doc, table) = build_word_doc_streams_fmt(pieces, lid, fmt);
- let mut streams: Vec<(&str, &[u8])> = vec![("WordDocument", &word_doc), ("1Table", &table)];
- if let Some(summary) = summary {
- streams.push(("\x05SummaryInformation", summary));
- }
- build_cfb(&streams)
- }
-
- fn build_summary_information(codepage: u16, title: &[u8], author: &[u8]) -> Vec<u8> {
- let mut props: Vec<(u32, Vec<u8>)> = Vec::new();
- let mut codepage_value = 0x0002u32.to_le_bytes().to_vec();
- codepage_value.extend_from_slice(&(codepage as u32).to_le_bytes());
- props.push((0x0001, codepage_value));
- for (pid, text) in [(0x0002u32, title), (0x0004u32, author)] {
- let mut value = 0x001Eu32.to_le_bytes().to_vec();
- value.extend_from_slice(&((text.len() + 1) as u32).to_le_bytes());
- value.extend_from_slice(text);
- value.push(0);
- props.push((pid, value));
- }
-
- let section_start = 0x30usize;
- let header_size = 8 + props.len() * 8;
- let mut entries = Vec::new();
- let mut values = Vec::new();
- for (pid, value) in &props {
- entries.extend_from_slice(&pid.to_le_bytes());
- entries.extend_from_slice(&((header_size + values.len()) as u32).to_le_bytes());
- values.extend_from_slice(value);
- while values.len() % 4 != 0 {
- values.push(0);
- }
- }
-
- let mut data = vec![0u8; section_start];
- data[0x00..0x02].copy_from_slice(&0xFFFEu16.to_le_bytes());
- data[0x18..0x1C].copy_from_slice(&1u32.to_le_bytes());
- data[0x2C..0x30].copy_from_slice(&(section_start as u32).to_le_bytes());
- data.extend_from_slice(&((header_size + values.len()) as u32).to_le_bytes());
- data.extend_from_slice(&(props.len() as u32).to_le_bytes());
- data.extend_from_slice(&entries);
- data.extend_from_slice(&values);
- data
- }
-
- /// Readable markup for inline assertions, e.g. `a <b>x</b> <a href=u>y</a>`.
- fn inline_markup(inlines: &[Inline]) -> String {
- inlines
- .iter()
- .map(|inline| match inline {
- Inline::Text(t) => t.clone(),
- Inline::LineBreak => "\n".to_string(),
- Inline::Strong(c) => format!("<b>{}</b>", inline_markup(c)),
- Inline::Em(c) => format!("<i>{}</i>", inline_markup(c)),
- Inline::Del(c) => format!("<del>{}</del>", inline_markup(c)),
- Inline::Link { href, children } => {
- format!("<a href={}>{}</a>", href, inline_markup(children))
- }
- _ => String::new(),
- })
- .collect()
- }
-
- fn inline_text(inlines: &[Inline]) -> String {
- inlines
- .iter()
- .map(|inline| match inline {
- Inline::Text(t) => t.clone(),
- Inline::LineBreak => "\n".to_string(),
- Inline::Link { children, .. } => inline_text(children),
- Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {
- inline_text(c)
- }
- _ => String::new(),
- })
- .collect()
- }
-
- fn paragraph_texts(document: &Document) -> Vec<String> {
- document
- .blocks
- .iter()
- .map(|block| match block {
- Block::Paragraph(p) => inline_text(&p.inlines),
- _ => String::new(),
- })
- .collect()
- }
-
- fn parse(data: &[u8]) -> Document {
- DocProvider::new().parse_buffer(data).unwrap()
- }
-
- fn cp1251(text: &str) -> Vec<u8> {
- encoding_rs::WINDOWS_1251.encode(text).0.into_owned()
- }
-
- #[test]
- fn unicode_pieces() {
- let data = build_doc(
- &[TestPiece::Unicode("Указ Президента України\rДодаток\r")],
- 0x0409,
- None,
- );
- assert_eq!(
- paragraph_texts(&parse(&data)),
- vec!["Указ Президента України", "Додаток"]
- );
- }
-
- #[test]
- fn compressed_cp1251_via_summary_codepage() {
- let summary = build_summary_information(1251, &cp1251("Указ"), &cp1251("Автор"));
- let data = build_doc(
- &[TestPiece::Compressed(cp1251("ПРЕЗИДЕНТ УКРАЇНИ\r"))],
- 0x0409,
- Some(&summary),
- );
- let document = parse(&data);
- assert_eq!(paragraph_texts(&document), vec!["ПРЕЗИДЕНТ УКРАЇНИ"]);
- assert_eq!(document.metadata.title.as_deref(), Some("Указ"));
- assert_eq!(document.metadata.author.as_deref(), Some("Автор"));
- }
-
- #[test]
- fn compressed_cp1251_via_lid() {
- let data = build_doc(
- &[TestPiece::Compressed(cp1251("Слава Україні\r"))],
- 0x0422,
- None,
- );
- assert_eq!(paragraph_texts(&parse(&data)), vec!["Слава Україні"]);
- }
-
- #[test]
- fn compressed_defaults_to_cp1252() {
- let data = build_doc(
- &[TestPiece::Compressed(b"Caf\xE9 \x93quoted\x94\r".to_vec())],
- 0x0409,
- None,
- );
- assert_eq!(
- paragraph_texts(&parse(&data)),
- vec!["Café \u{201C}quoted\u{201D}"]
- );
- }
-
- #[test]
- fn mixed_pieces_concatenate_in_cp_order() {
- let data = build_doc(
- &[
- TestPiece::Unicode("Перша частина "),
- TestPiece::Compressed(b"and ASCII tail\r".to_vec()),
- ],
- 0x0419,
- None,
- );
- assert_eq!(
- paragraph_texts(&parse(&data)),
- vec!["Перша частина and ASCII tail"]
- );
- }
-
- #[test]
- fn field_instructions_are_stripped() {
- let data = build_doc(
- &[TestPiece::Unicode(
- "See \u{13} HYPERLINK \"https://example.com\" \u{14}the site\u{15} now\r",
- )],
- 0x0409,
- None,
- );
- assert_eq!(paragraph_texts(&parse(&data)), vec!["See the site now"]);
- }
-
- #[test]
- fn hyperlink_fields_become_links() {
- let data = build_doc(
- &[TestPiece::Unicode(
- "See \u{13} HYPERLINK \"https://example.com\" \u{14}the site\u{15} now\r",
- )],
- 0x0409,
- None,
- );
- let document = parse(&data);
- let Block::Paragraph(p) = &document.blocks[0] else {
- panic!("expected paragraph");
- };
- assert_eq!(
- inline_markup(&p.inlines),
- "See <a href=https://example.com>the site</a> now"
- );
- }
-
- #[test]
- fn anchor_hyperlink_targets_bookmark() {
- let data = build_doc(
- &[TestPiece::Unicode(
- "Go \u{13} HYPERLINK \\l \"section1\" \u{14}there\u{15}\r",
- )],
- 0x0409,
- None,
- );
- let document = parse(&data);
- let Block::Paragraph(p) = &document.blocks[0] else {
- panic!("expected paragraph");
- };
- assert_eq!(inline_markup(&p.inlines), "Go <a href=#section1>there</a>");
- }
-
- #[test]
- fn non_hyperlink_fields_keep_result_text_only() {
- let data = build_doc(
- &[TestPiece::Unicode("Page \u{13} PAGE \u{14}7\u{15} of 9\r")],
- 0x0409,
- None,
- );
- let document = parse(&data);
- let Block::Paragraph(p) = &document.blocks[0] else {
- panic!("expected paragraph");
- };
- assert_eq!(inline_markup(&p.inlines), "Page 7 of 9");
- }
-
- #[test]
- fn cell_marks_and_line_breaks() {
- let data = build_doc(
- &[TestPiece::Unicode(
- "cell one\u{7}cell two\u{7}line\u{b}break\r",
- )],
- 0x0409,
- None,
- );
- assert_eq!(
- paragraph_texts(&parse(&data)),
- vec!["cell one", "cell two", "line\nbreak"]
- );
- }
-
- #[test]
- fn bold_italic_strike_runs() {
- let text = "plain bold and style\r";
- let fmt = TestFmt {
- char_runs: vec![
- (ufc(0), ufc(6), vec![]),
- (ufc(6), ufc(10), sprm1(0x0835, 1)),
- (ufc(10), ufc(15), vec![]),
- (
- ufc(15),
- ufc(20),
- [sprm1(0x0836, 1), sprm1(0x0837, 1)].concat(),
- ),
- (ufc(20), ufc(21), vec![]),
- ],
- ..TestFmt::default()
- };
- let data = build_doc_fmt(&[TestPiece::Unicode(text)], 0x0409, None, &fmt);
- let document = parse(&data);
- let Block::Paragraph(p) = &document.blocks[0] else {
- panic!("expected paragraph");
- };
- assert_eq!(
- inline_markup(&p.inlines),
- "plain <b>bold</b> and <i><del>style</del></i>"
- );
- }
-
- #[test]
- fn headings_from_istd_and_outline_level() {
- let text = "Title\rSection\rBody\r";
- let fmt = TestFmt {
- para_runs: vec![
- (ufc(0), ufc(6), papx(1, &[])),
- (ufc(6), ufc(14), papx(0, &[sprm1(0x2640, 1)])),
- (ufc(14), ufc(19), papx(0, &[])),
- ],
- ..TestFmt::default()
- };
- let data = build_doc_fmt(&[TestPiece::Unicode(text)], 0x0409, None, &fmt);
- let document = parse(&data);
- let kinds: Vec<ParagraphKind> = document
- .blocks
- .iter()
- .filter_map(|b| match b {
- Block::Paragraph(p) => Some(p.kind),
- _ => None,
- })
- .collect();
- assert_eq!(
- kinds,
- vec![
- ParagraphKind::Heading(1),
- ParagraphKind::Heading(2),
- ParagraphKind::Normal
- ]
- );
- }
-
- #[test]
- fn table_from_cell_and_row_marks() {
- let text = "A1\u{7}B1\u{7}\u{7}A2\u{7}B2\u{7}\u{7}after\r";
- let in_table = papx(0, &[sprm1(0x2416, 1)]);
- let row_end = papx(0, &[sprm1(0x2416, 1), sprm1(0x2417, 1)]);
- let fmt = TestFmt {
- para_runs: vec![
- (ufc(0), ufc(3), in_table.clone()),
- (ufc(3), ufc(6), in_table.clone()),
- (ufc(6), ufc(7), row_end.clone()),
- (ufc(7), ufc(10), in_table.clone()),
- (ufc(10), ufc(13), in_table),
- (ufc(13), ufc(14), row_end),
- (ufc(14), ufc(20), papx(0, &[])),
- ],
- ..TestFmt::default()
- };
- let data = build_doc_fmt(&[TestPiece::Unicode(text)], 0x0409, None, &fmt);
- let document = parse(&data);
-
- let Block::Table(table) = &document.blocks[0] else {
- panic!("expected table, got {:?}", document.blocks[0]);
- };
- let cells: Vec<Vec<String>> = table
- .rows
- .iter()
- .map(|row| {
- row
- .cells
- .iter()
- .map(|cell| {
- cell
- .blocks
- .iter()
- .map(|b| match b {
- Block::Paragraph(p) => inline_text(&p.inlines),
- _ => String::new(),
- })
- .collect::<String>()
- })
- .collect()
- })
- .collect();
- assert_eq!(cells, vec![vec!["A1", "B1"], vec!["A2", "B2"]]);
- assert_eq!(paragraph_texts(&parse(&data)).last().unwrap(), "after");
- }
-
- #[test]
- fn legacy_word_95_contiguous_text() {
- let text_bytes = cp1251("Стаття перша.\rСтаття друга.\r");
- let fc_min = 0x200usize;
- let mut word_doc = vec![0u8; fc_min];
- word_doc.extend_from_slice(&text_bytes);
- word_doc[0x00..0x02].copy_from_slice(&MAGIC_WORD_6_95.to_le_bytes());
- word_doc[0x06..0x08].copy_from_slice(&0x0419u16.to_le_bytes());
- word_doc[0x18..0x1C].copy_from_slice(&(fc_min as u32).to_le_bytes());
- word_doc[0x1C..0x20].copy_from_slice(&((fc_min + text_bytes.len()) as u32).to_le_bytes());
- let data = build_cfb(&[("WordDocument", &word_doc)]);
- assert_eq!(
- paragraph_texts(&parse(&data)),
- vec!["Стаття перша.", "Стаття друга."]
- );
- }
-
- #[test]
- fn malformed_piece_table_output_is_capped() {
- // 200 pieces of 100k chars all mapped to the same 100k file bytes; the
- // claimed 20M chars must clamp to MAX_TEXT_CHARS
- let text_len = 100_000u32;
- let num_pieces = 200u32;
- let mut word_doc = vec![0u8; TEXT_START];
- word_doc.extend_from_slice(&vec![b'A'; text_len as usize]);
-
- let mut plc = Vec::new();
- for k in 0..=num_pieces {
- plc.extend_from_slice(&(k * text_len).to_le_bytes());
- }
- let fc = (TEXT_START as u32 * 2) | 0x4000_0000;
- for _ in 0..num_pieces {
- plc.extend_from_slice(&0u16.to_le_bytes());
- plc.extend_from_slice(&fc.to_le_bytes());
- plc.extend_from_slice(&0u16.to_le_bytes());
- }
- let mut table = vec![0x02];
- table.extend_from_slice(&(plc.len() as u32).to_le_bytes());
- table.extend_from_slice(&plc);
-
- word_doc[0x00..0x02].copy_from_slice(&MAGIC_WORD_97.to_le_bytes());
- word_doc[0x0A..0x0C].copy_from_slice(&0x0200u16.to_le_bytes());
- word_doc[0x4C..0x50].copy_from_slice(&(num_pieces * text_len).to_le_bytes());
- word_doc[0x1A6..0x1AA].copy_from_slice(&(table.len() as u32).to_le_bytes());
-
- let data = build_cfb(&[("WordDocument", &word_doc), ("1Table", &table)]);
- let total: usize = paragraph_texts(&parse(&data)).iter().map(|t| t.len()).sum();
- assert!(total > 0);
- assert!(total <= MAX_TEXT_CHARS);
- }
-
- #[test]
- fn encrypted_documents_error() {
- let (mut word_doc, table) = build_word_doc_streams(&[TestPiece::Unicode("secret\r")], 0x0409);
- word_doc[0x0A..0x0C].copy_from_slice(&0x0300u16.to_le_bytes());
- let data = build_cfb(&[("WordDocument", &word_doc), ("1Table", &table)]);
- let err = DocProvider::new().parse_buffer(&data).unwrap_err();
- assert!(err.to_string().contains("encrypted"));
- }
-
- #[test]
- fn corrupt_piece_table_falls_back_to_fc_range() {
- let (mut word_doc, _) =
- build_word_doc_streams(&[TestPiece::Unicode("good text here\r")], 0x0409);
- word_doc[0x0A..0x0C].copy_from_slice(&0x1200u16.to_le_bytes());
- let fc_mac = word_doc.len() as u32;
- word_doc[0x18..0x1C].copy_from_slice(&(TEXT_START as u32).to_le_bytes());
- word_doc[0x1C..0x20].copy_from_slice(&fc_mac.to_le_bytes());
- let garbage_table = vec![0xFFu8; 64];
- let data = build_cfb(&[("WordDocument", &word_doc), ("1Table", &garbage_table)]);
- assert_eq!(paragraph_texts(&parse(&data)), vec!["good text here"]);
- }
-}
diff --git a/apps/api/native/src/document/providers/docx.rs b/apps/api/native/src/document/providers/docx.rs
deleted file mode 100644
index 2e2b60c8b..000000000
--- a/apps/api/native/src/document/providers/docx.rs
+++ /dev/null
@@ -1,816 +0,0 @@
-//! DOCX (Office Open XML) provider.
-
-use crate::document::model::*;
-use crate::document::providers::DocumentProvider;
-use crate::document::xml::{attr, child, children, is_tag, read_zip_text, strip_bom};
-use chrono::{DateTime, Utc};
-use roxmltree::{Document as XmlDoc, Node};
-use std::collections::{BTreeSet, HashMap};
-use std::error::Error;
-use std::io::{Cursor, Read, Seek};
-use std::num::NonZeroU32;
-use zip::read::ZipArchive;
-
-pub struct DocxProvider;
-
-impl DocxProvider {
- pub fn new() -> Self {
- Self
- }
-}
-
-/// Everything block parsing needs besides the XML itself.
-#[derive(Clone, Copy)]
-struct Ctx<'a> {
- rels: &'a Relationships,
- styles: &'a StylesInfo,
- size_buckets: &'a HashMap<String, Vec<u32>>,
- numbering: &'a NumberingInfo,
-}
-
-impl DocumentProvider for DocxProvider {
- fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {
- let mut zip = ZipArchive::new(Cursor::new(data))?;
-
- let metadata = read_core_properties(&mut zip).unwrap_or_default();
- let styles = read_styles(&mut zip);
- let numbering = read_numbering(&mut zip);
- let rels = read_relationships(&mut zip, "word/_rels/document.xml.rels");
-
- let document_xml = read_zip_text(&mut zip, "word/document.xml")
- .ok_or("Missing word/document.xml in document")?;
- let xml = XmlDoc::parse(strip_bom(&document_xml))?;
-
- let size_buckets = compute_style_size_buckets(&xml, &styles);
- let ctx = Ctx {
- rels: &rels,
- styles: &styles,
- size_buckets: &size_buckets,
- numbering: &numbering,
- };
-
- let blocks = xml
- .descendants()
- .find(|n| is_tag(n, "body"))
- .map(|body| parse_blocks(&body, ctx))
- .unwrap_or_default();
-
- let mut notes = read_notes(&mut zip, NoteKind::Footnote, ctx);
- notes.extend(read_notes(&mut zip, NoteKind::Endnote, ctx));
- let comments = read_comments(&mut zip, ctx);
-
- Ok(Document {
- blocks,
- metadata,
- notes,
- comments,
- })
- }
-
- fn name(&self) -> &'static str {
- "docx"
- }
-}
-
-#[derive(Debug, Clone, Default)]
-struct Relationships {
- targets: HashMap<String, String>,
-}
-
-impl Relationships {
- fn get(&self, id: &str) -> Option<&str> {
- self.targets.get(id).map(|s| s.as_str())
- }
-}
-
-fn read_relationships<R: Read + Seek>(zip: &mut ZipArchive<R>, path: &str) -> Relationships {
- let mut rels = Relationships::default();
- let Some(text) = read_zip_text(zip, path) else {
- return rels;
- };
- let Ok(xml) = XmlDoc::parse(strip_bom(&text)) else {
- return rels;
- };
- for rel in xml.descendants().filter(|n| is_tag(n, "Relationship")) {
- if let (Some(id), Some(target)) = (attr(&rel, "Id"), attr(&rel, "Target")) {
- rels.targets.insert(id.to_string(), target.to_string());
- }
- }
- rels
-}
-
-fn read_core_properties<R: Read + Seek>(zip: &mut ZipArchive<R>) -> Option<DocumentMetadata> {
- let text = read_zip_text(zip, "docProps/core.xml")?;
- let xml = XmlDoc::parse(strip_bom(&text)).ok()?;
- let element_text = |local: &str| {
- xml
- .descendants()
- .find(|n| is_tag(n, local))
- .and_then(|n| n.text())
- };
-
- let mut meta = DocumentMetadata::default();
- if let Some(title) = element_text("title") {
- let trimmed = title.trim();
- if !trimmed.is_empty() {
- meta.title = Some(trimmed.to_string());
- }
- }
- if let Some(author) = element_text("creator") {
- let trimmed = author.trim();
- if !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("unknown") {
- meta.author = Some(trimmed.to_string());
- }
- }
- if let Some(created) = element_text("created") {
- if let Ok(dt) = DateTime::parse_from_rfc3339(created) {
- meta.created = Some(DateTime::<Utc>::from(dt));
- }
- }
- Some(meta)
-}
-
-#[derive(Debug, Default)]
-struct StylesInfo {
- outline_level_by_style_id: HashMap<String, u8>,
- name_by_style_id: HashMap<String, String>,
- default_size_by_style_id: HashMap<String, u32>,
-}
-
-fn read_styles<R: Read + Seek>(zip: &mut ZipArchive<R>) -> StylesInfo {
- let mut info = StylesInfo::default();
- let Some(text) = read_zip_text(zip, "word/styles.xml") else {
- return info;
- };
- let Ok(xml) = XmlDoc::parse(strip_bom(&text)) else {
- return info;
- };
-
- for style in xml.descendants().filter(|n| is_tag(n, "style")) {
- if attr(&style, "type") != Some("paragraph") {
- continue;
- }
- let Some(style_id) = attr(&style, "styleId") else {
- continue;
- };
-
- if let Some(name) = child(&style, "name").and_then(|n| attr(&n, "val")) {
- info
- .name_by_style_id
- .insert(style_id.to_string(), name.to_string());
- }
- if let Some(size) = child(&style, "rPr")
- .and_then(|rpr| child(&rpr, "sz"))
- .and_then(|n| attr(&n, "val"))
- .and_then(|v| v.parse::<u32>().ok())
- {
- info
- .default_size_by_style_id
- .insert(style_id.to_string(), size);
- }
- if let Some(level) = child(&style, "pPr")
- .and_then(|ppr| child(&ppr, "outlineLvl"))
- .and_then(|n| attr(&n, "val"))
- .and_then(|v| v.parse::<u8>().ok())
- {
- info
- .outline_level_by_style_id
- .insert(style_id.to_string(), (level + 1).min(6));
- }
- }
- info
-}
-
-enum StyleKind {
- Heading(u8),
- Blockquote,
-}
-
-/// Heading or quote semantics of a paragraph style, from the style's outline
-/// level when declared and name/id conventions otherwise.
-fn style_kind(style_id: &str, styles: &StylesInfo) -> Option<StyleKind> {
- if let Some(level) = styles.outline_level_by_style_id.get(style_id) {
- return Some(StyleKind::Heading(*level));
- }
-
- let name = styles.name_by_style_id.get(style_id);
- if let Some(name) = name {
- if let Some(level) = parse_heading_level(name) {
- return Some(StyleKind::Heading(level));
- }
- if name.to_ascii_lowercase().contains("quote") {
- return Some(StyleKind::Blockquote);
- }
- }
- if let Some(level) = parse_heading_level(style_id) {
- return Some(StyleKind::Heading(level));
- }
-
- let id_lower = style_id.to_ascii_lowercase();
- let name_lower = name.map(|s| s.to_ascii_lowercase()).unwrap_or_default();
- if name_lower.contains("title") || id_lower.contains("title") {
- Some(StyleKind::Heading(1))
- } else if name_lower.contains("heading") || id_lower.contains("heading") {
- Some(StyleKind::Heading(2))
- } else if name_lower.contains("quote") || id_lower.contains("quote") {
- Some(StyleKind::Blockquote)
- } else {
- None
- }
-}
-
-fn parse_heading_level(s: &str) -> Option<u8> {
- let lower = s.to_ascii_lowercase();
- let rest = &lower[lower.find("heading")? + "heading".len()..];
- let digits: String = rest
- .chars()
- .skip_while(|c| c.is_whitespace() || *c == '-')
- .take_while(|c| c.is_ascii_digit())
- .collect();
- match digits.parse::<u8>() {
- Ok(n) if n >= 1 => Some(n.min(6)),
- _ => None,
- }
-}
-
-fn paragraph_style_id<'a>(p: &Node<'a, 'a>) -> Option<&'a str> {
- child(p, "pPr")
- .and_then(|ppr| child(&ppr, "pStyle"))
- .and_then(|n| attr(&n, "val"))
-}
-
-fn paragraph_kind(p: &Node, ctx: Ctx) -> ParagraphKind {
- let Some(ppr) = child(p, "pPr") else {
- return ParagraphKind::Normal;
- };
-
- if let Some(level) = child(&ppr, "outlineLvl")
- .and_then(|n| attr(&n, "val"))
- .and_then(|v| v.parse::<u8>().ok())
- {
- return ParagraphKind::Heading((level + 1).min(6));
- }
-
- let Some(style_id) = paragraph_style_id(p) else {
- return ParagraphKind::Normal;
- };
- match style_kind(style_id, ctx.styles) {
- Some(StyleKind::Blockquote) => ParagraphKind::Blockquote,
- Some(StyleKind::Heading(mut level)) => {
- // Same style at distinct font sizes: demote the smaller sizes.
- if let (Some(buckets), Some(size)) = (
- ctx.size_buckets.get(style_id),
- paragraph_effective_size(p, ctx.styles, style_id),
- ) {
- if let Some(index) = buckets.iter().position(|&s| s == size) {
- level = (level + index as u8).min(6);
- }
- }
- ParagraphKind::Heading(level)
- }
- None => ParagraphKind::Normal,
- }
-}
-
-fn paragraph_effective_size(p: &Node, styles: &StylesInfo, style_id: &str) -> Option<u32> {
- let run_sizes = children(p, "r")
- .filter_map(|r| child(&r, "rPr"))
- .chain(child(p, "pPr").and_then(|ppr| child(&ppr, "rPr")))
- .filter_map(|rpr| child(&rpr, "sz"))
- .filter_map(|n| attr(&n, "val"))
- .filter_map(|v| v.parse::<u32>().ok());
- run_sizes
- .max()
- .or_else(|| styles.default_size_by_style_id.get(style_id).copied())
-}
-
-/// Distinct font sizes seen per heading/title style, largest first.
-fn compute_style_size_buckets(xml: &XmlDoc, styles: &StylesInfo) -> HashMap<String, Vec<u32>> {
- let mut sets: HashMap<String, BTreeSet<u32>> = HashMap::new();
-
- for p in xml.descendants().filter(|n| is_tag(n, "p")) {
- let Some(style_id) = paragraph_style_id(&p) else {
- continue;
- };
- let id_lower = style_id.to_ascii_lowercase();
- let name_lower = styles
- .name_by_style_id
- .get(style_id)
- .map(|s| s.to_ascii_lowercase())
- .unwrap_or_default();
- if !(id_lower.contains("heading")
- || id_lower.contains("title")
- || name_lower.contains("heading")
- || name_lower.contains("title"))
- {
- continue;
- }
- if let Some(size) = paragraph_effective_size(&p, styles, style_id) {
- sets.entry(style_id.to_string()).or_default().insert(size);
- }
- }
-
- sets
- .into_iter()
- .map(|(style_id, sizes)| {
- let mut sizes: Vec<u32> = sizes.into_iter().collect();
- sizes.reverse();
- (style_id, sizes)
- })
- .collect()
-}
-
-#[derive(Clone, Default)]
-struct RunStyle {
- bold: Option<bool>,
- italic: Option<bool>,
- strike: Option<bool>,
- code: Option<bool>,
- vert_align: Option<VerticalAlign>,
-}
-
-#[derive(Clone, Copy)]
-enum VerticalAlign {
- Baseline,
- Superscript,
- Subscript,
-}
-
-impl RunStyle {
- fn merged(&self, overrides: &RunStyle) -> RunStyle {
- RunStyle {
- bold: overrides.bold.or(self.bold),
- italic: overrides.italic.or(self.italic),
- strike: overrides.strike.or(self.strike),
- code: overrides.code.or(self.code),
- vert_align: overrides.vert_align.or(self.vert_align),
- }
- }
-
- fn apply(&self, local: &RunStyle, mut inlines: Vec<Inline>) -> Vec<Inline> {
- let resolved = self.merged(local);
-
- if resolved.code.unwrap_or(false) {
- let code_text: String = inlines
- .iter()
- .filter_map(|i| match i {
- Inline::Text(s) => Some(s.as_str()),
- _ => None,
- })
- .collect();
- if !code_text.is_empty() {
- return vec![Inline::Code(code_text)];
- }
- }
-
- if resolved.strike.unwrap_or(false) {
- inlines = vec![Inline::Del(inlines)];
- }
- if resolved.italic.unwrap_or(false) {
- inlines = vec![Inline::Em(inlines)];
- }
- if resolved.bold.unwrap_or(false) {
- inlines = vec![Inline::Strong(inlines)];
- }
- match resolved.vert_align.unwrap_or(VerticalAlign::Baseline) {
- VerticalAlign::Superscript => vec![Inline::Sup(inlines)],
- VerticalAlign::Subscript => vec![Inline::Sub(inlines)],
- VerticalAlign::Baseline => inlines,
- }
- }
-}
-
-fn read_on_off(node: &Node) -> bool {
- !matches!(
- attr(node, "val").map(|v| v.to_ascii_lowercase()).as_deref(),
- Some("0") | Some("false") | Some("off")
- )
-}
-
-fn run_style_from_rpr(rpr: &Node) -> RunStyle {
- let mut style = RunStyle::default();
- if let Some(b) = child(rpr, "b") {
- style.bold = Some(read_on_off(&b));
- }
- if let Some(i) = child(rpr, "i") {
- style.italic = Some(read_on_off(&i));
- }
- if let Some(s) = child(rpr, "strike") {
- style.strike = Some(read_on_off(&s));
- }
- if let Some(rstyle) = child(rpr, "rStyle").and_then(|n| attr(&n, "val")) {
- if rstyle.to_ascii_lowercase().contains("code") {
- style.code = Some(true);
- }
- }
- if let Some(va) = child(rpr, "vertAlign").and_then(|n| attr(&n, "val")) {
- style.vert_align = Some(match va.to_ascii_lowercase().as_str() {
- "sup" | "superscript" => VerticalAlign::Superscript,
- "sub" | "subscript" => VerticalAlign::Subscript,
- _ => VerticalAlign::Baseline,
- });
- }
- style
-}
-
-fn paragraph_run_style(p: &Node) -> RunStyle {
- child(p, "pPr")
- .and_then(|ppr| child(&ppr, "rPr"))
- .map(|rpr| run_style_from_rpr(&rpr))
- .unwrap_or_default()
-}
-
-fn parse_paragraph(node: &Node, ctx: Ctx) -> Paragraph {
- let kind = paragraph_kind(node, ctx);
- let base_style = paragraph_run_style(node);
- let mut inlines = Vec::new();
-
- for c in node.children().filter(|n| n.is_element()) {
- if is_tag(&c, "r") {
- inlines.extend(parse_run(&c, &base_style));
- } else if is_tag(&c, "hyperlink") {
- if let Some(link) = parse_hyperlink(&c, ctx.rels, &base_style) {
- inlines.push(link);
- }
- } else if is_tag(&c, "bookmarkStart") {
- if let Some(name) = attr(&c, "name") {
- inlines.push(Inline::Bookmark(BookmarkId(name.to_string())));
- }
- } else if is_tag(&c, "br") {
- inlines.push(Inline::LineBreak);
- }
- }
-
- Paragraph { kind, inlines }
-}
-
-fn parse_run(run: &Node, base_style: &RunStyle) -> Vec<Inline> {
- let local_style = child(run, "rPr")
- .map(|rpr| run_style_from_rpr(&rpr))
- .unwrap_or_default();
-
- let mut out = Vec::new();
- for c in run.children().filter(|n| n.is_element()) {
- if is_tag(&c, "t") {
- if let Some(text) = c.text() {
- out.push(Inline::Text(text.to_string()));
- }
- } else if is_tag(&c, "br") {
- out.push(Inline::LineBreak);
- } else if is_tag(&c, "tab") {
- out.push(Inline::Text("\t".to_string()));
- } else if is_tag(&c, "footnoteReference") {
- if let Some(id) = attr(&c, "id") {
- out.push(Inline::FootnoteRef(NoteId(id.to_string())));
- }
- } else if is_tag(&c, "endnoteReference") {
- if let Some(id) = attr(&c, "id") {
- out.push(Inline::EndnoteRef(NoteId(id.to_string())));
- }
- } else if is_tag(&c, "commentReference") {
- if let Some(id) = attr(&c, "id") {
- out.push(Inline::CommentRef(CommentId(id.to_string())));
- }
- }
- }
-
- base_style.apply(&local_style, out)
-}
-
-fn parse_hyperlink(node: &Node, rels: &Relationships, base_style: &RunStyle) -> Option<Inline> {
- let href = if let Some(id) = attr(node, "id") {
- rels.get(id).map(|s| s.to_string())
- } else {
- attr(node, "anchor").map(|anchor| format!("#{anchor}"))
- }?;
-
- let link_style = child(node, "rPr")
- .map(|rpr| run_style_from_rpr(&rpr))
- .unwrap_or_default();
- let combined_style = base_style.merged(&link_style);
-
- let mut link_children = Vec::new();
- for run in children(node, "r") {
- link_children.extend(parse_run(&run, &combined_style));
- }
-
- Some(Inline::Link {
- href,
- children: link_children,
- })
-}
-
-fn parse_blocks(parent: &Node, ctx: Ctx) -> Vec<Block> {
- let nodes: Vec<Node> = parent.children().filter(|n| n.is_element()).collect();
- let mut out = Vec::new();
- let mut i = 0;
-
- while i < nodes.len() {
- let node = &nodes[i];
- if is_tag(node, "p") {
- if paragraph_list_info(node, ctx.numbering).is_some() {
- let (list, next_i) = parse_list(&nodes, i, ctx);
- if !list.items.is_empty() {
- out.push(Block::List(list));
- }
- i = next_i;
- continue;
- }
- if let Some(image) = parse_image_paragraph(node, ctx.rels) {
- out.push(Block::Image(image));
- } else {
- let para = parse_paragraph(node, ctx);
- if has_visible_content(&para.inlines) {
- out.push(Block::Paragraph(para));
- }
- }
- } else if is_tag(node, "tbl") {
- out.push(Block::Table(parse_table(node, ctx)));
- }
- i += 1;
- }
- out
-}
-
-fn parse_table(node: &Node, ctx: Ctx) -> Table {
- let mut rows = Vec::new();
- for tr in children(node, "tr") {
- let header = child(&tr, "trPr")
- .and_then(|trpr| child(&trpr, "tblHeader"))
- .is_some();
- let cells = children(&tr, "tc")
- .map(|tc| TableCell {
- blocks: parse_blocks(&tc, ctx),
- colspan: NonZeroU32::MIN,
- rowspan: NonZeroU32::MIN,
- })
- .collect();
- rows.push(TableRow {
- cells,
- kind: if header {
- TableRowKind::Header
- } else {
- TableRowKind::Body
- },
- });
- }
-
- if rows.iter().any(|r| matches!(r.kind, TableRowKind::Header)) {
- if let Some(last) = rows.last_mut() {
- last.kind = TableRowKind::Footer;
- }
- }
- Table { rows }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-struct ListInfo {
- list_type: ListType,
- num_id: String,
- ilvl: u32,
-}
-
-#[derive(Debug, Default)]
-struct NumberingInfo {
- num_to_abstract: HashMap<String, String>,
- abstract_levels: HashMap<String, HashMap<String, ListType>>,
-}
-
-impl NumberingInfo {
- fn list_type(&self, num_id: &str, ilvl: &str) -> Option<ListType> {
- self
- .abstract_levels
- .get(self.num_to_abstract.get(num_id)?)?
- .get(ilvl)
- .copied()
- }
-}
-
-fn read_numbering<R: Read + Seek>(zip: &mut ZipArchive<R>) -> NumberingInfo {
- let mut info = NumberingInfo::default();
- let Some(text) = read_zip_text(zip, "word/numbering.xml") else {
- return info;
- };
- let Ok(xml) = XmlDoc::parse(strip_bom(&text)) else {
- return info;
- };
-
- for num in xml.descendants().filter(|n| is_tag(n, "num")) {
- if let (Some(num_id), Some(abs)) = (
- attr(&num, "numId"),
- child(&num, "abstractNumId").and_then(|n| attr(&n, "val")),
- ) {
- info
- .num_to_abstract
- .insert(num_id.to_string(), abs.to_string());
- }
- }
-
- for abs in xml.descendants().filter(|n| is_tag(n, "abstractNum")) {
- let Some(abs_id) = attr(&abs, "abstractNumId") else {
- continue;
- };
- let mut levels = HashMap::new();
- for lvl in children(&abs, "lvl") {
- if let Some(ilvl) = attr(&lvl, "ilvl") {
- let fmt = child(&lvl, "numFmt").and_then(|n| attr(&n, "val"));
- let list_type = match fmt.unwrap_or("") {
- "bullet" => ListType::Unordered,
- _ => ListType::Ordered,
- };
- levels.insert(ilvl.to_string(), list_type);
- }
- }
- info.abstract_levels.insert(abs_id.to_string(), levels);
- }
- info
-}
-
-fn paragraph_list_info(p: &Node, numbering: &NumberingInfo) -> Option<ListInfo> {
- let numpr = child(p, "pPr").and_then(|ppr| child(&ppr, "numPr"))?;
- let ilvl_str = child(&numpr, "ilvl").and_then(|n| attr(&n, "val"))?;
- let num_id = child(&numpr, "numId").and_then(|n| attr(&n, "val"))?;
- Some(ListInfo {
- list_type: numbering
- .list_type(num_id, ilvl_str)
- .unwrap_or(ListType::Unordered),
- num_id: num_id.to_string(),
- ilvl: ilvl_str.parse().unwrap_or(0),
- })
-}
-
-/// Consumes consecutive numbered paragraphs at one level into a list,
-/// recursing for deeper levels. Returns the list and the next unconsumed
-/// node index.
-fn parse_list(nodes: &[Node], mut i: usize, ctx: Ctx) -> (List, usize) {
- let mut list = List {
- items: Vec::new(),
- list_type: ListType::Unordered,
- };
- let Some(base) = nodes
- .get(i)
- .and_then(|n| paragraph_list_info(n, ctx.numbering))
- else {
- return (list, i + 1);
- };
- list.list_type = base.list_type;
-
- while i < nodes.len() {
- let node = &nodes[i];
- if !is_tag(node, "p") {
- break;
- }
- let Some(info) = paragraph_list_info(node, ctx.numbering) else {
- break;
- };
- if info.ilvl < base.ilvl
- || (info.ilvl == base.ilvl
- && (info.list_type != base.list_type || info.num_id != base.num_id))
- {
- break;
- }
- if info.ilvl > base.ilvl {
- // deeper level with no parent item yet; let the caller handle it
- break;
- }
-
- let mut blocks = Vec::new();
- if let Some(image) = parse_image_paragraph(node, ctx.rels) {
- blocks.push(Block::Image(image));
- } else {
- let para = parse_paragraph(node, ctx);
- if has_visible_content(&para.inlines) {
- blocks.push(Block::Paragraph(para));
- }
- }
- list.items.push(ListItem { blocks });
- i += 1;
-
- while i < nodes.len() && is_tag(&nodes[i], "p") {
- match paragraph_list_info(&nodes[i], ctx.numbering) {
- Some(sub) if sub.ilvl > base.ilvl => {
- let (sublist, next_i) = parse_list(nodes, i, ctx);
- if let Some(last) = list.items.last_mut() {
- last.blocks.push(Block::List(sublist));
- }
- i = next_i;
- }
- _ => break,
- }
- }
-
- if list.items.last().is_some_and(|last| last.blocks.is_empty()) {
- list.items.pop();
- }
- }
-
- (list, i)
-}
-
-fn parse_image_paragraph(p: &Node, rels: &Relationships) -> Option<Image> {
- let has_text = p
- .descendants()
- .filter(|n| is_tag(n, "t"))
- .any(|t| t.text().map(|s| !s.trim().is_empty()).unwrap_or(false));
- if has_text {
- return None;
- }
-
- if let Some(drawing) = p.descendants().find(|n| is_tag(n, "drawing")) {
- let blip = drawing.descendants().find(|n| is_tag(n, "blip"))?;
- let rel_id = attr(&blip, "embed").or_else(|| attr(&blip, "link"))?;
- let alt = drawing
- .descendants()
- .find(|n| is_tag(n, "docPr"))
- .and_then(|n| attr(&n, "descr").or_else(|| attr(&n, "title")))
- .map(|s| s.to_string());
- return external_image(rels.get(rel_id)?, alt);
- }
-
- if let Some(pict) = p.descendants().find(|n| is_tag(n, "pict")) {
- let imagedata = pict.descendants().find(|n| is_tag(n, "imagedata"))?;
- let rel_id = attr(&imagedata, "id")?;
- let alt = attr(&imagedata, "title").map(|s| s.to_string());
- return external_image(rels.get(rel_id)?, alt);
- }
- None
-}
-
-/// Only external (http/https) images are representable in the output.
-fn external_image(target: &str, alt: Option<String>) -> Option<Image> {
- if target.starts_with("http://") || target.starts_with("https://") {
- Some(Image {
- src: target.to_string(),
- alt,
- })
- } else {
- None
- }
-}
-
-fn read_notes<R: Read + Seek>(zip: &mut ZipArchive<R>, kind: NoteKind, ctx: Ctx) -> Vec<Note> {
- let (xml_path, rels_path, root_tag) = match kind {
- NoteKind::Footnote => (
- "word/footnotes.xml",
- "word/_rels/footnotes.xml.rels",
- "footnote",
- ),
- NoteKind::Endnote => (
- "word/endnotes.xml",
- "word/_rels/endnotes.xml.rels",
- "endnote",
- ),
- };
- let Some(text) = read_zip_text(zip, xml_path) else {
- return Vec::new();
- };
- let Ok(xml) = XmlDoc::parse(strip_bom(&text)) else {
- return Vec::new();
- };
- let rels = read_relationships(zip, rels_path);
- let ctx = Ctx { rels: &rels, ..ctx };
-
- let mut notes = Vec::new();
- for n in xml.descendants().filter(|n| is_tag(n, root_tag)) {
- let Some(id) = attr(&n, "id") else {
- continue;
- };
- if matches!(
- attr(&n, "type"),
- Some("separator" | "continuationSeparator")
- ) {
- continue;
- }
- notes.push(Note {
- id: NoteId(id.to_string()),
- kind,
- blocks: parse_blocks(&n, ctx),
- });
- }
- notes
-}
-
-fn read_comments<R: Read + Seek>(zip: &mut ZipArchive<R>, ctx: Ctx) -> Vec<Comment> {
- let Some(text) = read_zip_text(zip, "word/comments.xml") else {
- return Vec::new();
- };
- let Ok(xml) = XmlDoc::parse(strip_bom(&text)) else {
- return Vec::new();
- };
- let rels = read_relationships(zip, "word/_rels/comments.xml.rels");
- let ctx = Ctx { rels: &rels, ..ctx };
-
- let mut comments = Vec::new();
- for c in xml.descendants().filter(|n| is_tag(n, "comment")) {
- let Some(id) = attr(&c, "id") else {
- continue;
- };
- comments.push(Comment {
- id: CommentId(id.to_string()),
- author_name: attr(&c, "author").map(|s| s.to_string()),
- author_initials: attr(&c, "initials").map(|s| s.to_string()),
- blocks: parse_blocks(&c, ctx),
- });
- }
- comments
-}
diff --git a/apps/api/native/src/document/providers/factory.rs b/apps/api/native/src/document/providers/factory.rs
deleted file mode 100644
index b9bdb0b52..000000000
--- a/apps/api/native/src/document/providers/factory.rs
+++ /dev/null
@@ -1,47 +0,0 @@
-use super::doc::DocProvider;
-use super::docx::DocxProvider;
-use super::odt::OdtProvider;
-use super::rtf::RtfProvider;
-use super::xlsx::XlsxProvider;
-use super::DocumentProvider;
-use napi_derive::napi;
-
-#[napi]
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum DocumentType {
- Doc,
- Docx,
- Rtf,
- Odt,
- Xlsx,
-}
-
-pub struct ProviderFactory {
- doc_provider: DocProvider,
- docx_provider: DocxProvider,
- rtf_provider: RtfProvider,
- odt_provider: OdtProvider,
- xlsx_provider: XlsxProvider,
-}
-
-impl ProviderFactory {
- pub fn new() -> Self {
- Self {
- doc_provider: DocProvider::new(),
- docx_provider: DocxProvider::new(),
- rtf_provider: RtfProvider::new(),
- odt_provider: OdtProvider::new(),
- xlsx_provider: XlsxProvider::new(),
- }
- }
-
- pub fn get_provider(&self, doc_type: DocumentType) -> &dyn DocumentProvider {
- match doc_type {
- DocumentType::Doc => &self.doc_provider,
- DocumentType::Docx => &self.docx_provider,
- DocumentType::Rtf => &self.rtf_provider,
- DocumentType::Odt => &self.odt_provider,
- DocumentType::Xlsx => &self.xlsx_provider,
- }
- }
-}
diff --git a/apps/api/native/src/document/providers/mod.rs b/apps/api/native/src/document/providers/mod.rs
deleted file mode 100644
index 645d0fdba..000000000
--- a/apps/api/native/src/document/providers/mod.rs
+++ /dev/null
@@ -1,16 +0,0 @@
-use crate::document::model::Document;
-use std::error::Error;
-
-pub mod doc;
-pub mod docx;
-pub mod factory;
-pub mod odt;
-pub mod rtf;
-pub mod xlsx;
-
-pub trait DocumentProvider {
- fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>>;
-
- #[allow(dead_code)]
- fn name(&self) -> &'static str;
-}
diff --git a/apps/api/native/src/document/providers/odt.rs b/apps/api/native/src/document/providers/odt.rs
deleted file mode 100644
index 92a35593a..000000000
--- a/apps/api/native/src/document/providers/odt.rs
+++ /dev/null
@@ -1,617 +0,0 @@
-//! ODT (OpenDocument text) provider.
-
-use crate::document::model::*;
-use crate::document::providers::DocumentProvider;
-use crate::document::xml::{attr, child, children, is_tag, read_zip_text, strip_bom};
-use chrono::{DateTime, Utc};
-use roxmltree::{Document as XmlDoc, Node};
-use std::collections::{HashMap, HashSet};
-use std::error::Error;
-use std::io::{Cursor, Read, Seek};
-use std::num::NonZeroU32;
-use zip::read::ZipArchive;
-
-pub struct OdtProvider;
-
-impl OdtProvider {
- pub fn new() -> Self {
- Self
- }
-}
-
-impl DocumentProvider for OdtProvider {
- fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {
- let mut zip = ZipArchive::new(Cursor::new(data))?;
-
- let metadata = read_meta(&mut zip).unwrap_or_default();
- let styles = read_styles(&mut zip);
-
- let content =
- read_zip_text(&mut zip, "content.xml").ok_or("Missing content.xml in document")?;
- let xml = XmlDoc::parse(strip_bom(&content))?;
-
- let mut parser = Parser {
- styles: &styles,
- notes: Vec::new(),
- comments: Vec::new(),
- };
- let blocks = xml
- .descendants()
- .find(|n| is_tag(n, "text") && n.ancestors().any(|a| is_tag(&a, "body")))
- .map(|body| parser.parse_blocks(&body))
- .unwrap_or_default();
-
- Ok(Document {
- blocks,
- metadata,
- notes: parser.notes,
- comments: parser.comments,
- })
- }
-
- fn name(&self) -> &'static str {
- "odt"
- }
-}
-
-fn read_meta<R: Read + Seek>(zip: &mut ZipArchive<R>) -> Option<DocumentMetadata> {
- let text = read_zip_text(zip, "meta.xml")?;
- let xml = XmlDoc::parse(strip_bom(&text)).ok()?;
- let element_text = |local: &str| {
- xml
- .descendants()
- .find(|n| is_tag(n, local))
- .and_then(|n| n.text())
- };
-
- let mut meta = DocumentMetadata::default();
- if let Some(title) = element_text("title") {
- if !title.trim().is_empty() {
- meta.title = Some(title.to_string());
- }
- }
- if let Some(author) = element_text("creator").or_else(|| element_text("initial-creator")) {
- let trimmed = author.trim();
- if !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("unknown") {
- meta.author = Some(trimmed.to_string());
- }
- }
- if let Some(created) = element_text("creation-date") {
- if let Ok(dt) = DateTime::parse_from_rfc3339(created) {
- meta.created = Some(DateTime::<Utc>::from(dt));
- }
- }
- Some(meta)
-}
-
-#[derive(Debug, Default)]
-struct OdtStylesInfo {
- paragraph_style_names: HashSet<String>,
- paragraph_outline_level: HashMap<String, u8>,
- paragraph_text_props: HashMap<String, TextStyleProps>,
- text_props: HashMap<String, TextStyleProps>,
- list_is_ordered: HashMap<String, bool>,
-}
-
-#[derive(Debug, Default, Clone, Copy)]
-struct TextStyleProps {
- bold: bool,
- italic: bool,
- strike: bool,
- sup: bool,
- sub: bool,
- code: bool,
-}
-
-fn read_styles<R: Read + Seek>(zip: &mut ZipArchive<R>) -> OdtStylesInfo {
- let mut info = OdtStylesInfo::default();
- for path in ["styles.xml", "content.xml"] {
- if let Some(text) = read_zip_text(zip, path) {
- if let Ok(xml) = XmlDoc::parse(strip_bom(&text)) {
- harvest_styles(&xml, &mut info);
- }
- }
- }
- info
-}
-
-fn harvest_styles(xml: &XmlDoc, out: &mut OdtStylesInfo) {
- for style in xml.descendants().filter(|n| is_tag(n, "style")) {
- let (Some(family), Some(name)) = (attr(&style, "family"), attr(&style, "name")) else {
- continue;
- };
-
- match family {
- "paragraph" => {
- out.paragraph_style_names.insert(name.to_string());
-
- let outline_level = child(&style, "paragraph-properties")
- .and_then(|pp| attr(&pp, "outline-level"))
- .and_then(|v| v.parse::<u8>().ok())
- .map(|v| v.min(6))
- .or_else(|| attr(&style, "parent-style-name").and_then(parse_heading_level));
- if let Some(level) = outline_level {
- out.paragraph_outline_level.insert(name.to_string(), level);
- }
-
- if let Some(tp) = child(&style, "text-properties") {
- out
- .paragraph_text_props
- .insert(name.to_string(), parse_text_properties(&tp));
- }
- }
- "text" => {
- if let Some(tp) = child(&style, "text-properties") {
- out
- .text_props
- .insert(name.to_string(), parse_text_properties(&tp));
- }
- }
- "list" => {
- let is_ordered = style
- .children()
- .any(|c| is_tag(&c, "list-level-style-number"));
- out.list_is_ordered.insert(name.to_string(), is_ordered);
- }
- _ => {}
- }
- }
-
- for list_style in xml.descendants().filter(|n| is_tag(n, "list-style")) {
- if let Some(name) = attr(&list_style, "name") {
- let is_ordered = list_style
- .children()
- .any(|c| is_tag(&c, "list-level-style-number"));
- out.list_is_ordered.insert(name.to_string(), is_ordered);
- }
- }
-}
-
-fn parse_text_properties(tp: &Node) -> TextStyleProps {
- let mut props = TextStyleProps::default();
-
- if attr(tp, "font-weight").is_some_and(|v| v.eq_ignore_ascii_case("bold")) {
- props.bold = true;
- }
- if attr(tp, "font-style").is_some_and(|v| v.eq_ignore_ascii_case("italic")) {
- props.italic = true;
- }
- if attr(tp, "text-line-through-type")
- .or_else(|| attr(tp, "text-line-through-style"))
- .is_some_and(|v| v != "none")
- {
- props.strike = true;
- }
- if let Some(v) = attr(tp, "text-position") {
- let lower = v.to_ascii_lowercase();
- if lower.contains("sup") {
- props.sup = true;
- } else if lower.contains("sub") {
- props.sub = true;
- }
- }
- if let Some(font) = attr(tp, "font-name") {
- let lower = font.to_ascii_lowercase();
- if lower.contains("courier") || lower.contains("mono") {
- props.code = true;
- }
- }
- props
-}
-
-/// Heading level from a style name like "Heading_20_2" or "Title".
-fn parse_heading_level(style_name: &str) -> Option<u8> {
- let normalized = style_name.replace("_20_", " ").replace('_', " ");
- let lower = normalized.to_ascii_lowercase();
- if lower.contains("title") {
- return Some(1);
- }
- let tail = &lower[lower.find("heading")? + "heading".len()..];
- let digits: String = tail.chars().filter(|c| c.is_ascii_digit()).collect();
- digits.parse::<u8>().ok().map(|n| n.clamp(1, 6))
-}
-
-struct Parser<'a> {
- styles: &'a OdtStylesInfo,
- notes: Vec<Note>,
- comments: Vec<Comment>,
-}
-
-impl Parser<'_> {
- fn parse_blocks(&mut self, node: &Node) -> Vec<Block> {
- let mut blocks = Vec::new();
-
- for c in node.children().filter(|n| n.is_element()) {
- if is_tag(&c, "h") {
- self.push_paragraph(&c, &mut blocks);
- } else if is_tag(&c, "p") {
- if let Some(image) = image_from_paragraph(&c) {
- blocks.push(Block::Image(image));
- } else {
- self.push_paragraph(&c, &mut blocks);
- }
- } else if is_tag(&c, "list") {
- self.push_list(&c, &mut blocks);
- } else if is_tag(&c, "table") {
- blocks.push(Block::Table(self.parse_table(&c)));
- } else {
- blocks.extend(self.parse_blocks(&c));
- }
- }
- blocks
- }
-
- fn push_paragraph(&mut self, node: &Node, blocks: &mut Vec<Block>) {
- let para = self.parse_paragraph(node);
- if has_visible_content(&para.inlines) {
- blocks.push(Block::Paragraph(para));
- }
- }
-
- fn push_list(&mut self, list: &Node, blocks: &mut Vec<Block>) {
- // Writers wrap continuation sublists in a single-item shell list; unwrap
- // and attach them to the previous list's last item.
- let mut effective = *list;
- let mut inherited_style_name = attr(&effective, "style-name");
- let mut unwrapped = false;
- while let Some(inner) = unwrap_single_nested_list(&effective) {
- effective = inner;
- unwrapped = true;
- if inherited_style_name.is_none() {
- inherited_style_name = attr(&effective, "style-name");
- }
- }
-
- if is_heading_list(&effective) {
- for li in children(&effective, "list-item") {
- let headings: Vec<Node> = li.descendants().filter(|n| is_tag(n, "h")).collect();
- if let Some(h) = headings.first() {
- self.push_paragraph(h, blocks);
- }
- }
- return;
- }
-
- let parsed = self.parse_list(&effective, inherited_style_name);
- if unwrapped {
- if let Some(Block::List(prev)) = blocks.last_mut() {
- if let Some(last_item) = prev.items.last_mut() {
- last_item.blocks.push(Block::List(parsed));
- return;
- }
- }
- }
- blocks.push(Block::List(parsed));
- }
-
- fn parse_list(&mut self, node: &Node, inherited_style_name: Option<&str>) -> List {
- let style_name = attr(node, "style-name").or(inherited_style_name);
- let list_type = match style_name.and_then(|n| self.styles.list_is_ordered.get(n)) {
- Some(true) => ListType::Ordered,
- _ => ListType::Unordered,
- };
-
- let items = children(node, "list-item")
- .map(|li| ListItem {
- blocks: self.parse_blocks(&li),
- })
- .collect();
- List { items, list_type }
- }
-
- fn parse_table(&mut self, node: &Node) -> Table {
- let mut rows = Vec::new();
- for tr in children(node, "table-row") {
- let cells = children(&tr, "table-cell")
- .map(|tc| TableCell {
- blocks: self.parse_blocks(&tc),
- colspan: span_attr(&tc, "number-columns-spanned"),
- rowspan: span_attr(&tc, "number-rows-spanned"),
- })
- .collect();
- rows.push(TableRow {
- cells,
- kind: TableRowKind::Body,
- });
- }
- Table { rows }
- }
-
- fn parse_paragraph(&mut self, node: &Node) -> Paragraph {
- let kind = self.paragraph_kind(node);
- let base = attr(node, "style-name")
- .and_then(|name| self.styles.paragraph_text_props.get(name))
- .copied()
- .unwrap_or_default();
- let inlines = self.parse_inlines(node);
- Paragraph {
- kind,
- inlines: apply_text_style(inlines, None, self.styles, base),
- }
- }
-
- fn paragraph_kind(&self, p: &Node) -> ParagraphKind {
- if p.tag_name().name() == "h" {
- let level = attr(p, "outline-level")
- .and_then(|v| v.parse::<u8>().ok())
- .unwrap_or(1);
- return ParagraphKind::Heading(level.clamp(1, 6));
- }
-
- if let Some(style_name) = attr(p, "style-name") {
- if let Some(level) = self.styles.paragraph_outline_level.get(style_name) {
- return ParagraphKind::Heading((*level).min(6));
- }
- if self.styles.paragraph_style_names.contains(style_name)
- && style_name.to_ascii_lowercase().contains("quote")
- {
- return ParagraphKind::Blockquote;
- }
- }
- ParagraphKind::Normal
- }
-
- fn parse_inlines(&mut self, node: &Node) -> Vec<Inline> {
- let mut out = Vec::new();
-
- for c in node.children() {
- if c.is_text() {
- if let Some(t) = c.text() {
- if !t.is_empty() {
- out.push(Inline::Text(t.to_string()));
- }
- }
- continue;
- }
- if !c.is_element() {
- continue;
- }
-
- if is_tag(&c, "span") {
- let inner = self.parse_inlines(&c);
- let style_name = attr(&c, "style-name");
- out.extend(apply_text_style(
- inner,
- style_name,
- self.styles,
- TextStyleProps::default(),
- ));
- } else if is_tag(&c, "a") {
- let link_children = self.parse_inlines(&c);
- match attr(&c, "href") {
- Some(href) => out.push(Inline::Link {
- href: href.to_string(),
- children: link_children,
- }),
- None => out.extend(link_children),
- }
- } else if is_tag(&c, "line-break") {
- out.push(Inline::LineBreak);
- } else if is_tag(&c, "s") {
- let count = attr(&c, "c")
- .and_then(|v| v.parse::<usize>().ok())
- .unwrap_or(1)
- .min(1000);
- out.push(Inline::Text(" ".repeat(count)));
- } else if is_tag(&c, "tab") {
- out.push(Inline::Text("\t".to_string()));
- } else if is_tag(&c, "bookmark-start") {
- if let Some(name) = attr(&c, "name") {
- out.push(Inline::Bookmark(BookmarkId(name.to_string())));
- }
- } else if is_tag(&c, "note") {
- out.push(self.parse_note(&c));
- } else if is_tag(&c, "annotation") {
- out.push(self.parse_annotation(&c));
- } else {
- out.extend(self.parse_inlines(&c));
- }
- }
- out
- }
-
- fn parse_note(&mut self, node: &Node) -> Inline {
- let kind = match attr(node, "note-class") {
- Some("endnote") => NoteKind::Endnote,
- _ => NoteKind::Footnote,
- };
- let id = attr(node, "id")
- .map(|s| s.to_string())
- .unwrap_or_else(|| format!("odt-note-{}", self.notes.len() + 1));
-
- let blocks = child(node, "note-body")
- .map(|body| self.parse_note_body(&body))
- .unwrap_or_default();
- self.notes.push(Note {
- id: NoteId(id.clone()),
- kind,
- blocks,
- });
-
- match kind {
- NoteKind::Footnote => Inline::FootnoteRef(NoteId(id)),
- NoteKind::Endnote => Inline::EndnoteRef(NoteId(id)),
- }
- }
-
- fn parse_note_body(&mut self, node: &Node) -> Vec<Block> {
- let mut blocks = Vec::new();
- let paragraphs: Vec<Node> = node
- .children()
- .filter(|n| is_tag(n, "p") || is_tag(n, "h"))
- .collect();
- for p in paragraphs {
- let para = self.parse_paragraph(&p);
- if has_visible_content(&para.inlines) {
- blocks.push(Block::Paragraph(para));
- }
- }
- blocks
- }
-
- fn parse_annotation(&mut self, node: &Node) -> Inline {
- let id = format!("odt-comment-{}", self.comments.len() + 1);
- let descendant_text = |local: &str| {
- node
- .descendants()
- .find(|n| is_tag(n, local))
- .and_then(|n| n.text())
- .map(str::trim)
- .filter(|t| !t.is_empty())
- .map(str::to_string)
- };
- let author = descendant_text("creator");
- let initials = descendant_text("initials");
-
- let mut blocks = Vec::new();
- let paragraphs: Vec<Node> = node.children().filter(|n| is_tag(n, "p")).collect();
- for p in paragraphs {
- let inlines = self.parse_inlines(&p);
- if !inlines.is_empty() {
- blocks.push(Block::Paragraph(Paragraph {
- kind: ParagraphKind::Normal,
- inlines,
- }));
- }
- }
-
- self.comments.push(Comment {
- id: CommentId(id.clone()),
- author_name: author,
- author_initials: initials,
- blocks,
- });
- Inline::CommentRef(CommentId(id))
- }
-}
-
-fn span_attr(node: &Node, local: &str) -> NonZeroU32 {
- attr(node, local)
- .and_then(|v| v.parse::<u32>().ok())
- .and_then(NonZeroU32::new)
- .unwrap_or(NonZeroU32::MIN)
-}
-
-fn unwrap_single_nested_list<'a>(list: &Node<'a, 'a>) -> Option<Node<'a, 'a>> {
- let mut items = children(list, "list-item");
- let first = items.next()?;
- if items.next().is_some() {
- return None;
- }
- let mut inner_lists = first.children().filter(|n| is_tag(n, "list"));
- let inner = inner_lists.next()?;
- if inner_lists.next().is_some() {
- return None;
- }
- Some(inner)
-}
-
-fn is_heading_list(list: &Node) -> bool {
- let mut any = false;
- for li in children(list, "list-item") {
- any = true;
- if !li.descendants().any(|n| is_tag(&n, "h")) {
- return false;
- }
- }
- any
-}
-
-fn apply_text_style(
- mut inlines: Vec<Inline>,
- style_name: Option<&str>,
- styles: &OdtStylesInfo,
- base: TextStyleProps,
-) -> Vec<Inline> {
- let mut props = base;
- if let Some(name) = style_name {
- let mut style_props = styles.text_props.get(name).copied().unwrap_or_default();
- if name.to_ascii_lowercase().contains("code") {
- style_props.code = true;
- }
- props.bold |= style_props.bold;
- props.italic |= style_props.italic;
- props.strike |= style_props.strike;
- props.sup |= style_props.sup;
- props.sub |= style_props.sub;
- props.code |= style_props.code;
- }
-
- if props.code {
- let code_text: String = inlines
- .iter()
- .filter_map(|i| match i {
- Inline::Text(s) => Some(s.as_str()),
- _ => None,
- })
- .collect();
- if !code_text.is_empty() {
- return vec![Inline::Code(code_text)];
- }
- }
-
- if props.strike {
- inlines = vec![Inline::Del(inlines)];
- }
- if props.italic {
- inlines = vec![Inline::Em(inlines)];
- }
- if props.bold {
- inlines = vec![Inline::Strong(inlines)];
- }
- if props.sup {
- inlines = vec![Inline::Sup(inlines)];
- }
- if props.sub {
- inlines = vec![Inline::Sub(inlines)];
- }
- inlines
-}
-
-fn image_from_paragraph(p: &Node) -> Option<Image> {
- let image = p.descendants().find(|n| is_tag(n, "image"))?;
- let href = attr(&image, "href")?;
- // only external images are representable in the output
- if href.starts_with("http://") || href.starts_with("https://") {
- Some(Image {
- src: href.to_string(),
- alt: None,
- })
- } else {
- None
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use std::io::Write;
-
- #[test]
- fn huge_space_count_is_clamped() {
- let content = r#"<?xml version="1.0"?>
-<office:document-content xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0">
-<office:body><office:text><text:p>a<text:s text:c="4000000000"/>b</text:p></office:text></office:body>
-</office:document-content>"#;
- let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
- zip
- .start_file("content.xml", zip::write::SimpleFileOptions::default())
- .unwrap();
- zip.write_all(content.as_bytes()).unwrap();
- let data = zip.finish().unwrap().into_inner();
-
- let document = OdtProvider::new().parse_buffer(&data).unwrap();
- let Block::Paragraph(p) = &document.blocks[0] else {
- panic!("expected paragraph");
- };
- let text: String = p
- .inlines
- .iter()
- .map(|i| match i {
- Inline::Text(t) => t.as_str(),
- _ => "",
- })
- .collect();
- assert_eq!(text.len(), "ab".len() + 1000);
- }
-}
diff --git a/apps/api/native/src/document/providers/rtf.rs b/apps/api/native/src/document/providers/rtf.rs
deleted file mode 100644
index 260216114..000000000
--- a/apps/api/native/src/document/providers/rtf.rs
+++ /dev/null
@@ -1,822 +0,0 @@
-//! RTF provider. Tokenizes the RTF control stream and decodes 8-bit text
-//! through the code page the document declares with \ansicpg.
-
-use crate::document::encoding::encoding_for_codepage;
-use crate::document::model::*;
-use crate::document::providers::DocumentProvider;
-use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc};
-use encoding_rs::{Encoding, WINDOWS_1252};
-use std::error::Error;
-use std::num::NonZeroU32;
-
-pub struct RtfProvider;
-
-impl RtfProvider {
- pub fn new() -> Self {
- Self
- }
-}
-
-impl DocumentProvider for RtfProvider {
- fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {
- let encoding = detect_encoding(data);
- let metadata = extract_metadata(data, encoding);
- let blocks = parse_body(data, encoding);
-
- Ok(Document {
- blocks,
- metadata,
- notes: Vec::new(),
- comments: Vec::new(),
- })
- }
-
- fn name(&self) -> &'static str {
- "rtf"
- }
-}
-
-/// Code page declared by the first \ansicpg control word.
-fn detect_encoding(src: &[u8]) -> &'static Encoding {
- for token in Tokens::new(src) {
- if let Token::Control("ansicpg", Some(codepage)) = token {
- return u16::try_from(codepage)
- .ok()
- .and_then(encoding_for_codepage)
- .unwrap_or(WINDOWS_1252);
- }
- }
- WINDOWS_1252
-}
-
-#[derive(Debug, PartialEq)]
-enum Token<'a> {
- Open,
- Close,
- Control(&'a str, Option<i32>),
- Symbol(u8),
- Hex(u8),
- Byte(u8),
-}
-
-struct Tokens<'a> {
- src: &'a [u8],
- pos: usize,
-}
-
-impl<'a> Tokens<'a> {
- fn new(src: &'a [u8]) -> Self {
- Self { src, pos: 0 }
- }
-}
-
-impl<'a> Iterator for Tokens<'a> {
- type Item = Token<'a>;
-
- fn next(&mut self) -> Option<Token<'a>> {
- loop {
- let b = *self.src.get(self.pos)?;
- self.pos += 1;
- match b {
- b'\r' | b'\n' => continue,
- b'{' => return Some(Token::Open),
- b'}' => return Some(Token::Close),
- b'\\' => return self.next_after_backslash(),
- b => return Some(Token::Byte(b)),
- }
- }
- }
-}
-
-impl<'a> Tokens<'a> {
- fn next_after_backslash(&mut self) -> Option<Token<'a>> {
- let b = *self.src.get(self.pos)?;
- self.pos += 1;
- match b {
- b'\'' => {
- let hi = self.src.get(self.pos).copied().and_then(hex_val);
- let lo = self.src.get(self.pos + 1).copied().and_then(hex_val);
- match (hi, lo) {
- (Some(hi), Some(lo)) => {
- self.pos += 2;
- Some(Token::Hex((hi << 4) | lo))
- }
- // malformed escape; drop it and keep tokenizing
- _ => Some(Token::Symbol(b'\'')),
- }
- }
- b'a'..=b'z' | b'A'..=b'Z' => {
- let start = self.pos - 1;
- while self.src.get(self.pos).is_some_and(u8::is_ascii_alphabetic) {
- self.pos += 1;
- }
- let word = std::str::from_utf8(&self.src[start..self.pos]).ok()?;
-
- let mut value = None;
- let negative = self.src.get(self.pos) == Some(&b'-');
- let num_start = self.pos + negative as usize;
- let mut num_end = num_start;
- while self.src.get(num_end).is_some_and(u8::is_ascii_digit) {
- num_end += 1;
- }
- if num_end > num_start {
- // consumed even when the value overflows i32
- self.pos = num_end;
- if let Some(n) = std::str::from_utf8(&self.src[num_start..num_end])
- .ok()
- .and_then(|s| s.parse::<i32>().ok())
- {
- value = Some(if negative { -n } else { n });
- }
- }
- if self.src.get(self.pos) == Some(&b' ') {
- self.pos += 1;
- }
- Some(Token::Control(word, value))
- }
- b => {
- if b == b'*' && self.src.get(self.pos) == Some(&b' ') {
- self.pos += 1;
- }
- Some(Token::Symbol(b))
- }
- }
- }
-}
-
-/// Fallback byte count declared by \uc.
-fn uc_count(value: Option<i32>) -> usize {
- value.unwrap_or(1).max(0) as usize
-}
-
-fn hex_val(b: u8) -> Option<u8> {
- match b {
- b'0'..=b'9' => Some(b - b'0'),
- b'a'..=b'f' => Some(10 + b - b'a'),
- b'A'..=b'F' => Some(10 + b - b'A'),
- _ => None,
- }
-}
-
-/// Text accumulator; raw bytes are buffered and decoded as a unit.
-struct TextAccum {
- encoding: &'static Encoding,
- pending: Vec<u8>,
- text: String,
-}
-
-impl TextAccum {
- fn new(encoding: &'static Encoding) -> Self {
- Self {
- encoding,
- pending: Vec::new(),
- text: String::new(),
- }
- }
-
- fn push_byte(&mut self, b: u8) {
- self.pending.push(b);
- }
-
- fn push_char(&mut self, c: char) {
- self.flush_pending();
- self.text.push(c);
- }
-
- fn flush_pending(&mut self) {
- if self.pending.is_empty() {
- return;
- }
- let (decoded, _) = self.encoding.decode_without_bom_handling(&self.pending);
- for ch in decoded.chars() {
- if ch == '\t' || ch == '\u{00A0}' || ch as u32 >= 0x20 {
- self.text.push(ch);
- }
- }
- self.pending.clear();
- }
-
- fn take(&mut self) -> String {
- self.flush_pending();
- std::mem::take(&mut self.text)
- }
-
- fn is_empty(&mut self) -> bool {
- self.flush_pending();
- self.text.is_empty()
- }
-}
-
-#[derive(Clone, Default)]
-struct CharState {
- bold: bool,
- italic: bool,
- strike: bool,
- sup: bool,
- sub: bool,
-}
-
-struct Group {
- saved: CharState,
- skip: bool,
- name_seen: bool,
-}
-
-#[derive(Default)]
-struct TableBuilder {
- rows: Vec<TableRow>,
- current_row: Vec<TableCell>,
- current_cell_blocks: Vec<Block>,
-}
-
-impl TableBuilder {
- fn start_row(&mut self) {
- self.current_cell_blocks.clear();
- self.current_row.clear();
- }
-
- fn push_block(&mut self, block: Block) {
- self.current_cell_blocks.push(block);
- }
-
- fn finish_cell(&mut self) {
- if self.current_cell_blocks.is_empty() {
- return;
- }
- self.current_row.push(TableCell {
- blocks: std::mem::take(&mut self.current_cell_blocks),
- colspan: NonZeroU32::MIN,
- rowspan: NonZeroU32::MIN,
- });
- }
-
- fn finish_row(&mut self) {
- self.finish_cell();
- if self.current_row.is_empty() {
- return;
- }
- self.rows.push(TableRow {
- cells: std::mem::take(&mut self.current_row),
- kind: TableRowKind::Body,
- });
- }
-
- fn finalize(mut self) -> Option<Block> {
- self.finish_row();
- if self.rows.is_empty() {
- None
- } else {
- Some(Block::Table(Table { rows: self.rows }))
- }
- }
-}
-
-const SKIP_DESTS: &[&str] = &[
- "fonttbl",
- "colortbl",
- "stylesheet",
- "listtable",
- "listoverridetable",
- "themedata",
- "latentstyles",
- "rsidtbl",
- "xmlnstbl",
- "mmathPr",
- "wgrffmtfilter",
- "datastore",
- "filetbl",
- "colorschememapping",
- "pnseclvl1",
- "pnseclvl2",
- "pnseclvl3",
- "pnseclvl4",
- "pnseclvl5",
- "pnseclvl6",
- "pnseclvl7",
- "pnseclvl8",
- "pnseclvl9",
- "pict",
- "object",
- "info",
-];
-
-struct BodyParser {
- state: CharState,
- stack: Vec<Group>,
- blocks: Vec<Block>,
- inlines: Vec<Inline>,
- text: TextAccum,
- table: Option<TableBuilder>,
- in_table_cell: bool,
- uc_skip: usize,
- pending_uc_skip: usize,
-}
-
-fn parse_body(src: &[u8], encoding: &'static Encoding) -> Vec<Block> {
- let mut p = BodyParser {
- state: CharState::default(),
- stack: Vec::new(),
- blocks: Vec::new(),
- inlines: Vec::new(),
- text: TextAccum::new(encoding),
- table: None,
- in_table_cell: false,
- uc_skip: 1,
- pending_uc_skip: 0,
- };
-
- for token in Tokens::new(src) {
- p.handle_token(token);
- }
-
- if !p.text.is_empty() || !p.inlines.is_empty() {
- p.flush_paragraph();
- }
- p.flush_table();
- p.blocks
-}
-
-impl BodyParser {
- fn skipping(&self) -> bool {
- self.stack.last().map(|g| g.skip).unwrap_or(false)
- }
-
- fn handle_token(&mut self, token: Token) {
- if self.pending_uc_skip > 0 {
- match token {
- Token::Byte(_) | Token::Hex(_) => {
- self.pending_uc_skip -= 1;
- return;
- }
- _ => self.pending_uc_skip = 0,
- }
- }
-
- match token {
- Token::Open => {
- let inherited_skip = self.skipping();
- self.stack.push(Group {
- saved: self.state.clone(),
- skip: inherited_skip,
- name_seen: false,
- });
- }
- Token::Close => {
- if !self.skipping() {
- self.flush_text();
- }
- if let Some(group) = self.stack.pop() {
- self.state = group.saved;
- }
- }
- Token::Byte(b) | Token::Hex(b) => {
- if !self.skipping() {
- self.text.push_byte(b);
- }
- }
- Token::Symbol(b'*') => {
- if let Some(group) = self.stack.last_mut() {
- if !group.name_seen {
- group.name_seen = true;
- group.skip = true;
- }
- }
- }
- Token::Symbol(b @ (b'\\' | b'{' | b'}')) => {
- if !self.skipping() {
- self.text.push_byte(b);
- }
- }
- Token::Symbol(b'~') => {
- if !self.skipping() {
- self.text.push_char('\u{00A0}');
- }
- }
- Token::Symbol(b'-') => {
- if !self.skipping() {
- self.text.push_char('\u{00AD}');
- }
- }
- Token::Symbol(_) => {}
- Token::Control(word, value) => self.handle_control(word, value),
- }
- }
-
- fn handle_control(&mut self, word: &str, value: Option<i32>) {
- if let Some(group) = self.stack.last_mut() {
- if !group.name_seen {
- group.name_seen = true;
- if SKIP_DESTS.contains(&word) {
- group.skip = true;
- }
- }
- }
-
- if self.skipping() {
- return;
- }
- if word == "par" {
- self.flush_paragraph();
- return;
- }
-
- let on = |value: Option<i32>| value.map(|v| v != 0).unwrap_or(true);
- match word {
- "rquote" => self.text.push_char('\u{2019}'),
- "lquote" => self.text.push_char('\u{2018}'),
- "rdblquote" => self.text.push_char('\u{201D}'),
- "ldblquote" => self.text.push_char('\u{201C}'),
- "emdash" => self.text.push_char('\u{2014}'),
- "endash" => self.text.push_char('\u{2013}'),
- "bullet" => self.text.push_char('\u{2022}'),
- "tab" => self.text.push_char('\t'),
- "line" => {
- self.flush_text();
- self.inlines.push(Inline::LineBreak);
- }
- "u" => {
- if let Some(mut code) = value {
- if code < 0 {
- code += 65536;
- }
- if let Some(ch) = char::from_u32(code as u32) {
- self.text.push_char(ch);
- }
- self.pending_uc_skip = self.uc_skip;
- }
- }
- "uc" => self.uc_skip = uc_count(value),
- "b" => {
- self.flush_text();
- self.state.bold = on(value);
- }
- "i" => {
- self.flush_text();
- self.state.italic = on(value);
- }
- "strike" | "striked" | "striked1" => {
- self.flush_text();
- self.state.strike = on(value);
- }
- "super" => {
- self.flush_text();
- self.state.sup = on(value);
- if self.state.sup {
- self.state.sub = false;
- }
- }
- "sub" => {
- self.flush_text();
- self.state.sub = on(value);
- if self.state.sub {
- self.state.sup = false;
- }
- }
- "nosupersub" => {
- self.flush_text();
- self.state.sup = false;
- self.state.sub = false;
- }
- "plain" => {
- self.flush_text();
- self.state = CharState::default();
- }
- "trowd" => {
- self
- .table
- .get_or_insert_with(TableBuilder::default)
- .start_row();
- self.in_table_cell = false;
- }
- "intbl" => self.in_table_cell = true,
- "cell" => {
- self.in_table_cell = true;
- self.flush_paragraph();
- if let Some(table) = self.table.as_mut() {
- table.finish_cell();
- }
- self.in_table_cell = false;
- }
- "row" => {
- if let Some(table) = self.table.as_mut() {
- table.finish_row();
- }
- self.in_table_cell = false;
- }
- _ => {}
- }
- }
-
- fn flush_text(&mut self) {
- let text = self.text.take();
- if text.is_empty() {
- return;
- }
- let mut node = Inline::Text(text);
- if self.state.strike {
- node = Inline::Del(vec![node]);
- }
- if self.state.italic {
- node = Inline::Em(vec![node]);
- }
- if self.state.bold {
- node = Inline::Strong(vec![node]);
- }
- if self.state.sup {
- node = Inline::Sup(vec![node]);
- } else if self.state.sub {
- node = Inline::Sub(vec![node]);
- }
- self.inlines.push(node);
- }
-
- fn flush_paragraph(&mut self) {
- self.flush_text();
- if has_visible_content(&self.inlines) {
- let block = Block::Paragraph(Paragraph {
- kind: ParagraphKind::Normal,
- inlines: std::mem::take(&mut self.inlines),
- });
- if self.in_table_cell {
- if let Some(table) = self.table.as_mut() {
- table.push_block(block);
- } else {
- self.blocks.push(block);
- }
- } else {
- self.flush_table();
- self.blocks.push(block);
- }
- } else {
- self.inlines.clear();
- if !self.in_table_cell {
- self.flush_table();
- }
- }
- }
-
- fn flush_table(&mut self) {
- if let Some(table) = self.table.take() {
- if let Some(block) = table.finalize() {
- self.blocks.push(block);
- }
- }
- }
-}
-
-fn extract_metadata(src: &[u8], encoding: &'static Encoding) -> DocumentMetadata {
- let mut metadata = DocumentMetadata::default();
- let Some(info) = find_group(src, b"{\\info") else {
- return metadata;
- };
-
- if let Some(author) = find_group(info, b"{\\author").and_then(|g| group_text(g, encoding)) {
- if !author.eq_ignore_ascii_case("unknown") {
- metadata.author = Some(author);
- }
- }
- if let Some(title) = find_group(info, b"{\\title").and_then(|g| group_text(g, encoding)) {
- metadata.title = Some(title);
- }
- metadata.created = find_group(info, b"{\\creatim").and_then(parse_timestamp);
-
- metadata
-}
-
-fn find_group<'a>(buf: &'a [u8], start_tag: &[u8]) -> Option<&'a [u8]> {
- let start = buf.windows(start_tag.len()).position(|w| w == start_tag)?;
- let mut depth = 0usize;
- for (i, &b) in buf[start..].iter().enumerate() {
- match b {
- b'{' => depth += 1,
- b'}' => {
- depth -= 1;
- if depth == 0 {
- return Some(&buf[start..start + i + 1]);
- }
- }
- _ => {}
- }
- }
- None
-}
-
-/// Plain text of a destination group, flattening nested formatting groups
-/// and ignoring \*-prefixed destinations.
-fn group_text(group: &[u8], encoding: &'static Encoding) -> Option<String> {
- let mut accum = TextAccum::new(encoding);
- let mut depth = 0usize;
- let mut ignored_depth: Option<usize> = None;
- let mut just_opened = false;
- let mut pending_uc_skip = 0usize;
- let mut uc_skip = 1usize;
-
- for token in Tokens::new(group) {
- if just_opened {
- just_opened = false;
- if matches!(token, Token::Symbol(b'*')) && ignored_depth.is_none() {
- ignored_depth = Some(depth);
- continue;
- }
- }
- match token {
- Token::Open => {
- depth += 1;
- just_opened = true;
- continue;
- }
- Token::Close => {
- depth = depth.saturating_sub(1);
- if ignored_depth.is_some_and(|d| depth < d) {
- ignored_depth = None;
- }
- continue;
- }
- _ => {}
- }
- if ignored_depth.is_some() {
- continue;
- }
- if pending_uc_skip > 0 {
- match token {
- Token::Byte(_) | Token::Hex(_) => {
- pending_uc_skip -= 1;
- continue;
- }
- _ => pending_uc_skip = 0,
- }
- }
- match token {
- Token::Byte(b) | Token::Hex(b) => accum.push_byte(b),
- Token::Symbol(b @ (b'\\' | b'{' | b'}')) => accum.push_byte(b),
- Token::Control("u", Some(mut code)) => {
- if code < 0 {
- code += 65536;
- }
- if let Some(ch) = char::from_u32(code as u32) {
- accum.push_char(ch);
- }
- pending_uc_skip = uc_skip;
- }
- Token::Control("uc", value) => uc_skip = uc_count(value),
- _ => {}
- }
- }
-
- let text = accum.take().trim().to_string();
- if text.is_empty() {
- None
- } else {
- Some(text)
- }
-}
-
-fn parse_timestamp(group: &[u8]) -> Option<DateTime<Utc>> {
- let mut yr = None;
- let mut mo = None;
- let mut dy = None;
- let mut hr = None;
- let mut mi = None;
-
- for token in Tokens::new(group) {
- if let Token::Control(word, value) = token {
- match word {
- "yr" => yr = value,
- "mo" => mo = value.map(|v| v as u32),
- "dy" => dy = value.map(|v| v as u32),
- "hr" => hr = value.map(|v| v as u32),
- "min" => mi = value.map(|v| v as u32),
- _ => {}
- }
- }
- }
-
- let date = NaiveDate::from_ymd_opt(yr?, mo?, dy?)?;
- let time = chrono::NaiveTime::from_hms_opt(hr.unwrap_or(0), mi.unwrap_or(0), 0)?;
- Some(DateTime::<Utc>::from_naive_utc_and_offset(
- NaiveDateTime::new(date, time),
- Utc,
- ))
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::document::providers::DocumentProvider;
-
- fn paragraphs(data: &[u8]) -> Vec<String> {
- let document = RtfProvider::new().parse_buffer(data).unwrap();
- document
- .blocks
- .iter()
- .filter_map(|block| match block {
- Block::Paragraph(p) => Some(inline_text(&p.inlines)),
- _ => None,
- })
- .collect()
- }
-
- fn inline_text(inlines: &[Inline]) -> String {
- inlines
- .iter()
- .map(|inline| match inline {
- Inline::Text(t) => t.clone(),
- Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {
- inline_text(c)
- }
- _ => String::new(),
- })
- .collect()
- }
-
- #[test]
- fn ansicpg1251_hex_escapes() {
- let rtf = b"{\\rtf1\\ansi\\ansicpg1251 \\'d3\\'ea\\'f0\\'e0\\'bf\\'ed\\'e0\\par}";
- assert_eq!(paragraphs(rtf), vec!["Україна"]);
- }
-
- #[test]
- fn default_cp1252_hex_escapes() {
- let rtf = b"{\\rtf1\\ansi Caf\\'e9 \\'93ok\\'94\\par}";
- assert_eq!(paragraphs(rtf), vec!["Café \u{201C}ok\u{201D}"]);
- }
-
- #[test]
- fn unicode_escape_with_fallback() {
- let rtf = b"{\\rtf1\\ansi\\uc1 \\u1059?\\u1082?\\u1088? ok\\par}";
- assert_eq!(paragraphs(rtf), vec!["Укр ok"]);
- }
-
- #[test]
- fn shift_jis_double_byte_pairs() {
- let rtf = b"{\\rtf1\\ansi\\ansicpg932 \\'93\\'fa\\'96\\'7b\\par}";
- assert_eq!(paragraphs(rtf), vec!["日本"]);
- }
-
- #[test]
- fn styles_and_tables_still_parse() {
- let rtf =
- b"{\\rtf1\\ansi {\\b Bold} and {\\i italic}\\par\\trowd\\intbl A\\cell B\\cell\\row\\par}";
- let document = RtfProvider::new().parse_buffer(rtf).unwrap();
- assert!(matches!(document.blocks[0], Block::Paragraph(_)));
- assert!(document
- .blocks
- .iter()
- .any(|b| matches!(b, Block::Table(t) if t.rows[0].cells.len() == 2)));
- }
-
- #[test]
- fn info_metadata_decodes_with_document_codepage() {
- let rtf = b"{\\rtf1\\ansi\\ansicpg1251{\\info{\\title \\'d3\\'ea\\'e0\\'e7}{\\author Iryna}{\\creatim\\yr2022\\mo6\\dy9\\hr15\\min24}}Body\\par}";
- let document = RtfProvider::new().parse_buffer(rtf).unwrap();
- assert_eq!(document.metadata.title.as_deref(), Some("Указ"));
- assert_eq!(document.metadata.author.as_deref(), Some("Iryna"));
- assert_eq!(
- document.metadata.created.unwrap().to_rfc3339(),
- "2022-06-09T15:24:00+00:00"
- );
- }
-
- #[test]
- fn metadata_keeps_text_of_nested_formatting_groups() {
- let rtf = b"{\\rtf1{\\info{\\title Some {\\b Bold} Title{\\*\\junk secret}}}Body\\par}";
- let document = RtfProvider::new().parse_buffer(rtf).unwrap();
- assert_eq!(document.metadata.title.as_deref(), Some("Some Bold Title"));
- }
-
- #[test]
- fn par_inside_ignored_destination_does_not_split_paragraphs() {
- let rtf = b"{\\rtf1\\ansi Hello{\\*\\junkdest gone\\par gone too} world\\par}";
- assert_eq!(paragraphs(rtf), vec!["Hello world"]);
- }
-
- #[test]
- fn malformed_hex_escape_does_not_truncate_document() {
- let rtf = b"{\\rtf1\\ansi before \\'zz after\\par}";
- assert_eq!(paragraphs(rtf), vec!["before zz after"]);
- }
-
- #[test]
- fn escaped_ansicpg_literal_does_not_change_encoding() {
- let rtf = b"{\\rtf1\\ansi text \\\\ansicpg1251 \\'e9\\par}";
- assert_eq!(paragraphs(rtf), vec!["text \\ansicpg1251 é"]);
- }
-
- #[test]
- fn overflowing_control_parameter_digits_do_not_leak() {
- let rtf = b"{\\rtf1\\ansi a\\u99999999999999x b\\par}";
- assert_eq!(paragraphs(rtf), vec!["ax b"]);
- }
-
- #[test]
- fn uc_above_eight_skips_declared_fallback_bytes() {
- let rtf = b"{\\rtf1\\ansi\\uc10\\u1059 abcdefghijkl\\par}";
- assert_eq!(paragraphs(rtf), vec!["Уkl"]);
- }
-
- #[test]
- fn huge_uc_value_stays_bounded_by_input() {
- let rtf = b"{\\rtf1\\ansi\\uc2000000000\\u1059 abc\\par done\\par}";
- assert_eq!(paragraphs(rtf), vec!["У", "done"]);
- }
-}
diff --git a/apps/api/native/src/document/providers/xlsx.rs b/apps/api/native/src/document/providers/xlsx.rs
deleted file mode 100644
index 896cb1ae1..000000000
--- a/apps/api/native/src/document/providers/xlsx.rs
+++ /dev/null
@@ -1,87 +0,0 @@
-use crate::document::model::*;
-use crate::document::providers::DocumentProvider;
-use calamine::{open_workbook_auto_from_rs, Data, Reader};
-use std::error::Error;
-use std::io::Cursor;
-use std::num::NonZeroU32;
-
-const ONE: NonZeroU32 = unsafe { NonZeroU32::new_unchecked(1) };
-
-pub struct XlsxProvider;
-
-impl XlsxProvider {
- pub fn new() -> Self {
- Self
- }
-}
-
-impl DocumentProvider for XlsxProvider {
- fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {
- let cursor = Cursor::new(data);
- let mut workbook = open_workbook_auto_from_rs(cursor)?;
-
- let mut blocks: Vec<Block> = Vec::new();
-
- for sheet_name in workbook.sheet_names() {
- // Add sheet heading
- blocks.push(Block::Paragraph(Paragraph {
- kind: ParagraphKind::Heading(2),
- inlines: vec![Inline::Text(sheet_name.clone())],
- }));
-
- if let Ok(range) = workbook.worksheet_range(&sheet_name) {
- let mut rows: Vec<TableRow> = Vec::new();
- for r in range.rows() {
- let mut cells: Vec<TableCell> = Vec::new();
- for cell in r {
- let text = data_type_to_string(cell);
- let blocks_in_cell = if text.trim().is_empty() {
- Vec::new()
- } else {
- vec![Block::Paragraph(Paragraph {
- kind: ParagraphKind::Normal,
- inlines: vec![Inline::Text(text)],
- })]
- };
- cells.push(TableCell {
- blocks: blocks_in_cell,
- colspan: ONE,
- rowspan: ONE,
- });
- }
- rows.push(TableRow {
- cells,
- kind: TableRowKind::Body,
- });
- }
-
- blocks.push(Block::Table(Table { rows }));
- }
- }
-
- Ok(Document {
- blocks,
- metadata: DocumentMetadata::default(),
- notes: Vec::new(),
- comments: Vec::new(),
- })
- }
-
- fn name(&self) -> &'static str {
- "xlsx"
- }
-}
-
-fn data_type_to_string(cell: &Data) -> String {
- match cell {
- Data::Empty => String::new(),
- Data::String(s) => s.clone(),
- Data::Float(f) => f.to_string(),
- Data::Int(i) => i.to_string(),
- Data::Bool(b) => b.to_string(),
- Data::DateTime(v) => v.to_string(),
- Data::DateTimeIso(v) => v.to_string(),
- Data::DurationIso(v) => v.to_string(),
- Data::Error(e) => format!("#ERROR({e:?})"),
- }
-}
diff --git a/apps/api/native/src/document/renderers/html.rs b/apps/api/native/src/document/renderers/html.rs
deleted file mode 100644
index ea6582400..000000000
--- a/apps/api/native/src/document/renderers/html.rs
+++ /dev/null
@@ -1,237 +0,0 @@
-use crate::document::model::*;
-use maud::{html, Markup, DOCTYPE};
-
-pub struct HtmlRenderer;
-
-impl HtmlRenderer {
- pub fn new() -> Self {
- Self
- }
-
- pub fn render(&self, document: &Document) -> String {
- let title = document.metadata.title.as_deref().unwrap_or("Document");
-
- let footnotes: Vec<&Note> = document
- .notes
- .iter()
- .filter(|n| matches!(n.kind, NoteKind::Footnote))
- .collect();
-
- let endnotes: Vec<&Note> = document
- .notes
- .iter()
- .filter(|n| matches!(n.kind, NoteKind::Endnote))
- .collect();
-
- let author = document.metadata.author.as_deref();
- let page: Markup = html! {
- (DOCTYPE)
- html lang="en" {
- head {
- meta charset="UTF-8";
- meta name="viewport" content="width=device-width, initial-scale=1.0";
- title { (title) }
- @if let Some(author) = author {
- meta name="author" content=(author);
- }
- }
- body {
- main { (self.render_blocks(&document.blocks)) }
-
- @if !footnotes.is_empty() {
- section id="footnotes" {
- h2 { "Footnotes" }
- @for footnote in &footnotes {
- div id={ "footnote-" (&footnote.id.0) } {
- (self.render_blocks(&footnote.blocks))
- }
- }
- }
- }
-
- @if !endnotes.is_empty() {
- section id="endnotes" {
- h2 { "Endnotes" }
- @for endnote in &endnotes {
- div id={ "endnote-" (&endnote.id.0) } {
- (self.render_blocks(&endnote.blocks))
- }
- }
- }
- }
-
- @if !document.comments.is_empty() {
- section id="comments" {
- h2 { "Comments" }
- @for comment in &document.comments {
- article id={ "comment-" (&comment.id.0) } {
- @if let Some(author) = &comment.author_name {
- header {
- (author)
- @if let Some(initials) = &comment.author_initials {
- " (" (initials) ")"
- }
- }
- }
- (self.render_blocks(&comment.blocks))
- }
- }
- }
- }
- }
- }
- };
-
- page.into_string()
- }
-
- fn render_blocks(&self, blocks: &[Block]) -> Markup {
- html! {
- @for b in blocks {
- @match b {
- Block::Paragraph(p) => { (self.render_paragraph(p)) }
- Block::Table(t) => { (self.render_table(t)) }
- Block::List(l) => { (self.render_list(l)) }
- Block::Image(i) => { (self.render_image(i)) }
- }
- }
- }
- }
-
- fn render_blocks_inline(&self, blocks: &[Block]) -> Markup {
- if blocks.len() == 1 {
- if let Block::Paragraph(p) = &blocks[0] {
- if matches!(p.kind, ParagraphKind::Normal) {
- return self.render_inlines(&p.inlines);
- }
- }
- }
-
- self.render_blocks(blocks)
- }
-
- fn render_paragraph(&self, p: &Paragraph) -> Markup {
- match p.kind {
- ParagraphKind::Normal => html! { p { (self.render_inlines(&p.inlines)) } },
- ParagraphKind::Blockquote => html! {
- blockquote { p { (self.render_inlines(&p.inlines)) } }
- },
- ParagraphKind::Heading(level) => match level {
- 1 => html! { h1 { (self.render_inlines(&p.inlines)) } },
- 2 => html! { h2 { (self.render_inlines(&p.inlines)) } },
- 3 => html! { h3 { (self.render_inlines(&p.inlines)) } },
- 4 => html! { h4 { (self.render_inlines(&p.inlines)) } },
- 5 => html! { h5 { (self.render_inlines(&p.inlines)) } },
- _ => html! { h6 { (self.render_inlines(&p.inlines)) } },
- },
- }
- }
-
- fn render_table(&self, t: &Table) -> Markup {
- let mut head_rows = Vec::new();
- let mut body_rows = Vec::new();
- let mut foot_rows = Vec::new();
-
- for row in &t.rows {
- match row.kind {
- TableRowKind::Header => head_rows.push(row),
- TableRowKind::Body => body_rows.push(row),
- TableRowKind::Footer => foot_rows.push(row),
- }
- }
-
- html! {
- table {
- @if !head_rows.is_empty() {
- thead { @for row in head_rows { (self.render_table_row(row, true)) } }
- }
- tbody { @for row in body_rows { (self.render_table_row(row, false)) } }
- @if !foot_rows.is_empty() {
- tfoot { @for row in foot_rows { (self.render_table_row(row, false)) } }
- }
- }
- }
- }
-
- fn render_table_row(&self, row: &TableRow, header: bool) -> Markup {
- html! {
- tr {
- @for cell in &row.cells {
- @let cs = cell.colspan.get();
- @let rs = cell.rowspan.get();
- @let cs_attr = if cs > 1 { Some(cs) } else { None };
- @let rs_attr = if rs > 1 { Some(rs) } else { None };
-
- @if header {
- @if let (Some(cs), Some(rs)) = (cs_attr, rs_attr) {
- th colspan=(cs) rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }
- } @else if let Some(cs) = cs_attr {
- th colspan=(cs) { (self.render_blocks_inline(&cell.blocks)) }
- } @else if let Some(rs) = rs_attr {
- th rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }
- } @else {
- th { (self.render_blocks_inline(&cell.blocks)) }
- }
- } @else {
- @if let (Some(cs), Some(rs)) = (cs_attr, rs_attr) {
- td colspan=(cs) rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }
- } @else if let Some(cs) = cs_attr {
- td colspan=(cs) { (self.render_blocks_inline(&cell.blocks)) }
- } @else if let Some(rs) = rs_attr {
- td rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }
- } @else {
- td { (self.render_blocks_inline(&cell.blocks)) }
- }
- }
- }
- }
- }
- }
-
- fn render_list(&self, l: &List) -> Markup {
- match l.list_type {
- ListType::Ordered => html! {
- ol { @for item in &l.items { li { (self.render_blocks_inline(&item.blocks)) } } }
- },
- ListType::Unordered => html! {
- ul { @for item in &l.items { li { (self.render_blocks_inline(&item.blocks)) } } }
- },
- }
- }
-
- fn render_image(&self, i: &Image) -> Markup {
- match &i.alt {
- Some(alt) => html! { img src=(i.src) alt=(alt); },
- None => html! { img src=(i.src); },
- }
- }
-
- fn render_inlines(&self, inlines: &[Inline]) -> Markup {
- html! { @for inline in inlines { (self.render_inline(inline)) } }
- }
-
- fn render_inline(&self, inline: &Inline) -> Markup {
- match inline {
- Inline::Text(t) => html! { (t) },
- Inline::LineBreak => html! { br; },
-
- Inline::Link { href, children } => {
- html! { a href=(href) { (self.render_inlines(children)) } }
- }
-
- Inline::Strong(children) => html! { strong { (self.render_inlines(children)) } },
- Inline::Em(children) => html! { em { (self.render_inlines(children)) } },
- Inline::Del(children) => html! { del { (self.render_inlines(children)) } },
- Inline::Code(code) => html! { code { (code) } },
- Inline::Sup(children) => html! { sup { (self.render_inlines(children)) } },
- Inline::Sub(children) => html! { sub { (self.render_inlines(children)) } },
-
- Inline::FootnoteRef(id) => {
- html! { sup { a href={ "#footnote-" (&id.0) } { (&id.0) } } }
- }
- Inline::EndnoteRef(id) => html! { sup { a href={ "#endnote-" (&id.0) } { (&id.0) } } },
- Inline::CommentRef(id) => html! { a href={ "#comment-" (&id.0) } { "💬" } },
- Inline::Bookmark(id) => html! { a id=(&id.0) {} },
- }
- }
-}
diff --git a/apps/api/native/src/document/renderers/mod.rs b/apps/api/native/src/document/renderers/mod.rs
deleted file mode 100644
index 3d4613dd4..000000000
--- a/apps/api/native/src/document/renderers/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod html;
diff --git a/apps/api/native/src/document/xml.rs b/apps/api/native/src/document/xml.rs
deleted file mode 100644
index 45ce1cec7..000000000
--- a/apps/api/native/src/document/xml.rs
+++ /dev/null
@@ -1,49 +0,0 @@
-//! Helpers shared by the zip+XML based providers (docx, odt).
-
-use roxmltree::Node;
-use std::io::{Read, Seek};
-use zip::read::ZipArchive;
-
-pub fn read_zip_text<R: Read + Seek>(zip: &mut ZipArchive<R>, path: &str) -> Option<String> {
- let mut file = zip.by_name(path).ok()?;
- let mut s = String::new();
- file.read_to_string(&mut s).ok()?;
- Some(s)
-}
-
-pub fn strip_bom(s: &str) -> &str {
- s.strip_prefix('\u{FEFF}').unwrap_or(s)
-}
-
-pub fn is_tag(node: &Node, local: &str) -> bool {
- node.is_element() && node.tag_name().name() == local
-}
-
-/// Attribute value matched by local name, ignoring the namespace prefix.
-pub fn attr<'a>(node: &Node<'a, 'a>, local: &str) -> Option<&'a str> {
- node
- .attributes()
- .find(|a| {
- let name = a.name();
- match name.rsplit_once(':') {
- Some((_, l)) => l == local,
- None => name == local,
- }
- })
- .map(|a| a.value())
-}
-
-pub fn child<'a>(node: &Node<'a, 'a>, local: &str) -> Option<Node<'a, 'a>> {
- node
- .children()
- .find(|n| n.is_element() && n.tag_name().name() == local)
-}
-
-pub fn children<'a, 'b>(
- node: &Node<'a, 'a>,
- local: &'b str,
-) -> impl Iterator<Item = Node<'a, 'a>> + use<'a, 'b> {
- node
- .children()
- .filter(move |n| n.is_element() && n.tag_name().name() == local)
-}
diff --git a/apps/api/native/src/lib.rs b/apps/api/native/src/lib.rs
index f440be0cf..9735f28c8 100644
--- a/apps/api/native/src/lib.rs
+++ b/apps/api/native/src/lib.rs
@@ -1,14 +1,13 @@
#![deny(clippy::all)]
pub use crate::crawler::*;
+pub use crate::document::*;
pub use crate::engpicker::*;
pub use crate::html::*;
pub use crate::logging::*;
pub use crate::pdf::*;
pub use crate::utils::*;
-pub use crate::document::{DocumentConverter, DocumentType};
-
mod crawler;
mod document;
mod engpicker;
diff --git a/apps/api/src/__tests__/snips/v2/document-converter.test.ts b/apps/api/src/__tests__/snips/v2/document-converter.test.ts
index 3ed176fec..1bab83848 100644
--- a/apps/api/src/__tests__/snips/v2/document-converter.test.ts
+++ b/apps/api/src/__tests__/snips/v2/document-converter.test.ts
@@ -1,60 +1,69 @@
import path from "path";
import fs from "fs";
-import { DocumentConverter, DocumentType } from "@mendable/firecrawl-rs";
+import { convertDocumentToMarkdown } from "@mendable/firecrawl-rs";
describe("Document Converter tests", () => {
const samplesDir = path.join(process.cwd(), "samples");
- const expectedHtmlBase = (documentText: string) =>
- `<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Document</title></head><body><main><p><strong>Hello!</strong></p><p>${documentText} file to test the Firecrawl Document Converter.</p><p><em>Italic</em></p><p><strong>Bold</strong></p><p>Underlined</p><p><del>Strikethrough</del></p><table><tbody><tr><td>Header 1</td><td>Header 2</td></tr><tr><td>Value 1</td><td>Value 2</td></tr></tbody></table></main></body></html>`;
+ const expectedMarkdownBase = (documentText: string) =>
+ `**Hello!**\n\n${documentText} file to test the Firecrawl Document Converter.\n\n*Italic*\n\n**Bold**\n\nUnderlined\n\n~~Strikethrough~~\n\n| | |\n| --- | --- |\n| Header 1 | Header 2 |\n| Value 1 | Value 2 |\n`;
const sampleFiles = [
- { file: "sample.docx", type: DocumentType.Docx, name: "DOCX" },
- { file: "sample.odt", type: DocumentType.Odt, name: "ODT" },
- { file: "sample.rtf", type: DocumentType.Rtf, name: "RTF" },
+ { file: "sample.docx", name: "DOCX" },
+ { file: "sample.odt", name: "ODT" },
+ { file: "sample.rtf", name: "RTF" },
];
- const converter = new DocumentConverter();
+ describe.each(sampleFiles)("$name document conversion", ({ file, name }) => {
+ const filePath = path.join(samplesDir, file);
- describe.each(sampleFiles)(
- "$name document conversion",
- ({ file, type, name }) => {
- const filePath = path.join(samplesDir, file);
-
- beforeAll(() => {
- if (!fs.existsSync(filePath)) {
- throw new Error(`Sample file ${filePath} does not exist`);
- }
- });
+ beforeAll(() => {
+ if (!fs.existsSync(filePath)) {
+ throw new Error(`Sample file ${filePath} does not exist`);
+ }
+ });
- it.concurrent(
- `should convert ${name} document and return expected HTML`,
- async () => {
- const fileBuffer = fs.readFileSync(filePath);
+ it.concurrent(
+ `should convert ${name} document and return expected markdown`,
+ async () => {
+ const fileBuffer = fs.readFileSync(filePath);
- const html = await converter.convertBufferToHtml(
- new Uint8Array(fileBuffer),
- type,
- );
+ // No extension hint: the format must be detected from the bytes
+ const markdown = convertDocumentToMarkdown(new Uint8Array(fileBuffer));
- expect(html).toBe(expectedHtmlBase(name));
- },
- 10000,
- );
- },
- );
+ expect(markdown).toBe(expectedMarkdownBase(name));
+ },
+ 10000,
+ );
+ });
describe("XLSX document conversion", () => {
- const expectedHtml = `<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Document</title></head><body><main><h2>Sheet1</h2><table><tbody><tr><td>sample file</td><td>test</td></tr><tr><td></td><td></td></tr><tr><td>Name</td><td>Price</td></tr><tr><td>iPhone</td><td>1000</td></tr><tr><td>iPad</td><td>800</td></tr><tr><td>Macbook</td><td>1200</td></tr></tbody></table><h2>Sheet2</h2><table><tbody><tr><td>other tab</td><td></td></tr><tr><td></td><td></td></tr><tr><td>Name</td><td>Price</td></tr><tr><td>ChatGPT</td><td>20</td></tr><tr><td>Claude</td><td>17</td></tr><tr><td>Perplexity</td><td>20</td></tr></tbody></table></main></body></html>`;
+ const expectedMarkdown = `## Sheet1\n\n| | |\n| --- | --- |\n| sample file | test |\n| | |\n| Name | Price |\n| iPhone | 1000 |\n| iPad | 800 |\n| Macbook | 1200 |\n\n## Sheet2\n\n| | |\n| --- | --- |\n| other tab | |\n| | |\n| Name | Price |\n| ChatGPT | 20 |\n| Claude | 17 |\n| Perplexity | 20 |\n`;
- it("should convert XLSX document and return expected HTML", async () => {
+ it("should convert XLSX document and return expected markdown", () => {
const filePath = path.join(samplesDir, "sample.xlsx");
const fileBuffer = fs.readFileSync(filePath);
- const html = await converter.convertBufferToHtml(
- new Uint8Array(fileBuffer),
- DocumentType.Xlsx,
+ const markdown = convertDocumentToMarkdown(new Uint8Array(fileBuffer));
+ expect(markdown).toBe(expectedMarkdown);
+ });
+ });
+
+ describe("CSV document conversion", () => {
+ it("should convert CSV to a markdown table using the extension hint", () => {
+ // CSV has no magic bytes, so the extension hint selects the parser
+ const csv = Buffer.from("Name,Price\niPhone,1000\niPad,800\n");
+ const markdown = convertDocumentToMarkdown(new Uint8Array(csv), ".csv");
+ expect(markdown).toBe(
+ "| | |\n| --- | --- |\n| Name | Price |\n| iPhone | 1000 |\n| iPad | 800 |\n",
);
- expect(html).toBe(expectedHtml);
+ });
+ });
+
+ describe("unrecognized content", () => {
+ it("should throw on content that is not a supported document", () => {
+ expect(() =>
+ convertDocumentToMarkdown(new Uint8Array([1, 2, 3, 4, 5])),
+ ).toThrow();
});
});
});
diff --git a/apps/api/src/controllers/v2/parse-upload.ts b/apps/api/src/controllers/v2/parse-upload.ts
index 9eaac0027..32a4d9e0d 100644
--- a/apps/api/src/controllers/v2/parse-upload.ts
+++ b/apps/api/src/controllers/v2/parse-upload.ts
@@ -9,7 +9,7 @@ import { logger as _logger } from "../../lib/logger";
import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
import { getRedisConnection } from "../../services/queue-service";
import { RequestWithAuth, UploadedParseFile } from "./types";
-import { detectUploadedFileKind } from "./parse";
+import { detectUploadedFileKind, SUPPORTED_PARSE_FILE_TYPES } from "./parse";
const PARSE_UPLOAD_MAX_BYTES = 50 * 1024 * 1024;
const PARSE_UPLOAD_TTL_MS = 10 * 60 * 1000;
@@ -273,8 +273,7 @@ export async function parseUploadUrlController(
return res.status(400).json({
success: false,
code: "UNSUPPORTED_FILE_TYPE",
- error:
- "Unsupported upload type. Supported file extensions include .html, .htm, .xhtml, .pdf, .docx, .doc, .odt, .rtf, .xlsx, .xls, or matching supported MIME types.",
+ error: `Unsupported upload type. Supported file extensions include ${SUPPORTED_PARSE_FILE_TYPES}, or matching supported MIME types.`,
});
}
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index eae34cff1..3ccc60735 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -36,19 +36,14 @@ import { projectScrapeCredits } from "../../lib/keyless-credit-projection";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";
import path from "node:path";
+import {
+ DOCUMENT_EXTENSIONS,
+ documentExtensionFromContentType,
+} from "../../lib/document-formats";
const AGENT_INTEROP_CONCURRENCY_BOOST = 3;
-const SUPPORTED_PARSE_FILE_TYPES =
- ".html, .htm, .pdf, .docx, .doc, .odt, .rtf, .xlsx, .xls";
-
-const DOCUMENT_EXTENSIONS = new Set([
- ".docx",
- ".doc",
- ".odt",
- ".rtf",
- ".xlsx",
- ".xls",
-]);
+export const SUPPORTED_PARSE_FILE_TYPES =
+ ".html, .htm, .xhtml, .pdf, .docx, .doc, .docm, .odt, .ods, .odp, .rtf, .xlsx, .xls, .xlsm, .xlsb, .pptx, .ppt, .pptm, .epub, .csv";
export function detectUploadedFileKind(
filename: string,
@@ -68,17 +63,7 @@ export function detectUploadedFileKind(
const isDocument =
DOCUMENT_EXTENSIONS.has(extension) ||
- normalizedType.includes(
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- ) ||
- normalizedType.includes("application/vnd.ms-excel") ||
- normalizedType.includes(
- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
- ) ||
- normalizedType.includes("application/msword") ||
- normalizedType.includes("application/vnd.oasis.opendocument.text") ||
- normalizedType.includes("application/rtf") ||
- normalizedType.includes("text/rtf");
+ documentExtensionFromContentType(normalizedType) !== null;
if (isDocument) {
return "document";
@@ -378,9 +363,9 @@ export async function parseController(
);
if (!reservation.ok) {
applyAgentAuthDiscoveryHeader(res);
- return res.status(429).json(
- await keylessLimitBody(req.auth.team_id, "v2_parse"),
- );
+ return res
+ .status(429)
+ .json(await keylessLimitBody(req.auth.team_id, "v2_parse"));
}
reservedKeylessCredits = projectedKeylessCredits;
}
diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.ts
index d9ef0bbdc..6b826f2dc 100644
--- a/apps/api/src/controllers/v2/research-proxy.ts
+++ b/apps/api/src/controllers/v2/research-proxy.ts
@@ -95,7 +95,7 @@ const booleanFlag = z
value === undefined ? undefined : value === true || value === "true",
);
-export const developerSearchSchema = z.strictObject({
+const developerSearchSchema = z.strictObject({
query: z.string().min(1),
k: kSchema(100),
types: multiString,
@@ -113,7 +113,7 @@ export const developerSearchSchema = z.strictObject({
...commonQuery,
});
-export const DEVELOPER_SEARCH_QUERY_KEYS = [
+const DEVELOPER_SEARCH_QUERY_KEYS = [
"query",
"k",
"types",
diff --git a/apps/api/src/lib/document-formats.ts b/apps/api/src/lib/document-formats.ts
new file mode 100644
index 000000000..bce80f84a
--- /dev/null
+++ b/apps/api/src/lib/document-formats.ts
@@ -0,0 +1,65 @@
+// Document formats handled by the document engine (anydoc). PDF is
+// intentionally absent — it has its own engine with OCR support.
+
+const CONTENT_TYPE_TO_EXTENSION = new Map<string, string>([
+ [
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ ".docx",
+ ],
+ ["application/vnd.ms-word.document.macroenabled.12", ".docm"],
+ ["application/msword", ".doc"],
+ [
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ ".xlsx",
+ ],
+ ["application/vnd.ms-excel", ".xls"],
+ ["application/vnd.ms-excel.sheet.macroenabled.12", ".xlsm"],
+ ["application/vnd.ms-excel.sheet.binary.macroenabled.12", ".xlsb"],
+ [
+ "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+ ".pptx",
+ ],
+ ["application/vnd.ms-powerpoint", ".ppt"],
+ ["application/vnd.ms-powerpoint.presentation.macroenabled.12", ".pptm"],
+ ["application/vnd.oasis.opendocument.text", ".odt"],
+ ["application/vnd.oasis.opendocument.spreadsheet", ".ods"],
+ ["application/vnd.oasis.opendocument.presentation", ".odp"],
+ ["application/rtf", ".rtf"],
+ ["text/rtf", ".rtf"],
+ ["application/epub+zip", ".epub"],
+ ["text/csv", ".csv"],
+]);
+
+export const DOCUMENT_EXTENSIONS = new Set(CONTENT_TYPE_TO_EXTENSION.values());
+
+// Longest first so ".docx" wins over ".doc" when matching URL paths.
+const EXTENSIONS_BY_LENGTH = [...DOCUMENT_EXTENSIONS].sort(
+ (a, b) => b.length - a.length,
+);
+
+export function documentExtensionFromContentType(
+ contentType: string | null | undefined,
+): string | null {
+ if (!contentType) return null;
+ const mediaType = contentType.split(";")[0].trim().toLowerCase();
+ return CONTENT_TYPE_TO_EXTENSION.get(mediaType) ?? null;
+}
+
+export function documentContentTypeFromExtension(
+ extension: string,
+): string | null {
+ const ext = extension.toLowerCase();
+ for (const [contentType, mapped] of CONTENT_TYPE_TO_EXTENSION) {
+ if (mapped === ext) return contentType;
+ }
+ return null;
+}
+
+// Matches extensions at the end of the path or mid-path (e.g. file.xlsx/hash).
+export function documentExtensionFromUrlPath(urlPath: string): string | null {
+ const lowerPath = urlPath.toLowerCase();
+ for (const ext of EXTENSIONS_BY_LENGTH) {
+ if (lowerPath.endsWith(ext) || lowerPath.includes(ext + "/")) return ext;
+ }
+ return null;
+}
diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.ts
index a7aeb0543..ef8281483 100644
--- a/apps/api/src/lib/keyless.ts
+++ b/apps/api/src/lib/keyless.ts
@@ -87,7 +87,7 @@ export function isKeylessIpEligible(ip: string): boolean {
const requestsKey = (ip: string) => `keyless_requests:${ip}`;
const creditsKey = (ip: string) => `keyless_credits:${ip}`;
-export type KeylessQuotaReason = "requests" | "credits";
+type KeylessQuotaReason = "requests" | "credits";
type KeylessConsumeResult = {
ok: boolean;
@@ -286,11 +286,14 @@ return next
* consumption). Used by the hosted MCP before a keyless tool call so an
* ineligible caller receives structured recovery without an OAuth challenge.
*/
-export async function checkKeylessEligibility(
- ip: string,
-): Promise<{
+export async function checkKeylessEligibility(ip: string): Promise<{
eligible: boolean;
- reason?: KeylessQuotaReason | "disabled" | "ineligible_ip" | "suspicious" | "error";
+ reason?:
+ | KeylessQuotaReason
+ | "disabled"
+ | "ineligible_ip"
+ | "suspicious"
+ | "error";
retryAfterSeconds?: number;
}> {
if (!isKeylessConfigured()) return { eligible: false, reason: "disabled" };
diff --git a/apps/api/src/lib/research-upstream.ts b/apps/api/src/lib/research-upstream.ts
index b24673e23..85a398735 100644
--- a/apps/api/src/lib/research-upstream.ts
+++ b/apps/api/src/lib/research-upstream.ts
@@ -9,7 +9,7 @@ const dispatcher = new Agent({
bodyTimeout: TIMEOUT_MS,
});
-export function appendQuery(
+function appendQuery(
url: URL,
params: Record<string, unknown>,
allowed: string[],
diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
index b3ac85a68..6bde2f0bf 100644
--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
@@ -1,114 +1,17 @@
import { Meta } from "../..";
import { EngineScrapeResult } from "..";
import { fetchFileToBuffer } from "../utils/downloadFile";
-import { DocumentConverter, DocumentType } from "@mendable/firecrawl-rs";
+import { convertDocumentToMarkdown } from "@mendable/firecrawl-rs";
+import { safeMarkdownToHtml } from "../pdf/markdownToHtml";
import type { Response } from "undici";
+import { DocumentAntibotError, EngineUnsuccessfulError } from "../../error";
import {
- DocumentAntibotError,
- DocumentPrefetchFailed,
- EngineUnsuccessfulError,
-} from "../../error";
+ documentContentTypeFromExtension,
+ documentExtensionFromContentType,
+ documentExtensionFromUrlPath,
+} from "../../../../lib/document-formats";
import { readFile, unlink } from "node:fs/promises";
-const converter = new DocumentConverter();
-
-function getDocumentTypeFromUrl(url: string): DocumentType {
- const urlLower = url.toLowerCase();
-
- // Check for extensions at the end or in the middle (e.g., file.xlsx/hash)
- // Check .docx before .doc to avoid false matches
- if (urlLower.endsWith(".docx") || urlLower.includes(".docx/"))
- return DocumentType.Docx;
- if (urlLower.endsWith(".doc") || urlLower.includes(".doc/"))
- return DocumentType.Doc;
- if (urlLower.endsWith(".odt") || urlLower.includes(".odt/"))
- return DocumentType.Odt;
- if (urlLower.endsWith(".rtf") || urlLower.includes(".rtf/"))
- return DocumentType.Rtf;
- if (
- urlLower.endsWith(".xlsx") ||
- urlLower.endsWith(".xls") ||
- urlLower.includes(".xlsx/") ||
- urlLower.includes(".xls/")
- )
- return DocumentType.Xlsx;
-
- return DocumentType.Docx; // hope for the best
-}
-
-function getDocumentTypeFromContentType(
- contentType: string | null,
-): DocumentType | null {
- if (!contentType) return null;
-
- const ct = contentType.toLowerCase();
-
- // Check for modern .docx format first (Office Open XML)
- if (
- ct.includes(
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- )
- ) {
- return DocumentType.Docx;
- }
-
- // Legacy .doc format (OLE2/CFB binary format)
- if (ct.includes("application/msword")) {
- return DocumentType.Doc;
- }
-
- if (ct.includes("application/vnd.oasis.opendocument.text")) {
- return DocumentType.Odt;
- }
-
- if (ct.includes("application/rtf") || ct.includes("text/rtf")) {
- return DocumentType.Rtf;
- }
-
- if (
- ct.includes(
- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
- ) ||
- ct.includes("application/vnd.ms-excel")
- ) {
- return DocumentType.Xlsx;
- }
-
- return null;
-}
-
-function getContentTypeFromDocumentType(documentType: DocumentType): string {
- switch (documentType) {
- case DocumentType.Docx:
- return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
- case DocumentType.Doc:
- return "application/msword";
- case DocumentType.Odt:
- return "application/vnd.oasis.opendocument.text";
- case DocumentType.Rtf:
- return "application/rtf";
- case DocumentType.Xlsx:
- return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
- }
-}
-
-function isValidDocumentContentType(contentType: string | null): boolean {
- if (!contentType) return false;
-
- const ct = contentType.toLowerCase();
- const validTypes = [
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
- "application/vnd.ms-excel",
- "application/msword",
- "application/rtf",
- "text/rtf",
- "application/vnd.oasis.opendocument.text",
- ];
-
- return validTypes.some(type => ct.includes(type));
-}
-
export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
let response: Response;
let buffer: Buffer;
@@ -149,7 +52,7 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
// Validate content type only when fetching directly (not using prefetch)
const ct = response.headers.get("Content-Type");
- if (ct && !isValidDocumentContentType(ct)) {
+ if (ct && documentExtensionFromContentType(ct) === null) {
// The server declared a non-document type and nothing asked for a
// document (no `document` flag means neither a document URL extension nor
// a caller-requested format), so this engine is just the tail of the
@@ -170,22 +73,25 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
}
}
- const documentType =
- getDocumentTypeFromContentType(response.headers.get("content-type")) ??
- getDocumentTypeFromUrl(response.url);
+ const extension =
+ documentExtensionFromContentType(response.headers.get("content-type")) ??
+ documentExtensionFromUrlPath(new URL(response.url).pathname);
- const html = await converter.convertBufferToHtml(
+ const markdown = await convertDocumentToMarkdown(
new Uint8Array(buffer),
- documentType,
+ extension ?? undefined,
);
+ const html = await safeMarkdownToHtml(markdown, meta.logger, meta.id);
return {
url: response.url,
statusCode: response.status,
html,
+ markdown,
contentType:
response.headers.get("content-type") ??
- getContentTypeFromDocumentType(documentType),
+ (extension ? documentContentTypeFromExtension(extension) : null) ??
+ "application/octet-stream",
proxyUsed,
};
} finally {
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
index 464218f12..749454c1e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
@@ -5,6 +5,7 @@ import path from "path";
import os from "os";
import { writeFile } from "fs/promises";
import { Meta } from "../..";
+import { documentExtensionFromContentType } from "../../../../lib/document-formats";
async function feResToFilePrefetch(
logger: Logger,
@@ -46,23 +47,8 @@ async function feResToDocumentPrefetch(
contentType: string,
): Promise<Meta["documentPrefetch"]> {
// Determine file extension from content type
- let extension = "tmp";
- if (contentType.includes("wordprocessingml")) {
- // Modern .docx format (Office Open XML)
- extension = "docx";
- } else if (contentType.includes("msword")) {
- // Legacy .doc format (OLE2/CFB binary)
- extension = "doc";
- } else if (
- contentType.includes("spreadsheetml") ||
- contentType.includes("ms-excel")
- ) {
- extension = "xlsx";
- } else if (contentType.includes("opendocument.text")) {
- extension = "odt";
- } else if (contentType.includes("rtf")) {
- extension = "rtf";
- }
+ const extension =
+ documentExtensionFromContentType(contentType)?.slice(1) ?? "tmp";
return feResToFilePrefetch(logger, feRes, extension, "document", contentType);
}
@@ -83,17 +69,7 @@ export async function specialtyScrapeCheck(
return;
}
- const documentTypes = [
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
- "application/vnd.ms-excel",
- "application/msword",
- "application/rtf",
- "text/rtf",
- "application/vnd.oasis.opendocument.text",
- ];
-
- const isDocument = documentTypes.some(type => contentType.startsWith(type));
+ const isDocument = documentExtensionFromContentType(contentType) !== null;
const isPdf =
contentType === "application/pdf" ||
contentType.startsWith("application/pdf;");
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 029336c21..8d1a154e8 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -82,6 +82,12 @@ import {
import { htmlTransform } from "./lib/removeUnwantedElements";
import { postprocessors } from "./postprocessors";
import { rewriteUrl } from "./lib/rewriteUrl";
+import {
+ DOCUMENT_EXTENSIONS,
+ documentContentTypeFromExtension,
+ documentExtensionFromContentType,
+ documentExtensionFromUrlPath,
+} from "../../lib/document-formats";
import { writeFile } from "node:fs/promises";
import path from "node:path";
import { tmpdir } from "node:os";
@@ -245,19 +251,7 @@ function buildFeatureFlags(
const lowerPath = urlO.pathname.toLowerCase();
// Check for document types first (they take precedence over PDF)
- const isDocument =
- lowerPath.endsWith(".docx") ||
- lowerPath.endsWith(".odt") ||
- lowerPath.endsWith(".rtf") ||
- lowerPath.endsWith(".xlsx") ||
- lowerPath.endsWith(".xls") ||
- lowerPath.includes(".docx/") ||
- lowerPath.includes(".odt/") ||
- lowerPath.includes(".rtf/") ||
- lowerPath.includes(".xlsx/") ||
- lowerPath.includes(".xls/");
-
- if (isDocument) {
+ if (documentExtensionFromUrlPath(lowerPath) !== null) {
flags.add("document");
} else if (lowerPath.endsWith(".pdf") || lowerPath.includes(".pdf/")) {
// Only add PDF flag if it's not a document
@@ -276,15 +270,6 @@ function buildFeatureFlags(
// The meta object is usually immutable, except for the logs array, and in edge cases (e.g. a new feature is suddenly required)
// Having a meta object that is treated as immutable helps the code stay clean and easily tracable,
// while also retaining the benefits that WebScraper had from its OOP design.
-const DOCUMENT_EXTENSIONS = new Set([
- ".docx",
- ".doc",
- ".odt",
- ".rtf",
- ".xlsx",
- ".xls",
-]);
-
const HTML_EXTENSIONS = new Set([".html", ".htm", ".xhtml"]);
async function writeUploadedFileToTemp(
@@ -314,20 +299,9 @@ function isPdfUpload(filename: string, contentType?: string): boolean {
function isDocumentUpload(filename: string, contentType?: string): boolean {
const ext = path.extname(filename).toLowerCase();
- const normalizedType = contentType?.toLowerCase() ?? "";
return (
DOCUMENT_EXTENSIONS.has(ext) ||
- normalizedType.includes(
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- ) ||
- normalizedType.includes("application/vnd.ms-excel") ||
- normalizedType.includes(
- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
- ) ||
- normalizedType.includes("application/msword") ||
- normalizedType.includes("application/vnd.oasis.opendocument.text") ||
- normalizedType.includes("application/rtf") ||
- normalizedType.includes("text/rtf")
+ documentExtensionFromContentType(contentType) !== null
);
}
@@ -419,6 +393,7 @@ async function buildMetaObject(
proxyUsed: "basic",
contentType:
contentType ||
+ documentContentTypeFromExtension(fallbackExtension) ||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
};
} else if (isHtmlUpload(filename, contentType)) {