Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): replace custom document parser with anydoc (#4225)
tomsideguideaebeba7f9adf
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.tomlindex 4fbe8afb0..f5ee9a40c 100644--- a/apps/api/native/Cargo.toml+++ b/apps/api/native/Cargo.toml@@ -9,11 +9,11 @@ version = "0.1.0"crate-type = ["cdylib"][dependencies]+anydoc = "0.1.2"chrono = { version = "0.4", features = ["serde"] }kuchikiki = "0.8.2"lol_html = "2.6.0"pdf-inspector = "0.1.0"-maud = "0.27.0"napi = { version = "3.0.0", features = ["serde-json", "tokio_rt"] }napi-derive = "3.0.0"nodesig = { git = "https://github.com/firecrawl/nodesig" }@@ -25,11 +25,6 @@ serde_json = "1.0.143"strsim = "0.11"texting_robots = "0.2.2"url = "2.5.7"-zip = "5.0.0"-calamine = { git = "https://github.com/firecrawl/calamine", branch = "fc-prod" }-cfb = "0.10"-codepage = "0.1"-encoding_rs = "0.8"tokio = "1.48.0"tracing = "0.1"tracing-subscriber = { version = "0.3", default-features = false, features = ["registry"] }diff --git a/apps/api/native/src/document.rs b/apps/api/native/src/document.rsnew file mode 100644index 000000000..f019113d2--- /dev/null+++ b/apps/api/native/src/document.rs@@ -0,0 +1,22 @@+use napi::bindgen_prelude::*;+use napi_derive::napi;++/// Convert a document (doc, docx, odt/ods/odp, rtf, xls/xlsx, ppt/pptx, epub,+/// csv, ...) to GitHub-Flavored Markdown using anydoc. The format is detected+/// from the file content; `extension_hint` (with or without a leading dot) is+/// only consulted for signature-less formats like CSV.+#[napi]+pub fn convert_document_to_markdown(+ data: &[u8],+ extension_hint: Option<String>,+) -> Result<String> {+ let format = anydoc::Format::from_bytes(data).or_else(|| {+ extension_hint+ .as_deref()+ .map(|ext| ext.trim_start_matches('.'))+ .and_then(anydoc::Format::from_extension)+ });++ anydoc::to_markdown_bytes(data, format)+ .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))+}diff --git a/apps/api/native/src/document/encoding.rs b/apps/api/native/src/document/encoding.rsdeleted file mode 100644index 2b107feaa..000000000--- a/apps/api/native/src/document/encoding.rs+++ /dev/null@@ -1,38 +0,0 @@-use encoding_rs::{Encoding, WINDOWS_1252};--/// Encoding for a Windows code page, excluding the Unicode pseudo code-/// pages.-pub fn encoding_for_codepage(codepage: u16) -> Option<&'static Encoding> {- match codepage {- 0 | 1200 | 1201 | 65000 | 65001 => None,- cp => codepage::to_encoding(cp),- }-}--/// Default ANSI code page encoding for a Windows language ID.-pub fn encoding_for_lid(lid: u16) -> &'static Encoding {- let codepage = match lid {- // sublanguage decides the script for these- 0x0404 | 0x0C04 | 0x1404 | 0x7C04 => 950, // Chinese (Taiwan, Hong Kong, Macau, Traditional)- 0x0C1A | 0x1C1A => 1251, // Serbian (Cyrillic)- 0x082C | 0x0843 => 1251, // Azerbaijani, Uzbek (Cyrillic)- _ => match lid & 0x3FF {- 0x04 => 936, // Chinese (Simplified unless a Traditional sublanguage above)- // Central European- 0x05 | 0x0E | 0x15 | 0x18 | 0x1A | 0x1B | 0x1C | 0x24 => 1250,- // Cyrillic- 0x02 | 0x19 | 0x22 | 0x23 | 0x2F | 0x3F | 0x40 | 0x44 | 0x50 => 1251,- 0x08 => 1253, // Greek- 0x1F | 0x2C | 0x43 => 1254, // Turkish, Azerbaijani, Uzbek (Latin)- 0x0D => 1255, // Hebrew- 0x01 | 0x20 | 0x29 => 1256, // Arabic, Urdu, Persian- 0x25 | 0x26 | 0x27 => 1257, // Baltic- 0x2A => 1258, // Vietnamese- 0x1E => 874, // Thai- 0x11 => 932, // Japanese- 0x12 => 949, // Korean- _ => 1252,- },- };- encoding_for_codepage(codepage).unwrap_or(WINDOWS_1252)-}diff --git a/apps/api/native/src/document/mod.rs b/apps/api/native/src/document/mod.rsdeleted file mode 100644index 278c8c3c4..000000000--- a/apps/api/native/src/document/mod.rs+++ /dev/null@@ -1,53 +0,0 @@-pub mod encoding;-pub mod model;-pub mod oleps;-pub mod providers;-pub mod renderers;-pub mod xml;--pub use providers::factory::DocumentType;--use crate::document::model::Document;-use crate::document::providers::factory::ProviderFactory;-use crate::document::renderers::html::HtmlRenderer;-use napi::bindgen_prelude::*;-use napi_derive::napi;--#[napi]-pub struct DocumentConverter {- factory: ProviderFactory,- html_renderer: HtmlRenderer,-}--impl Default for DocumentConverter {- fn default() -> Self {- Self::new()- }-}--#[napi]-impl DocumentConverter {- #[napi(constructor)]- pub fn new() -> Self {- Self {- factory: ProviderFactory::new(),- html_renderer: HtmlRenderer::new(),- }- }-- #[napi]- pub fn convert_buffer_to_html(- &self,- data: &[u8],- doc_type: DocumentType,- ) -> napi::Result<String> {- let provider = self.factory.get_provider(doc_type);-- let document: Document = provider- .parse_buffer(data)- .map_err(|e| Error::new(Status::GenericFailure, format!("Provider error: {e}")))?;-- let html = self.html_renderer.render(&document);- Ok(html)- }-}diff --git a/apps/api/native/src/document/model/mod.rs b/apps/api/native/src/document/model/mod.rsdeleted file mode 100644index 444acb88e..000000000--- a/apps/api/native/src/document/model/mod.rs+++ /dev/null@@ -1,150 +0,0 @@-use chrono::{DateTime, Utc};-use std::num::NonZeroU32;--#[derive(Debug, Clone)]-pub struct Document {- pub blocks: Vec<Block>,- pub metadata: DocumentMetadata,- pub notes: Vec<Note>,- pub comments: Vec<Comment>,-}--#[derive(Debug, Clone, Default)]-pub struct DocumentMetadata {- pub title: Option<String>,- pub author: Option<String>,- pub created: Option<DateTime<Utc>>,-}--#[derive(Debug, Clone, PartialEq, Eq, Hash)]-pub struct NoteId(pub String);--#[derive(Debug, Clone, PartialEq, Eq, Hash)]-pub struct CommentId(pub String);--#[derive(Debug, Clone, PartialEq, Eq, Hash)]-pub struct BookmarkId(pub String);--#[derive(Debug, Clone)]-pub enum Block {- Paragraph(Paragraph),- Table(Table),- List(List),- Image(Image),-}--#[derive(Debug, Clone)]-pub struct Paragraph {- pub kind: ParagraphKind,- pub inlines: Vec<Inline>,-}--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum ParagraphKind {- Normal,- Heading(u8), // 1..=6 will render as <h1>.. <h6>- Blockquote,-}--#[derive(Debug, Clone)]-pub enum Inline {- Text(String),- LineBreak,- Link { href: String, children: Vec<Inline> },-- Strong(Vec<Inline>),- Em(Vec<Inline>),- Del(Vec<Inline>),- Code(String),- Sup(Vec<Inline>),- Sub(Vec<Inline>),-- FootnoteRef(NoteId),- EndnoteRef(NoteId),- CommentRef(CommentId),- Bookmark(BookmarkId),-}--/// True when the inlines would render something the reader can see.-pub fn has_visible_content(inlines: &[Inline]) -> bool {- inlines.iter().any(|inline| match inline {- Inline::Text(t) => !t.trim().is_empty(),- Inline::LineBreak => false,- Inline::Link { children, .. } => has_visible_content(children),- Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {- has_visible_content(c)- }- Inline::Code(t) => !t.trim().is_empty(),- Inline::FootnoteRef(_) | Inline::EndnoteRef(_) | Inline::CommentRef(_) => true,- Inline::Bookmark(_) => false,- })-}--#[derive(Debug, Clone)]-pub struct Image {- pub src: String,- pub alt: Option<String>,-}--#[derive(Debug, Clone)]-pub struct Table {- pub rows: Vec<TableRow>,-}--#[derive(Debug, Clone)]-pub struct TableRow {- pub cells: Vec<TableCell>,- pub kind: TableRowKind,-}--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum TableRowKind {- Header,- Body,- Footer,-}--#[derive(Debug, Clone)]-pub struct TableCell {- pub blocks: Vec<Block>,- pub colspan: NonZeroU32,- pub rowspan: NonZeroU32,-}--#[derive(Debug, Clone)]-pub struct List {- pub items: Vec<ListItem>,- pub list_type: ListType,-}--#[derive(Debug, Clone)]-pub struct ListItem {- pub blocks: Vec<Block>,-}--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum ListType {- Ordered,- Unordered,-}--#[derive(Debug, Clone)]-pub struct Note {- pub id: NoteId,- pub kind: NoteKind,- pub blocks: Vec<Block>,-}--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum NoteKind {- Footnote,- Endnote,-}--#[derive(Debug, Clone)]-pub struct Comment {- pub id: CommentId,- pub author_name: Option<String>,- pub author_initials: Option<String>,- pub blocks: Vec<Block>,-}diff --git a/apps/api/native/src/document/oleps.rs b/apps/api/native/src/document/oleps.rsdeleted file mode 100644index 7c2f09ce3..000000000--- a/apps/api/native/src/document/oleps.rs+++ /dev/null@@ -1,136 +0,0 @@-//! Minimal MS-OLEPS parser for the `\x05SummaryInformation` stream of OLE-//! compound files.--use crate::document::encoding::encoding_for_codepage;-use chrono::{DateTime, Utc};-use encoding_rs::WINDOWS_1252;--const PID_CODEPAGE: u32 = 0x0001;-const PIDSI_TITLE: u32 = 0x0002;-const PIDSI_AUTHOR: u32 = 0x0004;-const PIDSI_CREATE_DTM: u32 = 0x000C;--const VT_I2: u32 = 0x0002;-const VT_LPSTR: u32 = 0x001E;-const VT_LPWSTR: u32 = 0x001F;-const VT_FILETIME: u32 = 0x0040;--/// Seconds between the FILETIME epoch (1601) and the Unix epoch (1970).-const FILETIME_UNIX_EPOCH_DIFF_SECS: i64 = 11_644_473_600;--#[derive(Debug, Default)]-pub struct SummaryInformation {- pub codepage: Option<u16>,- pub title: Option<String>,- pub author: Option<String>,- pub created: Option<DateTime<Utc>>,-}--pub fn parse_summary_information(data: &[u8]) -> Option<SummaryInformation> {- // byte order mark at 0, property set count at 0x18, first section offset at 0x2C- if read_u16(data, 0)? != 0xFFFE {- return None;- }- if read_u32(data, 0x18)? == 0 {- return None;- }- let section_start = read_u32(data, 0x2C)? as usize;-- let num_properties = (read_u32(data, section_start + 4)? as usize).min(1024);- let mut properties = Vec::with_capacity(num_properties);- for i in 0..num_properties {- let entry = section_start + 8 + i * 8;- let prop_id = read_u32(data, entry)?;- let prop_offset = section_start + read_u32(data, entry + 4)? as usize;- properties.push((prop_id, prop_offset));- }-- let mut info = SummaryInformation::default();-- // code page first, it governs VT_LPSTR decoding- info.codepage = properties- .iter()- .find(|(id, _)| *id == PID_CODEPAGE)- .and_then(|(_, offset)| {- if read_u32(data, *offset)? != VT_I2 {- return None;- }- read_u16(data, offset + 4)- });- let string_encoding = info- .codepage- .and_then(encoding_for_codepage)- .unwrap_or(WINDOWS_1252);-- for (prop_id, offset) in properties {- match prop_id {- PIDSI_TITLE => info.title = read_string(data, offset, string_encoding),- PIDSI_AUTHOR => info.author = read_string(data, offset, string_encoding),- PIDSI_CREATE_DTM => info.created = read_filetime(data, offset),- _ => {}- }- }-- Some(info)-}--fn read_string(- data: &[u8],- offset: usize,- encoding: &'static encoding_rs::Encoding,-) -> Option<String> {- let vt = read_u32(data, offset)?;- let len = read_u32(data, offset + 4)? as usize;- let text = match vt {- VT_LPSTR => {- let bytes = data.get(offset + 8..offset + 8 + len)?;- let (decoded, _, _) = encoding.decode(bytes);- decoded.into_owned()- }- VT_LPWSTR => {- let bytes = data.get(offset + 8..offset + 8 + len.checked_mul(2)?)?;- let units: Vec<u16> = bytes- .chunks_exact(2)- .map(|c| u16::from_le_bytes([c[0], c[1]]))- .collect();- String::from_utf16_lossy(&units)- }- _ => return None,- };- let text = text.trim_end_matches('\0').trim();- if text.is_empty() {- None- } else {- Some(text.to_string())- }-}--fn read_filetime(data: &[u8], offset: usize) -> Option<DateTime<Utc>> {- if read_u32(data, offset)? != VT_FILETIME {- return None;- }- let filetime = read_u64(data, offset + 4)?;- if filetime == 0 {- return None;- }- let secs = (filetime / 10_000_000) as i64 - FILETIME_UNIX_EPOCH_DIFF_SECS;- let nanos = (filetime % 10_000_000) as u32 * 100;- DateTime::from_timestamp(secs, nanos)-}--fn read_u16(data: &[u8], offset: usize) -> Option<u16> {- let bytes = data.get(offset..offset + 2)?;- Some(u16::from_le_bytes([bytes[0], bytes[1]]))-}--fn read_u32(data: &[u8], offset: usize) -> Option<u32> {- let bytes = data.get(offset..offset + 4)?;- Some(u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]))-}--fn read_u64(data: &[u8], offset: usize) -> Option<u64> {- let bytes = data.get(offset..offset + 8)?;- Some(u64::from_le_bytes([- bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7],- ]))-}diff --git a/apps/api/native/src/document/providers/doc.rs b/apps/api/native/src/document/providers/doc.rsdeleted file mode 100644index 7fd10a85e..000000000--- a/apps/api/native/src/document/providers/doc.rs+++ /dev/null@@ -1,1501 +0,0 @@-//! Legacy Word binary (.doc) provider, following the MS-DOC piece table and-//! formatting bin table (FKP) formats.--use crate::document::encoding::{encoding_for_codepage, encoding_for_lid};-use crate::document::model::*;-use crate::document::oleps::parse_summary_information;-use crate::document::providers::DocumentProvider;-use cfb::CompoundFile;-use encoding_rs::Encoding;-use std::error::Error;-use std::io::{Cursor, Read, Seek};-use std::num::NonZeroU32;--const MAGIC_WORD_97: u16 = 0xA5EC;-const MAGIC_WORD_6_95: u16 = 0xA5DC;--/// Upper bound on text extracted from one document.-const MAX_TEXT_CHARS: usize = 10_000_000;--const FKP_PAGE_SIZE: usize = 512;-const MAX_FIELD_INSTRUCTION_CHARS: usize = 2048;--pub struct DocProvider;--impl DocProvider {- pub fn new() -> Self {- Self- }-}--impl DocumentProvider for DocProvider {- fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {- let mut cfb = CompoundFile::open(Cursor::new(data))?;-- let summary = read_stream(&mut cfb, "\x05SummaryInformation")- .ok()- .and_then(|bytes| parse_summary_information(&bytes));-- let word_doc = read_stream(&mut cfb, "WordDocument")?;- let fib = Fib::parse(&word_doc)?;-- let (primary, secondary) = if fib.table_stream_1 {- ("1Table", "0Table")- } else {- ("0Table", "1Table")- };- let table = read_stream(&mut cfb, primary)- .or_else(|_| read_stream(&mut cfb, secondary))- .ok();-- let ansi = summary- .as_ref()- .and_then(|s| s.codepage)- .and_then(encoding_for_codepage)- .unwrap_or_else(|| encoding_for_lid(fib.lid));-- let segments = extract_segments(&word_doc, table.as_deref(), &fib, ansi);- let formatting = if fib.legacy {- Formatting::default()- } else {- table- .as_deref()- .map(|table| Formatting::parse(&word_doc, table, &fib))- .unwrap_or_default()- };-- let mut metadata = DocumentMetadata::default();- if let Some(summary) = summary {- metadata.title = summary.title;- metadata.author = summary.author;- metadata.created = summary.created;- }-- Ok(Document {- blocks: build_blocks(&segments, &formatting),- metadata,- notes: Vec::new(),- comments: Vec::new(),- })- }-- fn name(&self) -> &'static str {- "doc"- }-}--fn read_stream<R: Read + Seek>(- cfb: &mut CompoundFile<R>,- name: &str,-) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {- let mut stream = cfb.open_stream(name)?;- let mut buf = Vec::new();- stream.read_to_end(&mut buf)?;- Ok(buf)-}--struct Fib {- legacy: bool,- lid: u16,- table_stream_1: bool,- ext_char: bool,- fc_min: usize,- fc_mac: usize,- ccp_text: usize,- fc_clx: usize,- lcb_clx: usize,- fc_plcf_bte_chpx: usize,- lcb_plcf_bte_chpx: usize,- fc_plcf_bte_papx: usize,- lcb_plcf_bte_papx: usize,-}--impl Fib {- fn parse(word_doc: &[u8]) -> Result<Self, Box<dyn Error + Send + Sync>> {- if word_doc.len() < 0x20 {- return Err("WordDocument stream too short".into());- }- let magic = read_u16(word_doc, 0x00);- let legacy = match magic {- MAGIC_WORD_97 => false,- MAGIC_WORD_6_95 => true,- _ => return Err(format!("not a Word document (magic {magic:#06X})").into()),- };-- let flags = read_u16(word_doc, 0x0A);- if flags & 0x0100 != 0 {- return Err("encrypted .doc files are not supported".into());- }-- let u32_at = |offset: usize| {- if word_doc.len() >= offset + 4 {- read_u32(word_doc, offset) as usize- } else {- 0- }- };-- Ok(Fib {- legacy,- lid: read_u16(word_doc, 0x06),- table_stream_1: flags & 0x0200 != 0,- ext_char: flags & 0x1000 != 0,- fc_min: read_u32(word_doc, 0x18) as usize,- fc_mac: read_u32(word_doc, 0x1C) as usize,- ccp_text: u32_at(0x4C),- fc_clx: u32_at(0x1A2),- lcb_clx: u32_at(0x1A6),- fc_plcf_bte_chpx: u32_at(0xFA),- lcb_plcf_bte_chpx: u32_at(0xFE),- fc_plcf_bte_papx: u32_at(0x102),- lcb_plcf_bte_papx: u32_at(0x106),- })- }-}--/// Decoded text run and the stream offset its bytes came from.-struct TextSegment {- text: String,- fc_start: u32,- bytes_per_char: u8,-}--fn extract_segments(- word_doc: &[u8],- table: Option<&[u8]>,- fib: &Fib,- ansi: &'static Encoding,-) -> Vec<TextSegment> {- if !fib.legacy {- if let Some(table) = table {- if let Some(pieces) = parse_piece_table(table, fib) {- return decode_pieces(word_doc, &pieces, fib.ccp_text, ansi);- }- }- }-- // Word 6/95 files and corrupt piece tables: text is at fcMin..fcMac.- let start = fib.fc_min.min(word_doc.len());- let end = fib.fc_mac.clamp(start, word_doc.len());- let bytes = &word_doc[start..end];- let (text, bytes_per_char) = if fib.ext_char {- (decode_utf16le(bytes), 2)- } else {- (ansi.decode_without_bom_handling(bytes).0.into_owned(), 1)- };- vec![TextSegment {- text,- fc_start: start as u32,- bytes_per_char,- }]-}--struct Piece {- start_cp: u32,- end_cp: u32,- offset: usize,- compressed: bool,-}--fn parse_piece_table(table: &[u8], fib: &Fib) -> Option<Vec<Piece>> {- if fib.lcb_clx == 0 {- return None;- }- let clx = table.get(fib.fc_clx..fib.fc_clx.checked_add(fib.lcb_clx)?)?;-- // Clx: zero or more Prc (0x01) blocks, then the Pcdt (0x02) piece table.- let mut i = 0;- while i < clx.len() {- match clx[i] {- 0x01 => {- let cb = read_u16(clx.get(i + 1..i + 3)?, 0) as usize;- i += 3 + cb;- }- 0x02 => {- let lcb = read_u32(clx.get(i + 1..i + 5)?, 0) as usize;- let plc = clx.get(i + 5..(i + 5).checked_add(lcb)?)?;- return parse_plc_pcd(plc);- }- _ => return None,- }- }- None-}--fn parse_plc_pcd(plc: &[u8]) -> Option<Vec<Piece>> {- // n+1 CPs (4 bytes each) followed by n PCDs (8 bytes each)- if plc.len() < 16 || (plc.len() - 4) % 12 != 0 {- return None;- }- let n = (plc.len() - 4) / 12;-- let mut pieces = Vec::with_capacity(n);- for k in 0..n {- let start_cp = read_u32(plc, k * 4);- let end_cp = read_u32(plc, (k + 1) * 4);- if end_cp < start_cp {- return None;- }- let fc = read_u32(plc, 4 * (n + 1) + 8 * k + 2);- let compressed = fc & 0x4000_0000 != 0;- let offset = if compressed {- ((fc & 0x3FFF_FFFF) / 2) as usize- } else {- (fc & 0x3FFF_FFFF) as usize- };- pieces.push(Piece {- start_cp,- end_cp,- offset,- compressed,- });- }- Some(pieces)-}--fn decode_pieces(- word_doc: &[u8],- pieces: &[Piece],- ccp_text: usize,- ansi: &'static Encoding,-) -> Vec<TextSegment> {- // piece cp ranges are contiguous, so this clamp bounds total output- let ccp_text = ccp_text.min(MAX_TEXT_CHARS);- let mut segments = Vec::new();- for piece in pieces {- // body text only; cps past ccpText are headers, footnotes etc.- if piece.start_cp as usize >= ccp_text {- continue;- }- let count = (piece.end_cp as usize).min(ccp_text) - piece.start_cp as usize;- let available = word_doc.len().saturating_sub(piece.offset);- let start = piece.offset.min(word_doc.len());- let (text, bytes_per_char) = if piece.compressed {- let bytes = &word_doc[start..][..count.min(available)];- (ansi.decode_without_bom_handling(bytes).0.into_owned(), 1)- } else {- let bytes = &word_doc[start..][..(count * 2).min(available)];- (decode_utf16le(bytes), 2)- };- segments.push(TextSegment {- text,- fc_start: start as u32,- bytes_per_char,- });- }- segments-}--fn decode_utf16le(bytes: &[u8]) -> String {- let units: Vec<u16> = bytes- .chunks_exact(2)- .map(|c| u16::from_le_bytes([c[0], c[1]]))- .collect();- String::from_utf16_lossy(&units)-}--#[derive(Clone, Copy, Default, PartialEq, Eq)]-struct CharProps {- bold: bool,- italic: bool,- strike: bool,-}--#[derive(Clone, Copy, Default)]-struct ParaProps {- in_table: bool,- row_end: bool,- heading: Option<u8>,-}--struct FcRun<T> {- fc_start: u32,- fc_end: u32,- props: T,-}--#[derive(Default)]-struct Formatting {- char_runs: Vec<FcRun<CharProps>>,- para_runs: Vec<FcRun<ParaProps>>,-}--impl Formatting {- fn parse(word_doc: &[u8], table: &[u8], fib: &Fib) -> Self {- Self {- char_runs: parse_bin_table(- word_doc,- table,- fib.fc_plcf_bte_chpx,- fib.lcb_plcf_bte_chpx,- parse_chpx_fkp,- ),- para_runs: parse_bin_table(- word_doc,- table,- fib.fc_plcf_bte_papx,- fib.lcb_plcf_bte_papx,- parse_papx_fkp,- ),- }- }-}--/// Walks a bin table: a PLC of page numbers of FKPs in the WordDocument-/// stream.-fn parse_bin_table<T>(- word_doc: &[u8],- table: &[u8],- fc: usize,- lcb: usize,- parse_fkp: impl Fn(&[u8], &mut Vec<FcRun<T>>),-) -> Vec<FcRun<T>> {- let mut runs = Vec::new();- let Some(end) = fc.checked_add(lcb) else {- return runs;- };- let Some(plc) = table.get(fc..end) else {- return runs;- };- if lcb < 12 || (lcb - 4) % 8 != 0 {- return runs;- }- let n = (lcb - 4) / 8;- for k in 0..n {- let pn = read_u32(plc, 4 * (n + 1) + 4 * k) as usize;- let Some(page_start) = pn.checked_mul(FKP_PAGE_SIZE) else {- continue;- };- if let Some(page) = word_doc.get(page_start..page_start + FKP_PAGE_SIZE) {- parse_fkp(page, &mut runs);- }- }- runs.sort_by_key(|run| run.fc_start);- runs-}--fn parse_chpx_fkp(page: &[u8], runs: &mut Vec<FcRun<CharProps>>) {- let crun = page[FKP_PAGE_SIZE - 1] as usize;- if crun == 0 || 4 * (crun + 1) + crun >= FKP_PAGE_SIZE {- return;- }- for i in 0..crun {- let fc_start = read_u32(page, 4 * i);- let fc_end = read_u32(page, 4 * (i + 1));- if fc_end <= fc_start {- continue;- }- // rgb entry: word offset of the Chpx within the page; 0 means default- let rgb = page[4 * (crun + 1) + i] as usize;- let mut props = CharProps::default();- if rgb != 0 {- let offset = rgb * 2;- if let Some(&cb) = page.get(offset) {- if let Some(grpprl) = page.get(offset + 1..offset + 1 + cb as usize) {- props = char_props_from_grpprl(grpprl);- }- }- }- runs.push(FcRun {- fc_start,- fc_end,- props,- });- }-}--fn parse_papx_fkp(page: &[u8], runs: &mut Vec<FcRun<ParaProps>>) {- let cpara = page[FKP_PAGE_SIZE - 1] as usize;- if cpara == 0 || 4 * (cpara + 1) + 13 * cpara >= FKP_PAGE_SIZE {- return;- }- for i in 0..cpara {- let fc_start = read_u32(page, 4 * i);- let fc_end = read_u32(page, 4 * (i + 1));- if fc_end <= fc_start {- continue;- }- // BxPap: first byte is the word offset of the PapxInFkp; 0 means default- let bx = page[4 * (cpara + 1) + 13 * i] as usize;- let mut props = ParaProps::default();- if bx != 0 {- let offset = bx * 2;- // PapxInFkp: grpprl length is cb*2 - 1, or cb2*2 when cb is 0- if let Some(&cb) = page.get(offset) {- let (start, len) = if cb == 0 {- match page.get(offset + 1) {- Some(&cb2) => (offset + 2, cb2 as usize * 2),- None => (offset, 0),- }- } else {- (offset + 1, (cb as usize) * 2 - 1)- };- if let Some(grpprl) = page.get(start..start + len) {- props = para_props_from_grpprl(grpprl);- }- }- }- runs.push(FcRun {- fc_start,- fc_end,- props,- });- }-}--fn char_props_from_grpprl(grpprl: &[u8]) -> CharProps {- let mut props = CharProps::default();- walk_sprms(grpprl, |sprm, operand| {- // toggle operand: 1 = on, 0x81 = invert style (assume on)- let on = matches!(operand.first(), Some(1) | Some(0x81));- match sprm {- 0x0835 => props.bold = on,- 0x0836 => props.italic = on,- 0x0837 => props.strike = on,- _ => {}- }- });- props-}--fn para_props_from_grpprl(grpprl: &[u8]) -> ParaProps {- let mut props = ParaProps::default();- if grpprl.len() < 2 {- return props;- }- props.heading = heading_from_istd(read_u16(grpprl, 0));- walk_sprms(&grpprl[2..], |sprm, operand| match sprm {- // sprmPFInTable, sprmPFTtp, sprmPOutLvl, sprmPIstd- 0x2416 => props.in_table = operand.first().is_some_and(|&v| v != 0),- 0x2417 => props.row_end = operand.first().is_some_and(|&v| v != 0),- 0x2640 => {- if let Some(&level) = operand.first() {- if level < 9 {- props.heading = Some((level + 1).min(6));- }- }- }- 0x4600 => {- if operand.len() >= 2 {- if let Some(level) = heading_from_istd(read_u16(operand, 0)) {- props.heading = Some(level);- }- }- }- _ => {}- });- props-}--/// Built-in style indexes 1..=9 are "heading 1".."heading 9".-fn heading_from_istd(istd: u16) -> Option<u8> {- if (1..=9).contains(&istd) {- Some((istd as u8).min(6))- } else {- None- }-}--/// Visits each sprm in a grpprl; operand size comes from the top 3 bits.-fn walk_sprms(grpprl: &[u8], mut visit: impl FnMut(u16, &[u8])) {- let mut i = 0;- while i + 2 <= grpprl.len() {- let sprm = read_u16(grpprl, i);- i += 2;- let size = match sprm >> 13 {- 0 | 1 => 1,- 2 | 4 | 5 => 2,- 7 => 3,- 3 => 4,- _ => {- // variable size: length byte first (u16 + 1 for sprmTDefTable)- if sprm == 0xD608 {- if i + 2 > grpprl.len() {- return;- }- read_u16(grpprl, i) as usize + 1- } else {- match grpprl.get(i) {- Some(&cb) => cb as usize + 1,- None => return,- }- }- }- };- let Some(operand) = grpprl.get(i..i + size) else {- return;- };- visit(sprm, operand);- i += size;- }-}--/// Props for the run containing `fc`, with a cursor for sequential lookups.-fn run_lookup<T: Copy + Default>(runs: &[FcRun<T>], fc: u32, cursor: &mut usize) -> T {- if runs.is_empty() {- return T::default();- }- if *cursor >= runs.len() || fc < runs[*cursor].fc_start {- *cursor = runs.partition_point(|run| run.fc_end <= fc);- } else {- while *cursor < runs.len() && runs[*cursor].fc_end <= fc {- *cursor += 1;- }- }- match runs.get(*cursor) {- Some(run) if fc >= run.fc_start && fc < run.fc_end => run.props,- _ => T::default(),- }-}--fn build_blocks(segments: &[TextSegment], formatting: &Formatting) -> Vec<Block> {- let mut builder = BlockBuilder {- formatting,- blocks: Vec::new(),- inlines: Vec::new(),- run_text: String::new(),- run_props: CharProps::default(),- char_cursor: 0,- para_cursor: 0,- fields: Vec::new(),- table_rows: Vec::new(),- row_cells: Vec::new(),- cell_blocks: Vec::new(),- };-- for segment in segments {- let mut fc = segment.fc_start;- for ch in segment.text.chars() {- builder.handle_char(ch, fc);- fc = fc.saturating_add(match segment.bytes_per_char {- 1 => 1,- _ => 2 * ch.len_utf16() as u32,- });- }- }- builder.finish()-}--struct FieldFrame {- in_instruction: bool,- instruction: String,- result_inline_start: usize,-}--struct BlockBuilder<'a> {- formatting: &'a Formatting,- blocks: Vec<Block>,- inlines: Vec<Inline>,- run_text: String,- run_props: CharProps,- char_cursor: usize,- para_cursor: usize,- fields: Vec<FieldFrame>,- table_rows: Vec<TableRow>,- row_cells: Vec<TableCell>,- cell_blocks: Vec<Block>,-}--impl BlockBuilder<'_> {- fn handle_char(&mut self, ch: char, fc: u32) {- match ch {- '\u{13}' => {- self.fields.push(FieldFrame {- in_instruction: true,- instruction: String::new(),- result_inline_start: usize::MAX,- });- return;- }- '\u{14}' => {- self.flush_run();- let start = self.inlines.len();- if let Some(frame) = self.fields.last_mut() {- frame.in_instruction = false;- frame.result_inline_start = start;- }- return;- }- '\u{15}' => {- self.flush_run();- if let Some(frame) = self.fields.pop() {- self.close_field(frame);- }- return;- }- _ => {}- }-- if let Some(frame) = self.fields.iter_mut().rev().find(|f| f.in_instruction) {- if frame.instruction.len() < MAX_FIELD_INSTRUCTION_CHARS {- frame.instruction.push(ch);- }- return;- }-- match ch {- '\r' | '\n' | '\u{c}' | '\u{e}' => {- let props = self.para_props(fc);- self.end_paragraph(props);- }- '\u{7}' => {- let props = self.para_props(fc);- if props.row_end {- self.end_row();- } else if props.in_table {- self.end_cell(props);- } else {- // no table context: behave like a paragraph mark- self.end_paragraph(props);- }- }- '\u{b}' => {- self.flush_run();- if !self.inlines.is_empty() {- self.inlines.push(Inline::LineBreak);- }- }- '\t' => self.push_char('\t', fc),- '\u{1e}' => self.push_char('-', fc),- c if c.is_control() || c == '\u{1f}' => {}- c => self.push_char(c, fc),- }- }-- fn push_char(&mut self, ch: char, fc: u32) {- let props = run_lookup(&self.formatting.char_runs, fc, &mut self.char_cursor);- if props != self.run_props {- self.flush_run();- self.run_props = props;- }- self.run_text.push(ch);- }-- fn para_props(&mut self, fc: u32) -> ParaProps {- run_lookup(&self.formatting.para_runs, fc, &mut self.para_cursor)- }-- fn flush_run(&mut self) {- if self.run_text.is_empty() {- return;- }- let mut node = Inline::Text(std::mem::take(&mut self.run_text));- if self.run_props.strike {- node = Inline::Del(vec![node]);- }- if self.run_props.italic {- node = Inline::Em(vec![node]);- }- if self.run_props.bold {- node = Inline::Strong(vec![node]);- }- self.inlines.push(node);- }-- fn close_field(&mut self, frame: FieldFrame) {- let Some(href) = hyperlink_target(&frame.instruction) else {- return;- };- let start = frame.result_inline_start.min(self.inlines.len());- let children = self.inlines.split_off(start);- if has_visible_content(&children) {- self.inlines.push(Inline::Link { href, children });- } else {- self.inlines.extend(children);- }- }-- /// Takes the pending inlines as a paragraph, or `None` when nothing- /// visible accumulated.- fn take_paragraph(&mut self, props: ParaProps) -> Option<Block> {- self.flush_run();- // links cannot span paragraphs- for frame in &mut self.fields {- frame.result_inline_start = 0;- }- while matches!(self.inlines.last(), Some(Inline::LineBreak)) {- self.inlines.pop();- }- if !has_visible_content(&self.inlines) {- self.inlines.clear();- return None;- }- if let Some(Inline::Text(first)) = self.inlines.first_mut() {- *first = first.trim_start().to_string();- }- if let Some(Inline::Text(last)) = self.inlines.last_mut() {- *last = last.trim_end().to_string();- }- let kind = match props.heading {- Some(level) => ParagraphKind::Heading(level),- None => ParagraphKind::Normal,- };- Some(Block::Paragraph(Paragraph {- kind,- inlines: std::mem::take(&mut self.inlines),- }))- }-- fn end_paragraph(&mut self, props: ParaProps) {- let paragraph = self.take_paragraph(props);- if props.in_table {- if let Some(paragraph) = paragraph {- self.cell_blocks.push(paragraph);- }- } else {- self.flush_table();- if let Some(paragraph) = paragraph {- self.blocks.push(paragraph);- }- }- }-- fn end_cell(&mut self, props: ParaProps) {- if let Some(paragraph) = self.take_paragraph(props) {- self.cell_blocks.push(paragraph);- }- self.row_cells.push(TableCell {- blocks: std::mem::take(&mut self.cell_blocks),- colspan: NonZeroU32::MIN,- rowspan: NonZeroU32::MIN,- });- }-- /// Row terminator: the TTP mark's own paragraph carries no cell content.- fn end_row(&mut self) {- self.flush_run();- self.inlines.clear();- if !self.cell_blocks.is_empty() {- self.row_cells.push(TableCell {- blocks: std::mem::take(&mut self.cell_blocks),- colspan: NonZeroU32::MIN,- rowspan: NonZeroU32::MIN,- });- }- if !self.row_cells.is_empty() {- self.table_rows.push(TableRow {- cells: std::mem::take(&mut self.row_cells),- kind: TableRowKind::Body,- });- }- }-- fn flush_table(&mut self) {- if !self.cell_blocks.is_empty() || !self.row_cells.is_empty() {- self.end_row();- }- if !self.table_rows.is_empty() {- self.blocks.push(Block::Table(Table {- rows: std::mem::take(&mut self.table_rows),- }));- }- }-- fn finish(mut self) -> Vec<Block> {- self.end_paragraph(ParaProps::default());- self.flush_table();- self.blocks- }-}--/// Target of a HYPERLINK field instruction.-fn hyperlink_target(instruction: &str) -> Option<String> {- let rest = instruction.trim_start();- if !rest- .get(..9)- .is_some_and(|p| p.eq_ignore_ascii_case("HYPERLINK"))- {- return None;- }- let mut tokens = field_tokens(&rest[9..]);- let mut anchor = false;- while let Some(token) = tokens.next() {- match token.as_str() {- "\\l" => anchor = true,- // switches whose argument is not the target- "\\o" | "\\t" => {- tokens.next();- }- t if t.starts_with('\\') => {}- t => {- return Some(if anchor {- format!("#{t}")- } else {- t.to_string()- });- }- }- }- None-}--/// Whitespace-separated tokens; quoted strings stay together, unquoted.-fn field_tokens(input: &str) -> impl Iterator<Item = String> + '_ {- let mut chars = input.chars().peekable();- std::iter::from_fn(move || {- while chars.peek().is_some_and(|c| c.is_whitespace()) {- chars.next();- }- match chars.peek() {- None => None,- Some('"') => {- chars.next();- let mut token = String::new();- for c in chars.by_ref() {- if c == '"' {- break;- }- token.push(c);- }- Some(token)- }- Some(_) => {- let mut token = String::new();- while let Some(&c) = chars.peek() {- if c.is_whitespace() {- break;- }- token.push(c);- chars.next();- }- Some(token)- }- }- })-}--fn read_u16(data: &[u8], offset: usize) -> u16 {- u16::from_le_bytes([data[offset], data[offset + 1]])-}--fn read_u32(data: &[u8], offset: usize) -> u32 {- u32::from_le_bytes([- data[offset],- data[offset + 1],- data[offset + 2],- data[offset + 3],- ])-}--#[cfg(test)]-mod tests {- use super::*;- use std::io::Write;-- enum TestPiece {- Compressed(Vec<u8>),- Unicode(&'static str),- }-- /// Contiguous formatting runs: (fc_start, fc_end, grpprl). An empty grpprl- /// becomes a default (offset 0) FKP entry.- #[derive(Default)]- struct TestFmt {- char_runs: Vec<(u32, u32, Vec<u8>)>,- para_runs: Vec<(u32, u32, Vec<u8>)>,- }-- fn utf16_bytes(text: &str) -> Vec<u8> {- text.encode_utf16().flat_map(u16::to_le_bytes).collect()- }-- const TEXT_START: usize = 0x200;-- /// FC of the i-th UTF-16 unit of a single Unicode test piece.- fn ufc(i: usize) -> u32 {- (TEXT_START + 2 * i) as u32- }-- fn sprm1(code: u16, value: u8) -> Vec<u8> {- let mut out = code.to_le_bytes().to_vec();- out.push(value);- out- }-- fn papx(istd: u16, sprms: &[Vec<u8>]) -> Vec<u8> {- let mut out = istd.to_le_bytes().to_vec();- for sprm in sprms {- out.extend_from_slice(sprm);- }- out- }-- fn build_chpx_fkp(runs: &[(u32, u32, Vec<u8>)]) -> Vec<u8> {- let mut page = vec![0u8; FKP_PAGE_SIZE];- let crun = runs.len();- for (i, (fc_start, fc_end, _)) in runs.iter().enumerate() {- page[4 * i..4 * i + 4].copy_from_slice(&fc_start.to_le_bytes());- page[4 * (i + 1)..4 * (i + 1) + 4].copy_from_slice(&fc_end.to_le_bytes());- }- let mut blob_end = FKP_PAGE_SIZE - 1;- for (i, (_, _, grpprl)) in runs.iter().enumerate() {- if grpprl.is_empty() {- continue;- }- let blob_len = 1 + grpprl.len();- let mut pos = blob_end - blob_len;- pos &= !1;- page[pos] = grpprl.len() as u8;- page[pos + 1..pos + 1 + grpprl.len()].copy_from_slice(grpprl);- page[4 * (crun + 1) + i] = (pos / 2) as u8;- blob_end = pos;- }- page[FKP_PAGE_SIZE - 1] = crun as u8;- page- }-- fn build_papx_fkp(runs: &[(u32, u32, Vec<u8>)]) -> Vec<u8> {- let mut page = vec![0u8; FKP_PAGE_SIZE];- let cpara = runs.len();- for (i, (fc_start, fc_end, _)) in runs.iter().enumerate() {- page[4 * i..4 * i + 4].copy_from_slice(&fc_start.to_le_bytes());- page[4 * (i + 1)..4 * (i + 1) + 4].copy_from_slice(&fc_end.to_le_bytes());- }- let mut blob_end = FKP_PAGE_SIZE - 1;- for (i, (_, _, grpprl)) in runs.iter().enumerate() {- if grpprl.is_empty() {- continue;- }- let blob = if grpprl.len() % 2 == 1 {- let mut b = vec![((grpprl.len() + 1) / 2) as u8];- b.extend_from_slice(grpprl);- b- } else {- let mut b = vec![0, (grpprl.len() / 2) as u8];- b.extend_from_slice(grpprl);- b- };- let mut pos = blob_end - blob.len();- pos &= !1;- page[pos..pos + blob.len()].copy_from_slice(&blob);- page[4 * (cpara + 1) + 13 * i] = (pos / 2) as u8;- blob_end = pos;- }- page[FKP_PAGE_SIZE - 1] = cpara as u8;- page- }-- fn build_word_doc_streams(pieces: &[TestPiece], lid: u16) -> (Vec<u8>, Vec<u8>) {- build_word_doc_streams_fmt(pieces, lid, &TestFmt::default())- }-- fn build_word_doc_streams_fmt(- pieces: &[TestPiece],- lid: u16,- fmt: &TestFmt,- ) -> (Vec<u8>, Vec<u8>) {- let mut word_doc = vec![0u8; TEXT_START];- let mut cps = vec![0u32];- let mut pcds = Vec::new();-- for piece in pieces {- let offset = word_doc.len();- let (count, fc) = match piece {- TestPiece::Compressed(bytes) => {- word_doc.extend_from_slice(bytes);- (bytes.len() as u32, (offset as u32 * 2) | 0x4000_0000)- }- TestPiece::Unicode(text) => {- let bytes = utf16_bytes(text);- word_doc.extend_from_slice(&bytes);- (text.encode_utf16().count() as u32, offset as u32)- }- };- cps.push(cps.last().unwrap() + count);- pcds.push(fc);- }-- let ccp_text = *cps.last().unwrap();- let mut plc = Vec::new();- for cp in &cps {- plc.extend_from_slice(&cp.to_le_bytes());- }- for fc in &pcds {- plc.extend_from_slice(&0u16.to_le_bytes());- plc.extend_from_slice(&fc.to_le_bytes());- plc.extend_from_slice(&0u16.to_le_bytes());- }- let mut table = vec![0x02];- table.extend_from_slice(&(plc.len() as u32).to_le_bytes());- table.extend_from_slice(&plc);- let lcb_clx = table.len() as u32;-- let append_fkp =- |word_doc: &mut Vec<u8>, table: &mut Vec<u8>, page: Vec<u8>, runs: &[(u32, u32, Vec<u8>)]| {- while word_doc.len() % FKP_PAGE_SIZE != 0 {- word_doc.push(0);- }- let pn = (word_doc.len() / FKP_PAGE_SIZE) as u32;- word_doc.extend_from_slice(&page);- let bin_table_offset = table.len() as u32;- table.extend_from_slice(&runs[0].0.to_le_bytes());- table.extend_from_slice(&runs.last().unwrap().1.to_le_bytes());- table.extend_from_slice(&pn.to_le_bytes());- (bin_table_offset, 12u32)- };-- let chpx_bt = (!fmt.char_runs.is_empty()).then(|| {- let page = build_chpx_fkp(&fmt.char_runs);- append_fkp(&mut word_doc, &mut table, page, &fmt.char_runs)- });- let papx_bt = (!fmt.para_runs.is_empty()).then(|| {- let page = build_papx_fkp(&fmt.para_runs);- append_fkp(&mut word_doc, &mut table, page, &fmt.para_runs)- });-- word_doc[0x00..0x02].copy_from_slice(&MAGIC_WORD_97.to_le_bytes());- word_doc[0x02..0x04].copy_from_slice(&0x00C1u16.to_le_bytes());- word_doc[0x06..0x08].copy_from_slice(&lid.to_le_bytes());- word_doc[0x0A..0x0C].copy_from_slice(&0x0200u16.to_le_bytes());- word_doc[0x4C..0x50].copy_from_slice(&ccp_text.to_le_bytes());- word_doc[0x1A2..0x1A6].copy_from_slice(&0u32.to_le_bytes());- word_doc[0x1A6..0x1AA].copy_from_slice(&lcb_clx.to_le_bytes());- if let Some((fc, lcb)) = chpx_bt {- word_doc[0xFA..0xFE].copy_from_slice(&fc.to_le_bytes());- word_doc[0xFE..0x102].copy_from_slice(&lcb.to_le_bytes());- }- if let Some((fc, lcb)) = papx_bt {- word_doc[0x102..0x106].copy_from_slice(&fc.to_le_bytes());- word_doc[0x106..0x10A].copy_from_slice(&lcb.to_le_bytes());- }-- (word_doc, table)- }-- fn build_cfb(streams: &[(&str, &[u8])]) -> Vec<u8> {- let mut comp = CompoundFile::create(Cursor::new(Vec::new())).unwrap();- for (name, data) in streams {- let mut stream = comp.create_stream(name).unwrap();- stream.write_all(data).unwrap();- }- comp.into_inner().into_inner()- }-- fn build_doc(pieces: &[TestPiece], lid: u16, summary: Option<&[u8]>) -> Vec<u8> {- build_doc_fmt(pieces, lid, summary, &TestFmt::default())- }-- fn build_doc_fmt(- pieces: &[TestPiece],- lid: u16,- summary: Option<&[u8]>,- fmt: &TestFmt,- ) -> Vec<u8> {- let (word_doc, table) = build_word_doc_streams_fmt(pieces, lid, fmt);- let mut streams: Vec<(&str, &[u8])> = vec![("WordDocument", &word_doc), ("1Table", &table)];- if let Some(summary) = summary {- streams.push(("\x05SummaryInformation", summary));- }- build_cfb(&streams)- }-- fn build_summary_information(codepage: u16, title: &[u8], author: &[u8]) -> Vec<u8> {- let mut props: Vec<(u32, Vec<u8>)> = Vec::new();- let mut codepage_value = 0x0002u32.to_le_bytes().to_vec();- codepage_value.extend_from_slice(&(codepage as u32).to_le_bytes());- props.push((0x0001, codepage_value));- for (pid, text) in [(0x0002u32, title), (0x0004u32, author)] {- let mut value = 0x001Eu32.to_le_bytes().to_vec();- value.extend_from_slice(&((text.len() + 1) as u32).to_le_bytes());- value.extend_from_slice(text);- value.push(0);- props.push((pid, value));- }-- let section_start = 0x30usize;- let header_size = 8 + props.len() * 8;- let mut entries = Vec::new();- let mut values = Vec::new();- for (pid, value) in &props {- entries.extend_from_slice(&pid.to_le_bytes());- entries.extend_from_slice(&((header_size + values.len()) as u32).to_le_bytes());- values.extend_from_slice(value);- while values.len() % 4 != 0 {- values.push(0);- }- }-- let mut data = vec![0u8; section_start];- data[0x00..0x02].copy_from_slice(&0xFFFEu16.to_le_bytes());- data[0x18..0x1C].copy_from_slice(&1u32.to_le_bytes());- data[0x2C..0x30].copy_from_slice(&(section_start as u32).to_le_bytes());- data.extend_from_slice(&((header_size + values.len()) as u32).to_le_bytes());- data.extend_from_slice(&(props.len() as u32).to_le_bytes());- data.extend_from_slice(&entries);- data.extend_from_slice(&values);- data- }-- /// Readable markup for inline assertions, e.g. `a <b>x</b> <a href=u>y</a>`.- fn inline_markup(inlines: &[Inline]) -> String {- inlines- .iter()- .map(|inline| match inline {- Inline::Text(t) => t.clone(),- Inline::LineBreak => "\n".to_string(),- Inline::Strong(c) => format!("<b>{}</b>", inline_markup(c)),- Inline::Em(c) => format!("<i>{}</i>", inline_markup(c)),- Inline::Del(c) => format!("<del>{}</del>", inline_markup(c)),- Inline::Link { href, children } => {- format!("<a href={}>{}</a>", href, inline_markup(children))- }- _ => String::new(),- })- .collect()- }-- fn inline_text(inlines: &[Inline]) -> String {- inlines- .iter()- .map(|inline| match inline {- Inline::Text(t) => t.clone(),- Inline::LineBreak => "\n".to_string(),- Inline::Link { children, .. } => inline_text(children),- Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {- inline_text(c)- }- _ => String::new(),- })- .collect()- }-- fn paragraph_texts(document: &Document) -> Vec<String> {- document- .blocks- .iter()- .map(|block| match block {- Block::Paragraph(p) => inline_text(&p.inlines),- _ => String::new(),- })- .collect()- }-- fn parse(data: &[u8]) -> Document {- DocProvider::new().parse_buffer(data).unwrap()- }-- fn cp1251(text: &str) -> Vec<u8> {- encoding_rs::WINDOWS_1251.encode(text).0.into_owned()- }-- #[test]- fn unicode_pieces() {- let data = build_doc(- &[TestPiece::Unicode("Указ Президента України\rДодаток\r")],- 0x0409,- None,- );- assert_eq!(- paragraph_texts(&parse(&data)),- vec!["Указ Президента України", "Додаток"]- );- }-- #[test]- fn compressed_cp1251_via_summary_codepage() {- let summary = build_summary_information(1251, &cp1251("Указ"), &cp1251("Автор"));- let data = build_doc(- &[TestPiece::Compressed(cp1251("ПРЕЗИДЕНТ УКРАЇНИ\r"))],- 0x0409,- Some(&summary),- );- let document = parse(&data);- assert_eq!(paragraph_texts(&document), vec!["ПРЕЗИДЕНТ УКРАЇНИ"]);- assert_eq!(document.metadata.title.as_deref(), Some("Указ"));- assert_eq!(document.metadata.author.as_deref(), Some("Автор"));- }-- #[test]- fn compressed_cp1251_via_lid() {- let data = build_doc(- &[TestPiece::Compressed(cp1251("Слава Україні\r"))],- 0x0422,- None,- );- assert_eq!(paragraph_texts(&parse(&data)), vec!["Слава Україні"]);- }-- #[test]- fn compressed_defaults_to_cp1252() {- let data = build_doc(- &[TestPiece::Compressed(b"Caf\xE9 \x93quoted\x94\r".to_vec())],- 0x0409,- None,- );- assert_eq!(- paragraph_texts(&parse(&data)),- vec!["Café \u{201C}quoted\u{201D}"]- );- }-- #[test]- fn mixed_pieces_concatenate_in_cp_order() {- let data = build_doc(- &[- TestPiece::Unicode("Перша частина "),- TestPiece::Compressed(b"and ASCII tail\r".to_vec()),- ],- 0x0419,- None,- );- assert_eq!(- paragraph_texts(&parse(&data)),- vec!["Перша частина and ASCII tail"]- );- }-- #[test]- fn field_instructions_are_stripped() {- let data = build_doc(- &[TestPiece::Unicode(- "See \u{13} HYPERLINK \"https://example.com\" \u{14}the site\u{15} now\r",- )],- 0x0409,- None,- );- assert_eq!(paragraph_texts(&parse(&data)), vec!["See the site now"]);- }-- #[test]- fn hyperlink_fields_become_links() {- let data = build_doc(- &[TestPiece::Unicode(- "See \u{13} HYPERLINK \"https://example.com\" \u{14}the site\u{15} now\r",- )],- 0x0409,- None,- );- let document = parse(&data);- let Block::Paragraph(p) = &document.blocks[0] else {- panic!("expected paragraph");- };- assert_eq!(- inline_markup(&p.inlines),- "See <a href=https://example.com>the site</a> now"- );- }-- #[test]- fn anchor_hyperlink_targets_bookmark() {- let data = build_doc(- &[TestPiece::Unicode(- "Go \u{13} HYPERLINK \\l \"section1\" \u{14}there\u{15}\r",- )],- 0x0409,- None,- );- let document = parse(&data);- let Block::Paragraph(p) = &document.blocks[0] else {- panic!("expected paragraph");- };- assert_eq!(inline_markup(&p.inlines), "Go <a href=#section1>there</a>");- }-- #[test]- fn non_hyperlink_fields_keep_result_text_only() {- let data = build_doc(- &[TestPiece::Unicode("Page \u{13} PAGE \u{14}7\u{15} of 9\r")],- 0x0409,- None,- );- let document = parse(&data);- let Block::Paragraph(p) = &document.blocks[0] else {- panic!("expected paragraph");- };- assert_eq!(inline_markup(&p.inlines), "Page 7 of 9");- }-- #[test]- fn cell_marks_and_line_breaks() {- let data = build_doc(- &[TestPiece::Unicode(- "cell one\u{7}cell two\u{7}line\u{b}break\r",- )],- 0x0409,- None,- );- assert_eq!(- paragraph_texts(&parse(&data)),- vec!["cell one", "cell two", "line\nbreak"]- );- }-- #[test]- fn bold_italic_strike_runs() {- let text = "plain bold and style\r";- let fmt = TestFmt {- char_runs: vec![- (ufc(0), ufc(6), vec![]),- (ufc(6), ufc(10), sprm1(0x0835, 1)),- (ufc(10), ufc(15), vec![]),- (- ufc(15),- ufc(20),- [sprm1(0x0836, 1), sprm1(0x0837, 1)].concat(),- ),- (ufc(20), ufc(21), vec![]),- ],- ..TestFmt::default()- };- let data = build_doc_fmt(&[TestPiece::Unicode(text)], 0x0409, None, &fmt);- let document = parse(&data);- let Block::Paragraph(p) = &document.blocks[0] else {- panic!("expected paragraph");- };- assert_eq!(- inline_markup(&p.inlines),- "plain <b>bold</b> and <i><del>style</del></i>"- );- }-- #[test]- fn headings_from_istd_and_outline_level() {- let text = "Title\rSection\rBody\r";- let fmt = TestFmt {- para_runs: vec![- (ufc(0), ufc(6), papx(1, &[])),- (ufc(6), ufc(14), papx(0, &[sprm1(0x2640, 1)])),- (ufc(14), ufc(19), papx(0, &[])),- ],- ..TestFmt::default()- };- let data = build_doc_fmt(&[TestPiece::Unicode(text)], 0x0409, None, &fmt);- let document = parse(&data);- let kinds: Vec<ParagraphKind> = document- .blocks- .iter()- .filter_map(|b| match b {- Block::Paragraph(p) => Some(p.kind),- _ => None,- })- .collect();- assert_eq!(- kinds,- vec![- ParagraphKind::Heading(1),- ParagraphKind::Heading(2),- ParagraphKind::Normal- ]- );- }-- #[test]- fn table_from_cell_and_row_marks() {- let text = "A1\u{7}B1\u{7}\u{7}A2\u{7}B2\u{7}\u{7}after\r";- let in_table = papx(0, &[sprm1(0x2416, 1)]);- let row_end = papx(0, &[sprm1(0x2416, 1), sprm1(0x2417, 1)]);- let fmt = TestFmt {- para_runs: vec![- (ufc(0), ufc(3), in_table.clone()),- (ufc(3), ufc(6), in_table.clone()),- (ufc(6), ufc(7), row_end.clone()),- (ufc(7), ufc(10), in_table.clone()),- (ufc(10), ufc(13), in_table),- (ufc(13), ufc(14), row_end),- (ufc(14), ufc(20), papx(0, &[])),- ],- ..TestFmt::default()- };- let data = build_doc_fmt(&[TestPiece::Unicode(text)], 0x0409, None, &fmt);- let document = parse(&data);-- let Block::Table(table) = &document.blocks[0] else {- panic!("expected table, got {:?}", document.blocks[0]);- };- let cells: Vec<Vec<String>> = table- .rows- .iter()- .map(|row| {- row- .cells- .iter()- .map(|cell| {- cell- .blocks- .iter()- .map(|b| match b {- Block::Paragraph(p) => inline_text(&p.inlines),- _ => String::new(),- })- .collect::<String>()- })- .collect()- })- .collect();- assert_eq!(cells, vec![vec!["A1", "B1"], vec!["A2", "B2"]]);- assert_eq!(paragraph_texts(&parse(&data)).last().unwrap(), "after");- }-- #[test]- fn legacy_word_95_contiguous_text() {- let text_bytes = cp1251("Стаття перша.\rСтаття друга.\r");- let fc_min = 0x200usize;- let mut word_doc = vec![0u8; fc_min];- word_doc.extend_from_slice(&text_bytes);- word_doc[0x00..0x02].copy_from_slice(&MAGIC_WORD_6_95.to_le_bytes());- word_doc[0x06..0x08].copy_from_slice(&0x0419u16.to_le_bytes());- word_doc[0x18..0x1C].copy_from_slice(&(fc_min as u32).to_le_bytes());- word_doc[0x1C..0x20].copy_from_slice(&((fc_min + text_bytes.len()) as u32).to_le_bytes());- let data = build_cfb(&[("WordDocument", &word_doc)]);- assert_eq!(- paragraph_texts(&parse(&data)),- vec!["Стаття перша.", "Стаття друга."]- );- }-- #[test]- fn malformed_piece_table_output_is_capped() {- // 200 pieces of 100k chars all mapped to the same 100k file bytes; the- // claimed 20M chars must clamp to MAX_TEXT_CHARS- let text_len = 100_000u32;- let num_pieces = 200u32;- let mut word_doc = vec![0u8; TEXT_START];- word_doc.extend_from_slice(&vec![b'A'; text_len as usize]);-- let mut plc = Vec::new();- for k in 0..=num_pieces {- plc.extend_from_slice(&(k * text_len).to_le_bytes());- }- let fc = (TEXT_START as u32 * 2) | 0x4000_0000;- for _ in 0..num_pieces {- plc.extend_from_slice(&0u16.to_le_bytes());- plc.extend_from_slice(&fc.to_le_bytes());- plc.extend_from_slice(&0u16.to_le_bytes());- }- let mut table = vec![0x02];- table.extend_from_slice(&(plc.len() as u32).to_le_bytes());- table.extend_from_slice(&plc);-- word_doc[0x00..0x02].copy_from_slice(&MAGIC_WORD_97.to_le_bytes());- word_doc[0x0A..0x0C].copy_from_slice(&0x0200u16.to_le_bytes());- word_doc[0x4C..0x50].copy_from_slice(&(num_pieces * text_len).to_le_bytes());- word_doc[0x1A6..0x1AA].copy_from_slice(&(table.len() as u32).to_le_bytes());-- let data = build_cfb(&[("WordDocument", &word_doc), ("1Table", &table)]);- let total: usize = paragraph_texts(&parse(&data)).iter().map(|t| t.len()).sum();- assert!(total > 0);- assert!(total <= MAX_TEXT_CHARS);- }-- #[test]- fn encrypted_documents_error() {- let (mut word_doc, table) = build_word_doc_streams(&[TestPiece::Unicode("secret\r")], 0x0409);- word_doc[0x0A..0x0C].copy_from_slice(&0x0300u16.to_le_bytes());- let data = build_cfb(&[("WordDocument", &word_doc), ("1Table", &table)]);- let err = DocProvider::new().parse_buffer(&data).unwrap_err();- assert!(err.to_string().contains("encrypted"));- }-- #[test]- fn corrupt_piece_table_falls_back_to_fc_range() {- let (mut word_doc, _) =- build_word_doc_streams(&[TestPiece::Unicode("good text here\r")], 0x0409);- word_doc[0x0A..0x0C].copy_from_slice(&0x1200u16.to_le_bytes());- let fc_mac = word_doc.len() as u32;- word_doc[0x18..0x1C].copy_from_slice(&(TEXT_START as u32).to_le_bytes());- word_doc[0x1C..0x20].copy_from_slice(&fc_mac.to_le_bytes());- let garbage_table = vec![0xFFu8; 64];- let data = build_cfb(&[("WordDocument", &word_doc), ("1Table", &garbage_table)]);- assert_eq!(paragraph_texts(&parse(&data)), vec!["good text here"]);- }-}diff --git a/apps/api/native/src/document/providers/docx.rs b/apps/api/native/src/document/providers/docx.rsdeleted file mode 100644index 2e2b60c8b..000000000--- a/apps/api/native/src/document/providers/docx.rs+++ /dev/null@@ -1,816 +0,0 @@-//! DOCX (Office Open XML) provider.--use crate::document::model::*;-use crate::document::providers::DocumentProvider;-use crate::document::xml::{attr, child, children, is_tag, read_zip_text, strip_bom};-use chrono::{DateTime, Utc};-use roxmltree::{Document as XmlDoc, Node};-use std::collections::{BTreeSet, HashMap};-use std::error::Error;-use std::io::{Cursor, Read, Seek};-use std::num::NonZeroU32;-use zip::read::ZipArchive;--pub struct DocxProvider;--impl DocxProvider {- pub fn new() -> Self {- Self- }-}--/// Everything block parsing needs besides the XML itself.-#[derive(Clone, Copy)]-struct Ctx<'a> {- rels: &'a Relationships,- styles: &'a StylesInfo,- size_buckets: &'a HashMap<String, Vec<u32>>,- numbering: &'a NumberingInfo,-}--impl DocumentProvider for DocxProvider {- fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {- let mut zip = ZipArchive::new(Cursor::new(data))?;-- let metadata = read_core_properties(&mut zip).unwrap_or_default();- let styles = read_styles(&mut zip);- let numbering = read_numbering(&mut zip);- let rels = read_relationships(&mut zip, "word/_rels/document.xml.rels");-- let document_xml = read_zip_text(&mut zip, "word/document.xml")- .ok_or("Missing word/document.xml in document")?;- let xml = XmlDoc::parse(strip_bom(&document_xml))?;-- let size_buckets = compute_style_size_buckets(&xml, &styles);- let ctx = Ctx {- rels: &rels,- styles: &styles,- size_buckets: &size_buckets,- numbering: &numbering,- };-- let blocks = xml- .descendants()- .find(|n| is_tag(n, "body"))- .map(|body| parse_blocks(&body, ctx))- .unwrap_or_default();-- let mut notes = read_notes(&mut zip, NoteKind::Footnote, ctx);- notes.extend(read_notes(&mut zip, NoteKind::Endnote, ctx));- let comments = read_comments(&mut zip, ctx);-- Ok(Document {- blocks,- metadata,- notes,- comments,- })- }-- fn name(&self) -> &'static str {- "docx"- }-}--#[derive(Debug, Clone, Default)]-struct Relationships {- targets: HashMap<String, String>,-}--impl Relationships {- fn get(&self, id: &str) -> Option<&str> {- self.targets.get(id).map(|s| s.as_str())- }-}--fn read_relationships<R: Read + Seek>(zip: &mut ZipArchive<R>, path: &str) -> Relationships {- let mut rels = Relationships::default();- let Some(text) = read_zip_text(zip, path) else {- return rels;- };- let Ok(xml) = XmlDoc::parse(strip_bom(&text)) else {- return rels;- };- for rel in xml.descendants().filter(|n| is_tag(n, "Relationship")) {- if let (Some(id), Some(target)) = (attr(&rel, "Id"), attr(&rel, "Target")) {- rels.targets.insert(id.to_string(), target.to_string());- }- }- rels-}--fn read_core_properties<R: Read + Seek>(zip: &mut ZipArchive<R>) -> Option<DocumentMetadata> {- let text = read_zip_text(zip, "docProps/core.xml")?;- let xml = XmlDoc::parse(strip_bom(&text)).ok()?;- let element_text = |local: &str| {- xml- .descendants()- .find(|n| is_tag(n, local))- .and_then(|n| n.text())- };-- let mut meta = DocumentMetadata::default();- if let Some(title) = element_text("title") {- let trimmed = title.trim();- if !trimmed.is_empty() {- meta.title = Some(trimmed.to_string());- }- }- if let Some(author) = element_text("creator") {- let trimmed = author.trim();- if !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("unknown") {- meta.author = Some(trimmed.to_string());- }- }- if let Some(created) = element_text("created") {- if let Ok(dt) = DateTime::parse_from_rfc3339(created) {- meta.created = Some(DateTime::<Utc>::from(dt));- }- }- Some(meta)-}--#[derive(Debug, Default)]-struct StylesInfo {- outline_level_by_style_id: HashMap<String, u8>,- name_by_style_id: HashMap<String, String>,- default_size_by_style_id: HashMap<String, u32>,-}--fn read_styles<R: Read + Seek>(zip: &mut ZipArchive<R>) -> StylesInfo {- let mut info = StylesInfo::default();- let Some(text) = read_zip_text(zip, "word/styles.xml") else {- return info;- };- let Ok(xml) = XmlDoc::parse(strip_bom(&text)) else {- return info;- };-- for style in xml.descendants().filter(|n| is_tag(n, "style")) {- if attr(&style, "type") != Some("paragraph") {- continue;- }- let Some(style_id) = attr(&style, "styleId") else {- continue;- };-- if let Some(name) = child(&style, "name").and_then(|n| attr(&n, "val")) {- info- .name_by_style_id- .insert(style_id.to_string(), name.to_string());- }- if let Some(size) = child(&style, "rPr")- .and_then(|rpr| child(&rpr, "sz"))- .and_then(|n| attr(&n, "val"))- .and_then(|v| v.parse::<u32>().ok())- {- info- .default_size_by_style_id- .insert(style_id.to_string(), size);- }- if let Some(level) = child(&style, "pPr")- .and_then(|ppr| child(&ppr, "outlineLvl"))- .and_then(|n| attr(&n, "val"))- .and_then(|v| v.parse::<u8>().ok())- {- info- .outline_level_by_style_id- .insert(style_id.to_string(), (level + 1).min(6));- }- }- info-}--enum StyleKind {- Heading(u8),- Blockquote,-}--/// Heading or quote semantics of a paragraph style, from the style's outline-/// level when declared and name/id conventions otherwise.-fn style_kind(style_id: &str, styles: &StylesInfo) -> Option<StyleKind> {- if let Some(level) = styles.outline_level_by_style_id.get(style_id) {- return Some(StyleKind::Heading(*level));- }-- let name = styles.name_by_style_id.get(style_id);- if let Some(name) = name {- if let Some(level) = parse_heading_level(name) {- return Some(StyleKind::Heading(level));- }- if name.to_ascii_lowercase().contains("quote") {- return Some(StyleKind::Blockquote);- }- }- if let Some(level) = parse_heading_level(style_id) {- return Some(StyleKind::Heading(level));- }-- let id_lower = style_id.to_ascii_lowercase();- let name_lower = name.map(|s| s.to_ascii_lowercase()).unwrap_or_default();- if name_lower.contains("title") || id_lower.contains("title") {- Some(StyleKind::Heading(1))- } else if name_lower.contains("heading") || id_lower.contains("heading") {- Some(StyleKind::Heading(2))- } else if name_lower.contains("quote") || id_lower.contains("quote") {- Some(StyleKind::Blockquote)- } else {- None- }-}--fn parse_heading_level(s: &str) -> Option<u8> {- let lower = s.to_ascii_lowercase();- let rest = &lower[lower.find("heading")? + "heading".len()..];- let digits: String = rest- .chars()- .skip_while(|c| c.is_whitespace() || *c == '-')- .take_while(|c| c.is_ascii_digit())- .collect();- match digits.parse::<u8>() {- Ok(n) if n >= 1 => Some(n.min(6)),- _ => None,- }-}--fn paragraph_style_id<'a>(p: &Node<'a, 'a>) -> Option<&'a str> {- child(p, "pPr")- .and_then(|ppr| child(&ppr, "pStyle"))- .and_then(|n| attr(&n, "val"))-}--fn paragraph_kind(p: &Node, ctx: Ctx) -> ParagraphKind {- let Some(ppr) = child(p, "pPr") else {- return ParagraphKind::Normal;- };-- if let Some(level) = child(&ppr, "outlineLvl")- .and_then(|n| attr(&n, "val"))- .and_then(|v| v.parse::<u8>().ok())- {- return ParagraphKind::Heading((level + 1).min(6));- }-- let Some(style_id) = paragraph_style_id(p) else {- return ParagraphKind::Normal;- };- match style_kind(style_id, ctx.styles) {- Some(StyleKind::Blockquote) => ParagraphKind::Blockquote,- Some(StyleKind::Heading(mut level)) => {- // Same style at distinct font sizes: demote the smaller sizes.- if let (Some(buckets), Some(size)) = (- ctx.size_buckets.get(style_id),- paragraph_effective_size(p, ctx.styles, style_id),- ) {- if let Some(index) = buckets.iter().position(|&s| s == size) {- level = (level + index as u8).min(6);- }- }- ParagraphKind::Heading(level)- }- None => ParagraphKind::Normal,- }-}--fn paragraph_effective_size(p: &Node, styles: &StylesInfo, style_id: &str) -> Option<u32> {- let run_sizes = children(p, "r")- .filter_map(|r| child(&r, "rPr"))- .chain(child(p, "pPr").and_then(|ppr| child(&ppr, "rPr")))- .filter_map(|rpr| child(&rpr, "sz"))- .filter_map(|n| attr(&n, "val"))- .filter_map(|v| v.parse::<u32>().ok());- run_sizes- .max()- .or_else(|| styles.default_size_by_style_id.get(style_id).copied())-}--/// Distinct font sizes seen per heading/title style, largest first.-fn compute_style_size_buckets(xml: &XmlDoc, styles: &StylesInfo) -> HashMap<String, Vec<u32>> {- let mut sets: HashMap<String, BTreeSet<u32>> = HashMap::new();-- for p in xml.descendants().filter(|n| is_tag(n, "p")) {- let Some(style_id) = paragraph_style_id(&p) else {- continue;- };- let id_lower = style_id.to_ascii_lowercase();- let name_lower = styles- .name_by_style_id- .get(style_id)- .map(|s| s.to_ascii_lowercase())- .unwrap_or_default();- if !(id_lower.contains("heading")- || id_lower.contains("title")- || name_lower.contains("heading")- || name_lower.contains("title"))- {- continue;- }- if let Some(size) = paragraph_effective_size(&p, styles, style_id) {- sets.entry(style_id.to_string()).or_default().insert(size);- }- }-- sets- .into_iter()- .map(|(style_id, sizes)| {- let mut sizes: Vec<u32> = sizes.into_iter().collect();- sizes.reverse();- (style_id, sizes)- })- .collect()-}--#[derive(Clone, Default)]-struct RunStyle {- bold: Option<bool>,- italic: Option<bool>,- strike: Option<bool>,- code: Option<bool>,- vert_align: Option<VerticalAlign>,-}--#[derive(Clone, Copy)]-enum VerticalAlign {- Baseline,- Superscript,- Subscript,-}--impl RunStyle {- fn merged(&self, overrides: &RunStyle) -> RunStyle {- RunStyle {- bold: overrides.bold.or(self.bold),- italic: overrides.italic.or(self.italic),- strike: overrides.strike.or(self.strike),- code: overrides.code.or(self.code),- vert_align: overrides.vert_align.or(self.vert_align),- }- }-- fn apply(&self, local: &RunStyle, mut inlines: Vec<Inline>) -> Vec<Inline> {- let resolved = self.merged(local);-- if resolved.code.unwrap_or(false) {- let code_text: String = inlines- .iter()- .filter_map(|i| match i {- Inline::Text(s) => Some(s.as_str()),- _ => None,- })- .collect();- if !code_text.is_empty() {- return vec![Inline::Code(code_text)];- }- }-- if resolved.strike.unwrap_or(false) {- inlines = vec![Inline::Del(inlines)];- }- if resolved.italic.unwrap_or(false) {- inlines = vec![Inline::Em(inlines)];- }- if resolved.bold.unwrap_or(false) {- inlines = vec![Inline::Strong(inlines)];- }- match resolved.vert_align.unwrap_or(VerticalAlign::Baseline) {- VerticalAlign::Superscript => vec![Inline::Sup(inlines)],- VerticalAlign::Subscript => vec![Inline::Sub(inlines)],- VerticalAlign::Baseline => inlines,- }- }-}--fn read_on_off(node: &Node) -> bool {- !matches!(- attr(node, "val").map(|v| v.to_ascii_lowercase()).as_deref(),- Some("0") | Some("false") | Some("off")- )-}--fn run_style_from_rpr(rpr: &Node) -> RunStyle {- let mut style = RunStyle::default();- if let Some(b) = child(rpr, "b") {- style.bold = Some(read_on_off(&b));- }- if let Some(i) = child(rpr, "i") {- style.italic = Some(read_on_off(&i));- }- if let Some(s) = child(rpr, "strike") {- style.strike = Some(read_on_off(&s));- }- if let Some(rstyle) = child(rpr, "rStyle").and_then(|n| attr(&n, "val")) {- if rstyle.to_ascii_lowercase().contains("code") {- style.code = Some(true);- }- }- if let Some(va) = child(rpr, "vertAlign").and_then(|n| attr(&n, "val")) {- style.vert_align = Some(match va.to_ascii_lowercase().as_str() {- "sup" | "superscript" => VerticalAlign::Superscript,- "sub" | "subscript" => VerticalAlign::Subscript,- _ => VerticalAlign::Baseline,- });- }- style-}--fn paragraph_run_style(p: &Node) -> RunStyle {- child(p, "pPr")- .and_then(|ppr| child(&ppr, "rPr"))- .map(|rpr| run_style_from_rpr(&rpr))- .unwrap_or_default()-}--fn parse_paragraph(node: &Node, ctx: Ctx) -> Paragraph {- let kind = paragraph_kind(node, ctx);- let base_style = paragraph_run_style(node);- let mut inlines = Vec::new();-- for c in node.children().filter(|n| n.is_element()) {- if is_tag(&c, "r") {- inlines.extend(parse_run(&c, &base_style));- } else if is_tag(&c, "hyperlink") {- if let Some(link) = parse_hyperlink(&c, ctx.rels, &base_style) {- inlines.push(link);- }- } else if is_tag(&c, "bookmarkStart") {- if let Some(name) = attr(&c, "name") {- inlines.push(Inline::Bookmark(BookmarkId(name.to_string())));- }- } else if is_tag(&c, "br") {- inlines.push(Inline::LineBreak);- }- }-- Paragraph { kind, inlines }-}--fn parse_run(run: &Node, base_style: &RunStyle) -> Vec<Inline> {- let local_style = child(run, "rPr")- .map(|rpr| run_style_from_rpr(&rpr))- .unwrap_or_default();-- let mut out = Vec::new();- for c in run.children().filter(|n| n.is_element()) {- if is_tag(&c, "t") {- if let Some(text) = c.text() {- out.push(Inline::Text(text.to_string()));- }- } else if is_tag(&c, "br") {- out.push(Inline::LineBreak);- } else if is_tag(&c, "tab") {- out.push(Inline::Text("\t".to_string()));- } else if is_tag(&c, "footnoteReference") {- if let Some(id) = attr(&c, "id") {- out.push(Inline::FootnoteRef(NoteId(id.to_string())));- }- } else if is_tag(&c, "endnoteReference") {- if let Some(id) = attr(&c, "id") {- out.push(Inline::EndnoteRef(NoteId(id.to_string())));- }- } else if is_tag(&c, "commentReference") {- if let Some(id) = attr(&c, "id") {- out.push(Inline::CommentRef(CommentId(id.to_string())));- }- }- }-- base_style.apply(&local_style, out)-}--fn parse_hyperlink(node: &Node, rels: &Relationships, base_style: &RunStyle) -> Option<Inline> {- let href = if let Some(id) = attr(node, "id") {- rels.get(id).map(|s| s.to_string())- } else {- attr(node, "anchor").map(|anchor| format!("#{anchor}"))- }?;-- let link_style = child(node, "rPr")- .map(|rpr| run_style_from_rpr(&rpr))- .unwrap_or_default();- let combined_style = base_style.merged(&link_style);-- let mut link_children = Vec::new();- for run in children(node, "r") {- link_children.extend(parse_run(&run, &combined_style));- }-- Some(Inline::Link {- href,- children: link_children,- })-}--fn parse_blocks(parent: &Node, ctx: Ctx) -> Vec<Block> {- let nodes: Vec<Node> = parent.children().filter(|n| n.is_element()).collect();- let mut out = Vec::new();- let mut i = 0;-- while i < nodes.len() {- let node = &nodes[i];- if is_tag(node, "p") {- if paragraph_list_info(node, ctx.numbering).is_some() {- let (list, next_i) = parse_list(&nodes, i, ctx);- if !list.items.is_empty() {- out.push(Block::List(list));- }- i = next_i;- continue;- }- if let Some(image) = parse_image_paragraph(node, ctx.rels) {- out.push(Block::Image(image));- } else {- let para = parse_paragraph(node, ctx);- if has_visible_content(¶.inlines) {- out.push(Block::Paragraph(para));- }- }- } else if is_tag(node, "tbl") {- out.push(Block::Table(parse_table(node, ctx)));- }- i += 1;- }- out-}--fn parse_table(node: &Node, ctx: Ctx) -> Table {- let mut rows = Vec::new();- for tr in children(node, "tr") {- let header = child(&tr, "trPr")- .and_then(|trpr| child(&trpr, "tblHeader"))- .is_some();- let cells = children(&tr, "tc")- .map(|tc| TableCell {- blocks: parse_blocks(&tc, ctx),- colspan: NonZeroU32::MIN,- rowspan: NonZeroU32::MIN,- })- .collect();- rows.push(TableRow {- cells,- kind: if header {- TableRowKind::Header- } else {- TableRowKind::Body- },- });- }-- if rows.iter().any(|r| matches!(r.kind, TableRowKind::Header)) {- if let Some(last) = rows.last_mut() {- last.kind = TableRowKind::Footer;- }- }- Table { rows }-}--#[derive(Debug, Clone, PartialEq, Eq)]-struct ListInfo {- list_type: ListType,- num_id: String,- ilvl: u32,-}--#[derive(Debug, Default)]-struct NumberingInfo {- num_to_abstract: HashMap<String, String>,- abstract_levels: HashMap<String, HashMap<String, ListType>>,-}--impl NumberingInfo {- fn list_type(&self, num_id: &str, ilvl: &str) -> Option<ListType> {- self- .abstract_levels- .get(self.num_to_abstract.get(num_id)?)?- .get(ilvl)- .copied()- }-}--fn read_numbering<R: Read + Seek>(zip: &mut ZipArchive<R>) -> NumberingInfo {- let mut info = NumberingInfo::default();- let Some(text) = read_zip_text(zip, "word/numbering.xml") else {- return info;- };- let Ok(xml) = XmlDoc::parse(strip_bom(&text)) else {- return info;- };-- for num in xml.descendants().filter(|n| is_tag(n, "num")) {- if let (Some(num_id), Some(abs)) = (- attr(&num, "numId"),- child(&num, "abstractNumId").and_then(|n| attr(&n, "val")),- ) {- info- .num_to_abstract- .insert(num_id.to_string(), abs.to_string());- }- }-- for abs in xml.descendants().filter(|n| is_tag(n, "abstractNum")) {- let Some(abs_id) = attr(&abs, "abstractNumId") else {- continue;- };- let mut levels = HashMap::new();- for lvl in children(&abs, "lvl") {- if let Some(ilvl) = attr(&lvl, "ilvl") {- let fmt = child(&lvl, "numFmt").and_then(|n| attr(&n, "val"));- let list_type = match fmt.unwrap_or("") {- "bullet" => ListType::Unordered,- _ => ListType::Ordered,- };- levels.insert(ilvl.to_string(), list_type);- }- }- info.abstract_levels.insert(abs_id.to_string(), levels);- }- info-}--fn paragraph_list_info(p: &Node, numbering: &NumberingInfo) -> Option<ListInfo> {- let numpr = child(p, "pPr").and_then(|ppr| child(&ppr, "numPr"))?;- let ilvl_str = child(&numpr, "ilvl").and_then(|n| attr(&n, "val"))?;- let num_id = child(&numpr, "numId").and_then(|n| attr(&n, "val"))?;- Some(ListInfo {- list_type: numbering- .list_type(num_id, ilvl_str)- .unwrap_or(ListType::Unordered),- num_id: num_id.to_string(),- ilvl: ilvl_str.parse().unwrap_or(0),- })-}--/// Consumes consecutive numbered paragraphs at one level into a list,-/// recursing for deeper levels. Returns the list and the next unconsumed-/// node index.-fn parse_list(nodes: &[Node], mut i: usize, ctx: Ctx) -> (List, usize) {- let mut list = List {- items: Vec::new(),- list_type: ListType::Unordered,- };- let Some(base) = nodes- .get(i)- .and_then(|n| paragraph_list_info(n, ctx.numbering))- else {- return (list, i + 1);- };- list.list_type = base.list_type;-- while i < nodes.len() {- let node = &nodes[i];- if !is_tag(node, "p") {- break;- }- let Some(info) = paragraph_list_info(node, ctx.numbering) else {- break;- };- if info.ilvl < base.ilvl- || (info.ilvl == base.ilvl- && (info.list_type != base.list_type || info.num_id != base.num_id))- {- break;- }- if info.ilvl > base.ilvl {- // deeper level with no parent item yet; let the caller handle it- break;- }-- let mut blocks = Vec::new();- if let Some(image) = parse_image_paragraph(node, ctx.rels) {- blocks.push(Block::Image(image));- } else {- let para = parse_paragraph(node, ctx);- if has_visible_content(¶.inlines) {- blocks.push(Block::Paragraph(para));- }- }- list.items.push(ListItem { blocks });- i += 1;-- while i < nodes.len() && is_tag(&nodes[i], "p") {- match paragraph_list_info(&nodes[i], ctx.numbering) {- Some(sub) if sub.ilvl > base.ilvl => {- let (sublist, next_i) = parse_list(nodes, i, ctx);- if let Some(last) = list.items.last_mut() {- last.blocks.push(Block::List(sublist));- }- i = next_i;- }- _ => break,- }- }-- if list.items.last().is_some_and(|last| last.blocks.is_empty()) {- list.items.pop();- }- }-- (list, i)-}--fn parse_image_paragraph(p: &Node, rels: &Relationships) -> Option<Image> {- let has_text = p- .descendants()- .filter(|n| is_tag(n, "t"))- .any(|t| t.text().map(|s| !s.trim().is_empty()).unwrap_or(false));- if has_text {- return None;- }-- if let Some(drawing) = p.descendants().find(|n| is_tag(n, "drawing")) {- let blip = drawing.descendants().find(|n| is_tag(n, "blip"))?;- let rel_id = attr(&blip, "embed").or_else(|| attr(&blip, "link"))?;- let alt = drawing- .descendants()- .find(|n| is_tag(n, "docPr"))- .and_then(|n| attr(&n, "descr").or_else(|| attr(&n, "title")))- .map(|s| s.to_string());- return external_image(rels.get(rel_id)?, alt);- }-- if let Some(pict) = p.descendants().find(|n| is_tag(n, "pict")) {- let imagedata = pict.descendants().find(|n| is_tag(n, "imagedata"))?;- let rel_id = attr(&imagedata, "id")?;- let alt = attr(&imagedata, "title").map(|s| s.to_string());- return external_image(rels.get(rel_id)?, alt);- }- None-}--/// Only external (http/https) images are representable in the output.-fn external_image(target: &str, alt: Option<String>) -> Option<Image> {- if target.starts_with("http://") || target.starts_with("https://") {- Some(Image {- src: target.to_string(),- alt,- })- } else {- None- }-}--fn read_notes<R: Read + Seek>(zip: &mut ZipArchive<R>, kind: NoteKind, ctx: Ctx) -> Vec<Note> {- let (xml_path, rels_path, root_tag) = match kind {- NoteKind::Footnote => (- "word/footnotes.xml",- "word/_rels/footnotes.xml.rels",- "footnote",- ),- NoteKind::Endnote => (- "word/endnotes.xml",- "word/_rels/endnotes.xml.rels",- "endnote",- ),- };- let Some(text) = read_zip_text(zip, xml_path) else {- return Vec::new();- };- let Ok(xml) = XmlDoc::parse(strip_bom(&text)) else {- return Vec::new();- };- let rels = read_relationships(zip, rels_path);- let ctx = Ctx { rels: &rels, ..ctx };-- let mut notes = Vec::new();- for n in xml.descendants().filter(|n| is_tag(n, root_tag)) {- let Some(id) = attr(&n, "id") else {- continue;- };- if matches!(- attr(&n, "type"),- Some("separator" | "continuationSeparator")- ) {- continue;- }- notes.push(Note {- id: NoteId(id.to_string()),- kind,- blocks: parse_blocks(&n, ctx),- });- }- notes-}--fn read_comments<R: Read + Seek>(zip: &mut ZipArchive<R>, ctx: Ctx) -> Vec<Comment> {- let Some(text) = read_zip_text(zip, "word/comments.xml") else {- return Vec::new();- };- let Ok(xml) = XmlDoc::parse(strip_bom(&text)) else {- return Vec::new();- };- let rels = read_relationships(zip, "word/_rels/comments.xml.rels");- let ctx = Ctx { rels: &rels, ..ctx };-- let mut comments = Vec::new();- for c in xml.descendants().filter(|n| is_tag(n, "comment")) {- let Some(id) = attr(&c, "id") else {- continue;- };- comments.push(Comment {- id: CommentId(id.to_string()),- author_name: attr(&c, "author").map(|s| s.to_string()),- author_initials: attr(&c, "initials").map(|s| s.to_string()),- blocks: parse_blocks(&c, ctx),- });- }- comments-}diff --git a/apps/api/native/src/document/providers/factory.rs b/apps/api/native/src/document/providers/factory.rsdeleted file mode 100644index b9bdb0b52..000000000--- a/apps/api/native/src/document/providers/factory.rs+++ /dev/null@@ -1,47 +0,0 @@-use super::doc::DocProvider;-use super::docx::DocxProvider;-use super::odt::OdtProvider;-use super::rtf::RtfProvider;-use super::xlsx::XlsxProvider;-use super::DocumentProvider;-use napi_derive::napi;--#[napi]-#[derive(Debug, Clone, PartialEq, Eq)]-pub enum DocumentType {- Doc,- Docx,- Rtf,- Odt,- Xlsx,-}--pub struct ProviderFactory {- doc_provider: DocProvider,- docx_provider: DocxProvider,- rtf_provider: RtfProvider,- odt_provider: OdtProvider,- xlsx_provider: XlsxProvider,-}--impl ProviderFactory {- pub fn new() -> Self {- Self {- doc_provider: DocProvider::new(),- docx_provider: DocxProvider::new(),- rtf_provider: RtfProvider::new(),- odt_provider: OdtProvider::new(),- xlsx_provider: XlsxProvider::new(),- }- }-- pub fn get_provider(&self, doc_type: DocumentType) -> &dyn DocumentProvider {- match doc_type {- DocumentType::Doc => &self.doc_provider,- DocumentType::Docx => &self.docx_provider,- DocumentType::Rtf => &self.rtf_provider,- DocumentType::Odt => &self.odt_provider,- DocumentType::Xlsx => &self.xlsx_provider,- }- }-}diff --git a/apps/api/native/src/document/providers/mod.rs b/apps/api/native/src/document/providers/mod.rsdeleted file mode 100644index 645d0fdba..000000000--- a/apps/api/native/src/document/providers/mod.rs+++ /dev/null@@ -1,16 +0,0 @@-use crate::document::model::Document;-use std::error::Error;--pub mod doc;-pub mod docx;-pub mod factory;-pub mod odt;-pub mod rtf;-pub mod xlsx;--pub trait DocumentProvider {- fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>>;-- #[allow(dead_code)]- fn name(&self) -> &'static str;-}diff --git a/apps/api/native/src/document/providers/odt.rs b/apps/api/native/src/document/providers/odt.rsdeleted file mode 100644index 92a35593a..000000000--- a/apps/api/native/src/document/providers/odt.rs+++ /dev/null@@ -1,617 +0,0 @@-//! ODT (OpenDocument text) provider.--use crate::document::model::*;-use crate::document::providers::DocumentProvider;-use crate::document::xml::{attr, child, children, is_tag, read_zip_text, strip_bom};-use chrono::{DateTime, Utc};-use roxmltree::{Document as XmlDoc, Node};-use std::collections::{HashMap, HashSet};-use std::error::Error;-use std::io::{Cursor, Read, Seek};-use std::num::NonZeroU32;-use zip::read::ZipArchive;--pub struct OdtProvider;--impl OdtProvider {- pub fn new() -> Self {- Self- }-}--impl DocumentProvider for OdtProvider {- fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {- let mut zip = ZipArchive::new(Cursor::new(data))?;-- let metadata = read_meta(&mut zip).unwrap_or_default();- let styles = read_styles(&mut zip);-- let content =- read_zip_text(&mut zip, "content.xml").ok_or("Missing content.xml in document")?;- let xml = XmlDoc::parse(strip_bom(&content))?;-- let mut parser = Parser {- styles: &styles,- notes: Vec::new(),- comments: Vec::new(),- };- let blocks = xml- .descendants()- .find(|n| is_tag(n, "text") && n.ancestors().any(|a| is_tag(&a, "body")))- .map(|body| parser.parse_blocks(&body))- .unwrap_or_default();-- Ok(Document {- blocks,- metadata,- notes: parser.notes,- comments: parser.comments,- })- }-- fn name(&self) -> &'static str {- "odt"- }-}--fn read_meta<R: Read + Seek>(zip: &mut ZipArchive<R>) -> Option<DocumentMetadata> {- let text = read_zip_text(zip, "meta.xml")?;- let xml = XmlDoc::parse(strip_bom(&text)).ok()?;- let element_text = |local: &str| {- xml- .descendants()- .find(|n| is_tag(n, local))- .and_then(|n| n.text())- };-- let mut meta = DocumentMetadata::default();- if let Some(title) = element_text("title") {- if !title.trim().is_empty() {- meta.title = Some(title.to_string());- }- }- if let Some(author) = element_text("creator").or_else(|| element_text("initial-creator")) {- let trimmed = author.trim();- if !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("unknown") {- meta.author = Some(trimmed.to_string());- }- }- if let Some(created) = element_text("creation-date") {- if let Ok(dt) = DateTime::parse_from_rfc3339(created) {- meta.created = Some(DateTime::<Utc>::from(dt));- }- }- Some(meta)-}--#[derive(Debug, Default)]-struct OdtStylesInfo {- paragraph_style_names: HashSet<String>,- paragraph_outline_level: HashMap<String, u8>,- paragraph_text_props: HashMap<String, TextStyleProps>,- text_props: HashMap<String, TextStyleProps>,- list_is_ordered: HashMap<String, bool>,-}--#[derive(Debug, Default, Clone, Copy)]-struct TextStyleProps {- bold: bool,- italic: bool,- strike: bool,- sup: bool,- sub: bool,- code: bool,-}--fn read_styles<R: Read + Seek>(zip: &mut ZipArchive<R>) -> OdtStylesInfo {- let mut info = OdtStylesInfo::default();- for path in ["styles.xml", "content.xml"] {- if let Some(text) = read_zip_text(zip, path) {- if let Ok(xml) = XmlDoc::parse(strip_bom(&text)) {- harvest_styles(&xml, &mut info);- }- }- }- info-}--fn harvest_styles(xml: &XmlDoc, out: &mut OdtStylesInfo) {- for style in xml.descendants().filter(|n| is_tag(n, "style")) {- let (Some(family), Some(name)) = (attr(&style, "family"), attr(&style, "name")) else {- continue;- };-- match family {- "paragraph" => {- out.paragraph_style_names.insert(name.to_string());-- let outline_level = child(&style, "paragraph-properties")- .and_then(|pp| attr(&pp, "outline-level"))- .and_then(|v| v.parse::<u8>().ok())- .map(|v| v.min(6))- .or_else(|| attr(&style, "parent-style-name").and_then(parse_heading_level));- if let Some(level) = outline_level {- out.paragraph_outline_level.insert(name.to_string(), level);- }-- if let Some(tp) = child(&style, "text-properties") {- out- .paragraph_text_props- .insert(name.to_string(), parse_text_properties(&tp));- }- }- "text" => {- if let Some(tp) = child(&style, "text-properties") {- out- .text_props- .insert(name.to_string(), parse_text_properties(&tp));- }- }- "list" => {- let is_ordered = style- .children()- .any(|c| is_tag(&c, "list-level-style-number"));- out.list_is_ordered.insert(name.to_string(), is_ordered);- }- _ => {}- }- }-- for list_style in xml.descendants().filter(|n| is_tag(n, "list-style")) {- if let Some(name) = attr(&list_style, "name") {- let is_ordered = list_style- .children()- .any(|c| is_tag(&c, "list-level-style-number"));- out.list_is_ordered.insert(name.to_string(), is_ordered);- }- }-}--fn parse_text_properties(tp: &Node) -> TextStyleProps {- let mut props = TextStyleProps::default();-- if attr(tp, "font-weight").is_some_and(|v| v.eq_ignore_ascii_case("bold")) {- props.bold = true;- }- if attr(tp, "font-style").is_some_and(|v| v.eq_ignore_ascii_case("italic")) {- props.italic = true;- }- if attr(tp, "text-line-through-type")- .or_else(|| attr(tp, "text-line-through-style"))- .is_some_and(|v| v != "none")- {- props.strike = true;- }- if let Some(v) = attr(tp, "text-position") {- let lower = v.to_ascii_lowercase();- if lower.contains("sup") {- props.sup = true;- } else if lower.contains("sub") {- props.sub = true;- }- }- if let Some(font) = attr(tp, "font-name") {- let lower = font.to_ascii_lowercase();- if lower.contains("courier") || lower.contains("mono") {- props.code = true;- }- }- props-}--/// Heading level from a style name like "Heading_20_2" or "Title".-fn parse_heading_level(style_name: &str) -> Option<u8> {- let normalized = style_name.replace("_20_", " ").replace('_', " ");- let lower = normalized.to_ascii_lowercase();- if lower.contains("title") {- return Some(1);- }- let tail = &lower[lower.find("heading")? + "heading".len()..];- let digits: String = tail.chars().filter(|c| c.is_ascii_digit()).collect();- digits.parse::<u8>().ok().map(|n| n.clamp(1, 6))-}--struct Parser<'a> {- styles: &'a OdtStylesInfo,- notes: Vec<Note>,- comments: Vec<Comment>,-}--impl Parser<'_> {- fn parse_blocks(&mut self, node: &Node) -> Vec<Block> {- let mut blocks = Vec::new();-- for c in node.children().filter(|n| n.is_element()) {- if is_tag(&c, "h") {- self.push_paragraph(&c, &mut blocks);- } else if is_tag(&c, "p") {- if let Some(image) = image_from_paragraph(&c) {- blocks.push(Block::Image(image));- } else {- self.push_paragraph(&c, &mut blocks);- }- } else if is_tag(&c, "list") {- self.push_list(&c, &mut blocks);- } else if is_tag(&c, "table") {- blocks.push(Block::Table(self.parse_table(&c)));- } else {- blocks.extend(self.parse_blocks(&c));- }- }- blocks- }-- fn push_paragraph(&mut self, node: &Node, blocks: &mut Vec<Block>) {- let para = self.parse_paragraph(node);- if has_visible_content(¶.inlines) {- blocks.push(Block::Paragraph(para));- }- }-- fn push_list(&mut self, list: &Node, blocks: &mut Vec<Block>) {- // Writers wrap continuation sublists in a single-item shell list; unwrap- // and attach them to the previous list's last item.- let mut effective = *list;- let mut inherited_style_name = attr(&effective, "style-name");- let mut unwrapped = false;- while let Some(inner) = unwrap_single_nested_list(&effective) {- effective = inner;- unwrapped = true;- if inherited_style_name.is_none() {- inherited_style_name = attr(&effective, "style-name");- }- }-- if is_heading_list(&effective) {- for li in children(&effective, "list-item") {- let headings: Vec<Node> = li.descendants().filter(|n| is_tag(n, "h")).collect();- if let Some(h) = headings.first() {- self.push_paragraph(h, blocks);- }- }- return;- }-- let parsed = self.parse_list(&effective, inherited_style_name);- if unwrapped {- if let Some(Block::List(prev)) = blocks.last_mut() {- if let Some(last_item) = prev.items.last_mut() {- last_item.blocks.push(Block::List(parsed));- return;- }- }- }- blocks.push(Block::List(parsed));- }-- fn parse_list(&mut self, node: &Node, inherited_style_name: Option<&str>) -> List {- let style_name = attr(node, "style-name").or(inherited_style_name);- let list_type = match style_name.and_then(|n| self.styles.list_is_ordered.get(n)) {- Some(true) => ListType::Ordered,- _ => ListType::Unordered,- };-- let items = children(node, "list-item")- .map(|li| ListItem {- blocks: self.parse_blocks(&li),- })- .collect();- List { items, list_type }- }-- fn parse_table(&mut self, node: &Node) -> Table {- let mut rows = Vec::new();- for tr in children(node, "table-row") {- let cells = children(&tr, "table-cell")- .map(|tc| TableCell {- blocks: self.parse_blocks(&tc),- colspan: span_attr(&tc, "number-columns-spanned"),- rowspan: span_attr(&tc, "number-rows-spanned"),- })- .collect();- rows.push(TableRow {- cells,- kind: TableRowKind::Body,- });- }- Table { rows }- }-- fn parse_paragraph(&mut self, node: &Node) -> Paragraph {- let kind = self.paragraph_kind(node);- let base = attr(node, "style-name")- .and_then(|name| self.styles.paragraph_text_props.get(name))- .copied()- .unwrap_or_default();- let inlines = self.parse_inlines(node);- Paragraph {- kind,- inlines: apply_text_style(inlines, None, self.styles, base),- }- }-- fn paragraph_kind(&self, p: &Node) -> ParagraphKind {- if p.tag_name().name() == "h" {- let level = attr(p, "outline-level")- .and_then(|v| v.parse::<u8>().ok())- .unwrap_or(1);- return ParagraphKind::Heading(level.clamp(1, 6));- }-- if let Some(style_name) = attr(p, "style-name") {- if let Some(level) = self.styles.paragraph_outline_level.get(style_name) {- return ParagraphKind::Heading((*level).min(6));- }- if self.styles.paragraph_style_names.contains(style_name)- && style_name.to_ascii_lowercase().contains("quote")- {- return ParagraphKind::Blockquote;- }- }- ParagraphKind::Normal- }-- fn parse_inlines(&mut self, node: &Node) -> Vec<Inline> {- let mut out = Vec::new();-- for c in node.children() {- if c.is_text() {- if let Some(t) = c.text() {- if !t.is_empty() {- out.push(Inline::Text(t.to_string()));- }- }- continue;- }- if !c.is_element() {- continue;- }-- if is_tag(&c, "span") {- let inner = self.parse_inlines(&c);- let style_name = attr(&c, "style-name");- out.extend(apply_text_style(- inner,- style_name,- self.styles,- TextStyleProps::default(),- ));- } else if is_tag(&c, "a") {- let link_children = self.parse_inlines(&c);- match attr(&c, "href") {- Some(href) => out.push(Inline::Link {- href: href.to_string(),- children: link_children,- }),- None => out.extend(link_children),- }- } else if is_tag(&c, "line-break") {- out.push(Inline::LineBreak);- } else if is_tag(&c, "s") {- let count = attr(&c, "c")- .and_then(|v| v.parse::<usize>().ok())- .unwrap_or(1)- .min(1000);- out.push(Inline::Text(" ".repeat(count)));- } else if is_tag(&c, "tab") {- out.push(Inline::Text("\t".to_string()));- } else if is_tag(&c, "bookmark-start") {- if let Some(name) = attr(&c, "name") {- out.push(Inline::Bookmark(BookmarkId(name.to_string())));- }- } else if is_tag(&c, "note") {- out.push(self.parse_note(&c));- } else if is_tag(&c, "annotation") {- out.push(self.parse_annotation(&c));- } else {- out.extend(self.parse_inlines(&c));- }- }- out- }-- fn parse_note(&mut self, node: &Node) -> Inline {- let kind = match attr(node, "note-class") {- Some("endnote") => NoteKind::Endnote,- _ => NoteKind::Footnote,- };- let id = attr(node, "id")- .map(|s| s.to_string())- .unwrap_or_else(|| format!("odt-note-{}", self.notes.len() + 1));-- let blocks = child(node, "note-body")- .map(|body| self.parse_note_body(&body))- .unwrap_or_default();- self.notes.push(Note {- id: NoteId(id.clone()),- kind,- blocks,- });-- match kind {- NoteKind::Footnote => Inline::FootnoteRef(NoteId(id)),- NoteKind::Endnote => Inline::EndnoteRef(NoteId(id)),- }- }-- fn parse_note_body(&mut self, node: &Node) -> Vec<Block> {- let mut blocks = Vec::new();- let paragraphs: Vec<Node> = node- .children()- .filter(|n| is_tag(n, "p") || is_tag(n, "h"))- .collect();- for p in paragraphs {- let para = self.parse_paragraph(&p);- if has_visible_content(¶.inlines) {- blocks.push(Block::Paragraph(para));- }- }- blocks- }-- fn parse_annotation(&mut self, node: &Node) -> Inline {- let id = format!("odt-comment-{}", self.comments.len() + 1);- let descendant_text = |local: &str| {- node- .descendants()- .find(|n| is_tag(n, local))- .and_then(|n| n.text())- .map(str::trim)- .filter(|t| !t.is_empty())- .map(str::to_string)- };- let author = descendant_text("creator");- let initials = descendant_text("initials");-- let mut blocks = Vec::new();- let paragraphs: Vec<Node> = node.children().filter(|n| is_tag(n, "p")).collect();- for p in paragraphs {- let inlines = self.parse_inlines(&p);- if !inlines.is_empty() {- blocks.push(Block::Paragraph(Paragraph {- kind: ParagraphKind::Normal,- inlines,- }));- }- }-- self.comments.push(Comment {- id: CommentId(id.clone()),- author_name: author,- author_initials: initials,- blocks,- });- Inline::CommentRef(CommentId(id))- }-}--fn span_attr(node: &Node, local: &str) -> NonZeroU32 {- attr(node, local)- .and_then(|v| v.parse::<u32>().ok())- .and_then(NonZeroU32::new)- .unwrap_or(NonZeroU32::MIN)-}--fn unwrap_single_nested_list<'a>(list: &Node<'a, 'a>) -> Option<Node<'a, 'a>> {- let mut items = children(list, "list-item");- let first = items.next()?;- if items.next().is_some() {- return None;- }- let mut inner_lists = first.children().filter(|n| is_tag(n, "list"));- let inner = inner_lists.next()?;- if inner_lists.next().is_some() {- return None;- }- Some(inner)-}--fn is_heading_list(list: &Node) -> bool {- let mut any = false;- for li in children(list, "list-item") {- any = true;- if !li.descendants().any(|n| is_tag(&n, "h")) {- return false;- }- }- any-}--fn apply_text_style(- mut inlines: Vec<Inline>,- style_name: Option<&str>,- styles: &OdtStylesInfo,- base: TextStyleProps,-) -> Vec<Inline> {- let mut props = base;- if let Some(name) = style_name {- let mut style_props = styles.text_props.get(name).copied().unwrap_or_default();- if name.to_ascii_lowercase().contains("code") {- style_props.code = true;- }- props.bold |= style_props.bold;- props.italic |= style_props.italic;- props.strike |= style_props.strike;- props.sup |= style_props.sup;- props.sub |= style_props.sub;- props.code |= style_props.code;- }-- if props.code {- let code_text: String = inlines- .iter()- .filter_map(|i| match i {- Inline::Text(s) => Some(s.as_str()),- _ => None,- })- .collect();- if !code_text.is_empty() {- return vec![Inline::Code(code_text)];- }- }-- if props.strike {- inlines = vec![Inline::Del(inlines)];- }- if props.italic {- inlines = vec![Inline::Em(inlines)];- }- if props.bold {- inlines = vec![Inline::Strong(inlines)];- }- if props.sup {- inlines = vec![Inline::Sup(inlines)];- }- if props.sub {- inlines = vec![Inline::Sub(inlines)];- }- inlines-}--fn image_from_paragraph(p: &Node) -> Option<Image> {- let image = p.descendants().find(|n| is_tag(n, "image"))?;- let href = attr(&image, "href")?;- // only external images are representable in the output- if href.starts_with("http://") || href.starts_with("https://") {- Some(Image {- src: href.to_string(),- alt: None,- })- } else {- None- }-}--#[cfg(test)]-mod tests {- use super::*;- use std::io::Write;-- #[test]- fn huge_space_count_is_clamped() {- let content = r#"<?xml version="1.0"?>-<office:document-content xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0">-<office:body><office:text><text:p>a<text:s text:c="4000000000"/>b</text:p></office:text></office:body>-</office:document-content>"#;- let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));- zip- .start_file("content.xml", zip::write::SimpleFileOptions::default())- .unwrap();- zip.write_all(content.as_bytes()).unwrap();- let data = zip.finish().unwrap().into_inner();-- let document = OdtProvider::new().parse_buffer(&data).unwrap();- let Block::Paragraph(p) = &document.blocks[0] else {- panic!("expected paragraph");- };- let text: String = p- .inlines- .iter()- .map(|i| match i {- Inline::Text(t) => t.as_str(),- _ => "",- })- .collect();- assert_eq!(text.len(), "ab".len() + 1000);- }-}diff --git a/apps/api/native/src/document/providers/rtf.rs b/apps/api/native/src/document/providers/rtf.rsdeleted file mode 100644index 260216114..000000000--- a/apps/api/native/src/document/providers/rtf.rs+++ /dev/null@@ -1,822 +0,0 @@-//! RTF provider. Tokenizes the RTF control stream and decodes 8-bit text-//! through the code page the document declares with \ansicpg.--use crate::document::encoding::encoding_for_codepage;-use crate::document::model::*;-use crate::document::providers::DocumentProvider;-use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc};-use encoding_rs::{Encoding, WINDOWS_1252};-use std::error::Error;-use std::num::NonZeroU32;--pub struct RtfProvider;--impl RtfProvider {- pub fn new() -> Self {- Self- }-}--impl DocumentProvider for RtfProvider {- fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {- let encoding = detect_encoding(data);- let metadata = extract_metadata(data, encoding);- let blocks = parse_body(data, encoding);-- Ok(Document {- blocks,- metadata,- notes: Vec::new(),- comments: Vec::new(),- })- }-- fn name(&self) -> &'static str {- "rtf"- }-}--/// Code page declared by the first \ansicpg control word.-fn detect_encoding(src: &[u8]) -> &'static Encoding {- for token in Tokens::new(src) {- if let Token::Control("ansicpg", Some(codepage)) = token {- return u16::try_from(codepage)- .ok()- .and_then(encoding_for_codepage)- .unwrap_or(WINDOWS_1252);- }- }- WINDOWS_1252-}--#[derive(Debug, PartialEq)]-enum Token<'a> {- Open,- Close,- Control(&'a str, Option<i32>),- Symbol(u8),- Hex(u8),- Byte(u8),-}--struct Tokens<'a> {- src: &'a [u8],- pos: usize,-}--impl<'a> Tokens<'a> {- fn new(src: &'a [u8]) -> Self {- Self { src, pos: 0 }- }-}--impl<'a> Iterator for Tokens<'a> {- type Item = Token<'a>;-- fn next(&mut self) -> Option<Token<'a>> {- loop {- let b = *self.src.get(self.pos)?;- self.pos += 1;- match b {- b'\r' | b'\n' => continue,- b'{' => return Some(Token::Open),- b'}' => return Some(Token::Close),- b'\\' => return self.next_after_backslash(),- b => return Some(Token::Byte(b)),- }- }- }-}--impl<'a> Tokens<'a> {- fn next_after_backslash(&mut self) -> Option<Token<'a>> {- let b = *self.src.get(self.pos)?;- self.pos += 1;- match b {- b'\'' => {- let hi = self.src.get(self.pos).copied().and_then(hex_val);- let lo = self.src.get(self.pos + 1).copied().and_then(hex_val);- match (hi, lo) {- (Some(hi), Some(lo)) => {- self.pos += 2;- Some(Token::Hex((hi << 4) | lo))- }- // malformed escape; drop it and keep tokenizing- _ => Some(Token::Symbol(b'\'')),- }- }- b'a'..=b'z' | b'A'..=b'Z' => {- let start = self.pos - 1;- while self.src.get(self.pos).is_some_and(u8::is_ascii_alphabetic) {- self.pos += 1;- }- let word = std::str::from_utf8(&self.src[start..self.pos]).ok()?;-- let mut value = None;- let negative = self.src.get(self.pos) == Some(&b'-');- let num_start = self.pos + negative as usize;- let mut num_end = num_start;- while self.src.get(num_end).is_some_and(u8::is_ascii_digit) {- num_end += 1;- }- if num_end > num_start {- // consumed even when the value overflows i32- self.pos = num_end;- if let Some(n) = std::str::from_utf8(&self.src[num_start..num_end])- .ok()- .and_then(|s| s.parse::<i32>().ok())- {- value = Some(if negative { -n } else { n });- }- }- if self.src.get(self.pos) == Some(&b' ') {- self.pos += 1;- }- Some(Token::Control(word, value))- }- b => {- if b == b'*' && self.src.get(self.pos) == Some(&b' ') {- self.pos += 1;- }- Some(Token::Symbol(b))- }- }- }-}--/// Fallback byte count declared by \uc.-fn uc_count(value: Option<i32>) -> usize {- value.unwrap_or(1).max(0) as usize-}--fn hex_val(b: u8) -> Option<u8> {- match b {- b'0'..=b'9' => Some(b - b'0'),- b'a'..=b'f' => Some(10 + b - b'a'),- b'A'..=b'F' => Some(10 + b - b'A'),- _ => None,- }-}--/// Text accumulator; raw bytes are buffered and decoded as a unit.-struct TextAccum {- encoding: &'static Encoding,- pending: Vec<u8>,- text: String,-}--impl TextAccum {- fn new(encoding: &'static Encoding) -> Self {- Self {- encoding,- pending: Vec::new(),- text: String::new(),- }- }-- fn push_byte(&mut self, b: u8) {- self.pending.push(b);- }-- fn push_char(&mut self, c: char) {- self.flush_pending();- self.text.push(c);- }-- fn flush_pending(&mut self) {- if self.pending.is_empty() {- return;- }- let (decoded, _) = self.encoding.decode_without_bom_handling(&self.pending);- for ch in decoded.chars() {- if ch == '\t' || ch == '\u{00A0}' || ch as u32 >= 0x20 {- self.text.push(ch);- }- }- self.pending.clear();- }-- fn take(&mut self) -> String {- self.flush_pending();- std::mem::take(&mut self.text)- }-- fn is_empty(&mut self) -> bool {- self.flush_pending();- self.text.is_empty()- }-}--#[derive(Clone, Default)]-struct CharState {- bold: bool,- italic: bool,- strike: bool,- sup: bool,- sub: bool,-}--struct Group {- saved: CharState,- skip: bool,- name_seen: bool,-}--#[derive(Default)]-struct TableBuilder {- rows: Vec<TableRow>,- current_row: Vec<TableCell>,- current_cell_blocks: Vec<Block>,-}--impl TableBuilder {- fn start_row(&mut self) {- self.current_cell_blocks.clear();- self.current_row.clear();- }-- fn push_block(&mut self, block: Block) {- self.current_cell_blocks.push(block);- }-- fn finish_cell(&mut self) {- if self.current_cell_blocks.is_empty() {- return;- }- self.current_row.push(TableCell {- blocks: std::mem::take(&mut self.current_cell_blocks),- colspan: NonZeroU32::MIN,- rowspan: NonZeroU32::MIN,- });- }-- fn finish_row(&mut self) {- self.finish_cell();- if self.current_row.is_empty() {- return;- }- self.rows.push(TableRow {- cells: std::mem::take(&mut self.current_row),- kind: TableRowKind::Body,- });- }-- fn finalize(mut self) -> Option<Block> {- self.finish_row();- if self.rows.is_empty() {- None- } else {- Some(Block::Table(Table { rows: self.rows }))- }- }-}--const SKIP_DESTS: &[&str] = &[- "fonttbl",- "colortbl",- "stylesheet",- "listtable",- "listoverridetable",- "themedata",- "latentstyles",- "rsidtbl",- "xmlnstbl",- "mmathPr",- "wgrffmtfilter",- "datastore",- "filetbl",- "colorschememapping",- "pnseclvl1",- "pnseclvl2",- "pnseclvl3",- "pnseclvl4",- "pnseclvl5",- "pnseclvl6",- "pnseclvl7",- "pnseclvl8",- "pnseclvl9",- "pict",- "object",- "info",-];--struct BodyParser {- state: CharState,- stack: Vec<Group>,- blocks: Vec<Block>,- inlines: Vec<Inline>,- text: TextAccum,- table: Option<TableBuilder>,- in_table_cell: bool,- uc_skip: usize,- pending_uc_skip: usize,-}--fn parse_body(src: &[u8], encoding: &'static Encoding) -> Vec<Block> {- let mut p = BodyParser {- state: CharState::default(),- stack: Vec::new(),- blocks: Vec::new(),- inlines: Vec::new(),- text: TextAccum::new(encoding),- table: None,- in_table_cell: false,- uc_skip: 1,- pending_uc_skip: 0,- };-- for token in Tokens::new(src) {- p.handle_token(token);- }-- if !p.text.is_empty() || !p.inlines.is_empty() {- p.flush_paragraph();- }- p.flush_table();- p.blocks-}--impl BodyParser {- fn skipping(&self) -> bool {- self.stack.last().map(|g| g.skip).unwrap_or(false)- }-- fn handle_token(&mut self, token: Token) {- if self.pending_uc_skip > 0 {- match token {- Token::Byte(_) | Token::Hex(_) => {- self.pending_uc_skip -= 1;- return;- }- _ => self.pending_uc_skip = 0,- }- }-- match token {- Token::Open => {- let inherited_skip = self.skipping();- self.stack.push(Group {- saved: self.state.clone(),- skip: inherited_skip,- name_seen: false,- });- }- Token::Close => {- if !self.skipping() {- self.flush_text();- }- if let Some(group) = self.stack.pop() {- self.state = group.saved;- }- }- Token::Byte(b) | Token::Hex(b) => {- if !self.skipping() {- self.text.push_byte(b);- }- }- Token::Symbol(b'*') => {- if let Some(group) = self.stack.last_mut() {- if !group.name_seen {- group.name_seen = true;- group.skip = true;- }- }- }- Token::Symbol(b @ (b'\\' | b'{' | b'}')) => {- if !self.skipping() {- self.text.push_byte(b);- }- }- Token::Symbol(b'~') => {- if !self.skipping() {- self.text.push_char('\u{00A0}');- }- }- Token::Symbol(b'-') => {- if !self.skipping() {- self.text.push_char('\u{00AD}');- }- }- Token::Symbol(_) => {}- Token::Control(word, value) => self.handle_control(word, value),- }- }-- fn handle_control(&mut self, word: &str, value: Option<i32>) {- if let Some(group) = self.stack.last_mut() {- if !group.name_seen {- group.name_seen = true;- if SKIP_DESTS.contains(&word) {- group.skip = true;- }- }- }-- if self.skipping() {- return;- }- if word == "par" {- self.flush_paragraph();- return;- }-- let on = |value: Option<i32>| value.map(|v| v != 0).unwrap_or(true);- match word {- "rquote" => self.text.push_char('\u{2019}'),- "lquote" => self.text.push_char('\u{2018}'),- "rdblquote" => self.text.push_char('\u{201D}'),- "ldblquote" => self.text.push_char('\u{201C}'),- "emdash" => self.text.push_char('\u{2014}'),- "endash" => self.text.push_char('\u{2013}'),- "bullet" => self.text.push_char('\u{2022}'),- "tab" => self.text.push_char('\t'),- "line" => {- self.flush_text();- self.inlines.push(Inline::LineBreak);- }- "u" => {- if let Some(mut code) = value {- if code < 0 {- code += 65536;- }- if let Some(ch) = char::from_u32(code as u32) {- self.text.push_char(ch);- }- self.pending_uc_skip = self.uc_skip;- }- }- "uc" => self.uc_skip = uc_count(value),- "b" => {- self.flush_text();- self.state.bold = on(value);- }- "i" => {- self.flush_text();- self.state.italic = on(value);- }- "strike" | "striked" | "striked1" => {- self.flush_text();- self.state.strike = on(value);- }- "super" => {- self.flush_text();- self.state.sup = on(value);- if self.state.sup {- self.state.sub = false;- }- }- "sub" => {- self.flush_text();- self.state.sub = on(value);- if self.state.sub {- self.state.sup = false;- }- }- "nosupersub" => {- self.flush_text();- self.state.sup = false;- self.state.sub = false;- }- "plain" => {- self.flush_text();- self.state = CharState::default();- }- "trowd" => {- self- .table- .get_or_insert_with(TableBuilder::default)- .start_row();- self.in_table_cell = false;- }- "intbl" => self.in_table_cell = true,- "cell" => {- self.in_table_cell = true;- self.flush_paragraph();- if let Some(table) = self.table.as_mut() {- table.finish_cell();- }- self.in_table_cell = false;- }- "row" => {- if let Some(table) = self.table.as_mut() {- table.finish_row();- }- self.in_table_cell = false;- }- _ => {}- }- }-- fn flush_text(&mut self) {- let text = self.text.take();- if text.is_empty() {- return;- }- let mut node = Inline::Text(text);- if self.state.strike {- node = Inline::Del(vec![node]);- }- if self.state.italic {- node = Inline::Em(vec![node]);- }- if self.state.bold {- node = Inline::Strong(vec![node]);- }- if self.state.sup {- node = Inline::Sup(vec![node]);- } else if self.state.sub {- node = Inline::Sub(vec![node]);- }- self.inlines.push(node);- }-- fn flush_paragraph(&mut self) {- self.flush_text();- if has_visible_content(&self.inlines) {- let block = Block::Paragraph(Paragraph {- kind: ParagraphKind::Normal,- inlines: std::mem::take(&mut self.inlines),- });- if self.in_table_cell {- if let Some(table) = self.table.as_mut() {- table.push_block(block);- } else {- self.blocks.push(block);- }- } else {- self.flush_table();- self.blocks.push(block);- }- } else {- self.inlines.clear();- if !self.in_table_cell {- self.flush_table();- }- }- }-- fn flush_table(&mut self) {- if let Some(table) = self.table.take() {- if let Some(block) = table.finalize() {- self.blocks.push(block);- }- }- }-}--fn extract_metadata(src: &[u8], encoding: &'static Encoding) -> DocumentMetadata {- let mut metadata = DocumentMetadata::default();- let Some(info) = find_group(src, b"{\\info") else {- return metadata;- };-- if let Some(author) = find_group(info, b"{\\author").and_then(|g| group_text(g, encoding)) {- if !author.eq_ignore_ascii_case("unknown") {- metadata.author = Some(author);- }- }- if let Some(title) = find_group(info, b"{\\title").and_then(|g| group_text(g, encoding)) {- metadata.title = Some(title);- }- metadata.created = find_group(info, b"{\\creatim").and_then(parse_timestamp);-- metadata-}--fn find_group<'a>(buf: &'a [u8], start_tag: &[u8]) -> Option<&'a [u8]> {- let start = buf.windows(start_tag.len()).position(|w| w == start_tag)?;- let mut depth = 0usize;- for (i, &b) in buf[start..].iter().enumerate() {- match b {- b'{' => depth += 1,- b'}' => {- depth -= 1;- if depth == 0 {- return Some(&buf[start..start + i + 1]);- }- }- _ => {}- }- }- None-}--/// Plain text of a destination group, flattening nested formatting groups-/// and ignoring \*-prefixed destinations.-fn group_text(group: &[u8], encoding: &'static Encoding) -> Option<String> {- let mut accum = TextAccum::new(encoding);- let mut depth = 0usize;- let mut ignored_depth: Option<usize> = None;- let mut just_opened = false;- let mut pending_uc_skip = 0usize;- let mut uc_skip = 1usize;-- for token in Tokens::new(group) {- if just_opened {- just_opened = false;- if matches!(token, Token::Symbol(b'*')) && ignored_depth.is_none() {- ignored_depth = Some(depth);- continue;- }- }- match token {- Token::Open => {- depth += 1;- just_opened = true;- continue;- }- Token::Close => {- depth = depth.saturating_sub(1);- if ignored_depth.is_some_and(|d| depth < d) {- ignored_depth = None;- }- continue;- }- _ => {}- }- if ignored_depth.is_some() {- continue;- }- if pending_uc_skip > 0 {- match token {- Token::Byte(_) | Token::Hex(_) => {- pending_uc_skip -= 1;- continue;- }- _ => pending_uc_skip = 0,- }- }- match token {- Token::Byte(b) | Token::Hex(b) => accum.push_byte(b),- Token::Symbol(b @ (b'\\' | b'{' | b'}')) => accum.push_byte(b),- Token::Control("u", Some(mut code)) => {- if code < 0 {- code += 65536;- }- if let Some(ch) = char::from_u32(code as u32) {- accum.push_char(ch);- }- pending_uc_skip = uc_skip;- }- Token::Control("uc", value) => uc_skip = uc_count(value),- _ => {}- }- }-- let text = accum.take().trim().to_string();- if text.is_empty() {- None- } else {- Some(text)- }-}--fn parse_timestamp(group: &[u8]) -> Option<DateTime<Utc>> {- let mut yr = None;- let mut mo = None;- let mut dy = None;- let mut hr = None;- let mut mi = None;-- for token in Tokens::new(group) {- if let Token::Control(word, value) = token {- match word {- "yr" => yr = value,- "mo" => mo = value.map(|v| v as u32),- "dy" => dy = value.map(|v| v as u32),- "hr" => hr = value.map(|v| v as u32),- "min" => mi = value.map(|v| v as u32),- _ => {}- }- }- }-- let date = NaiveDate::from_ymd_opt(yr?, mo?, dy?)?;- let time = chrono::NaiveTime::from_hms_opt(hr.unwrap_or(0), mi.unwrap_or(0), 0)?;- Some(DateTime::<Utc>::from_naive_utc_and_offset(- NaiveDateTime::new(date, time),- Utc,- ))-}--#[cfg(test)]-mod tests {- use super::*;- use crate::document::providers::DocumentProvider;-- fn paragraphs(data: &[u8]) -> Vec<String> {- let document = RtfProvider::new().parse_buffer(data).unwrap();- document- .blocks- .iter()- .filter_map(|block| match block {- Block::Paragraph(p) => Some(inline_text(&p.inlines)),- _ => None,- })- .collect()- }-- fn inline_text(inlines: &[Inline]) -> String {- inlines- .iter()- .map(|inline| match inline {- Inline::Text(t) => t.clone(),- Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {- inline_text(c)- }- _ => String::new(),- })- .collect()- }-- #[test]- fn ansicpg1251_hex_escapes() {- let rtf = b"{\\rtf1\\ansi\\ansicpg1251 \\'d3\\'ea\\'f0\\'e0\\'bf\\'ed\\'e0\\par}";- assert_eq!(paragraphs(rtf), vec!["Україна"]);- }-- #[test]- fn default_cp1252_hex_escapes() {- let rtf = b"{\\rtf1\\ansi Caf\\'e9 \\'93ok\\'94\\par}";- assert_eq!(paragraphs(rtf), vec!["Café \u{201C}ok\u{201D}"]);- }-- #[test]- fn unicode_escape_with_fallback() {- let rtf = b"{\\rtf1\\ansi\\uc1 \\u1059?\\u1082?\\u1088? ok\\par}";- assert_eq!(paragraphs(rtf), vec!["Укр ok"]);- }-- #[test]- fn shift_jis_double_byte_pairs() {- let rtf = b"{\\rtf1\\ansi\\ansicpg932 \\'93\\'fa\\'96\\'7b\\par}";- assert_eq!(paragraphs(rtf), vec!["日本"]);- }-- #[test]- fn styles_and_tables_still_parse() {- let rtf =- b"{\\rtf1\\ansi {\\b Bold} and {\\i italic}\\par\\trowd\\intbl A\\cell B\\cell\\row\\par}";- let document = RtfProvider::new().parse_buffer(rtf).unwrap();- assert!(matches!(document.blocks[0], Block::Paragraph(_)));- assert!(document- .blocks- .iter()- .any(|b| matches!(b, Block::Table(t) if t.rows[0].cells.len() == 2)));- }-- #[test]- fn info_metadata_decodes_with_document_codepage() {- let rtf = b"{\\rtf1\\ansi\\ansicpg1251{\\info{\\title \\'d3\\'ea\\'e0\\'e7}{\\author Iryna}{\\creatim\\yr2022\\mo6\\dy9\\hr15\\min24}}Body\\par}";- let document = RtfProvider::new().parse_buffer(rtf).unwrap();- assert_eq!(document.metadata.title.as_deref(), Some("Указ"));- assert_eq!(document.metadata.author.as_deref(), Some("Iryna"));- assert_eq!(- document.metadata.created.unwrap().to_rfc3339(),- "2022-06-09T15:24:00+00:00"- );- }-- #[test]- fn metadata_keeps_text_of_nested_formatting_groups() {- let rtf = b"{\\rtf1{\\info{\\title Some {\\b Bold} Title{\\*\\junk secret}}}Body\\par}";- let document = RtfProvider::new().parse_buffer(rtf).unwrap();- assert_eq!(document.metadata.title.as_deref(), Some("Some Bold Title"));- }-- #[test]- fn par_inside_ignored_destination_does_not_split_paragraphs() {- let rtf = b"{\\rtf1\\ansi Hello{\\*\\junkdest gone\\par gone too} world\\par}";- assert_eq!(paragraphs(rtf), vec!["Hello world"]);- }-- #[test]- fn malformed_hex_escape_does_not_truncate_document() {- let rtf = b"{\\rtf1\\ansi before \\'zz after\\par}";- assert_eq!(paragraphs(rtf), vec!["before zz after"]);- }-- #[test]- fn escaped_ansicpg_literal_does_not_change_encoding() {- let rtf = b"{\\rtf1\\ansi text \\\\ansicpg1251 \\'e9\\par}";- assert_eq!(paragraphs(rtf), vec!["text \\ansicpg1251 é"]);- }-- #[test]- fn overflowing_control_parameter_digits_do_not_leak() {- let rtf = b"{\\rtf1\\ansi a\\u99999999999999x b\\par}";- assert_eq!(paragraphs(rtf), vec!["ax b"]);- }-- #[test]- fn uc_above_eight_skips_declared_fallback_bytes() {- let rtf = b"{\\rtf1\\ansi\\uc10\\u1059 abcdefghijkl\\par}";- assert_eq!(paragraphs(rtf), vec!["Уkl"]);- }-- #[test]- fn huge_uc_value_stays_bounded_by_input() {- let rtf = b"{\\rtf1\\ansi\\uc2000000000\\u1059 abc\\par done\\par}";- assert_eq!(paragraphs(rtf), vec!["У", "done"]);- }-}diff --git a/apps/api/native/src/document/providers/xlsx.rs b/apps/api/native/src/document/providers/xlsx.rsdeleted file mode 100644index 896cb1ae1..000000000--- a/apps/api/native/src/document/providers/xlsx.rs+++ /dev/null@@ -1,87 +0,0 @@-use crate::document::model::*;-use crate::document::providers::DocumentProvider;-use calamine::{open_workbook_auto_from_rs, Data, Reader};-use std::error::Error;-use std::io::Cursor;-use std::num::NonZeroU32;--const ONE: NonZeroU32 = unsafe { NonZeroU32::new_unchecked(1) };--pub struct XlsxProvider;--impl XlsxProvider {- pub fn new() -> Self {- Self- }-}--impl DocumentProvider for XlsxProvider {- fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {- let cursor = Cursor::new(data);- let mut workbook = open_workbook_auto_from_rs(cursor)?;-- let mut blocks: Vec<Block> = Vec::new();-- for sheet_name in workbook.sheet_names() {- // Add sheet heading- blocks.push(Block::Paragraph(Paragraph {- kind: ParagraphKind::Heading(2),- inlines: vec![Inline::Text(sheet_name.clone())],- }));-- if let Ok(range) = workbook.worksheet_range(&sheet_name) {- let mut rows: Vec<TableRow> = Vec::new();- for r in range.rows() {- let mut cells: Vec<TableCell> = Vec::new();- for cell in r {- let text = data_type_to_string(cell);- let blocks_in_cell = if text.trim().is_empty() {- Vec::new()- } else {- vec![Block::Paragraph(Paragraph {- kind: ParagraphKind::Normal,- inlines: vec![Inline::Text(text)],- })]- };- cells.push(TableCell {- blocks: blocks_in_cell,- colspan: ONE,- rowspan: ONE,- });- }- rows.push(TableRow {- cells,- kind: TableRowKind::Body,- });- }-- blocks.push(Block::Table(Table { rows }));- }- }-- Ok(Document {- blocks,- metadata: DocumentMetadata::default(),- notes: Vec::new(),- comments: Vec::new(),- })- }-- fn name(&self) -> &'static str {- "xlsx"- }-}--fn data_type_to_string(cell: &Data) -> String {- match cell {- Data::Empty => String::new(),- Data::String(s) => s.clone(),- Data::Float(f) => f.to_string(),- Data::Int(i) => i.to_string(),- Data::Bool(b) => b.to_string(),- Data::DateTime(v) => v.to_string(),- Data::DateTimeIso(v) => v.to_string(),- Data::DurationIso(v) => v.to_string(),- Data::Error(e) => format!("#ERROR({e:?})"),- }-}diff --git a/apps/api/native/src/document/renderers/html.rs b/apps/api/native/src/document/renderers/html.rsdeleted file mode 100644index ea6582400..000000000--- a/apps/api/native/src/document/renderers/html.rs+++ /dev/null@@ -1,237 +0,0 @@-use crate::document::model::*;-use maud::{html, Markup, DOCTYPE};--pub struct HtmlRenderer;--impl HtmlRenderer {- pub fn new() -> Self {- Self- }-- pub fn render(&self, document: &Document) -> String {- let title = document.metadata.title.as_deref().unwrap_or("Document");-- let footnotes: Vec<&Note> = document- .notes- .iter()- .filter(|n| matches!(n.kind, NoteKind::Footnote))- .collect();-- let endnotes: Vec<&Note> = document- .notes- .iter()- .filter(|n| matches!(n.kind, NoteKind::Endnote))- .collect();-- let author = document.metadata.author.as_deref();- let page: Markup = html! {- (DOCTYPE)- html lang="en" {- head {- meta charset="UTF-8";- meta name="viewport" content="width=device-width, initial-scale=1.0";- title { (title) }- @if let Some(author) = author {- meta name="author" content=(author);- }- }- body {- main { (self.render_blocks(&document.blocks)) }-- @if !footnotes.is_empty() {- section id="footnotes" {- h2 { "Footnotes" }- @for footnote in &footnotes {- div id={ "footnote-" (&footnote.id.0) } {- (self.render_blocks(&footnote.blocks))- }- }- }- }-- @if !endnotes.is_empty() {- section id="endnotes" {- h2 { "Endnotes" }- @for endnote in &endnotes {- div id={ "endnote-" (&endnote.id.0) } {- (self.render_blocks(&endnote.blocks))- }- }- }- }-- @if !document.comments.is_empty() {- section id="comments" {- h2 { "Comments" }- @for comment in &document.comments {- article id={ "comment-" (&comment.id.0) } {- @if let Some(author) = &comment.author_name {- header {- (author)- @if let Some(initials) = &comment.author_initials {- " (" (initials) ")"- }- }- }- (self.render_blocks(&comment.blocks))- }- }- }- }- }- }- };-- page.into_string()- }-- fn render_blocks(&self, blocks: &[Block]) -> Markup {- html! {- @for b in blocks {- @match b {- Block::Paragraph(p) => { (self.render_paragraph(p)) }- Block::Table(t) => { (self.render_table(t)) }- Block::List(l) => { (self.render_list(l)) }- Block::Image(i) => { (self.render_image(i)) }- }- }- }- }-- fn render_blocks_inline(&self, blocks: &[Block]) -> Markup {- if blocks.len() == 1 {- if let Block::Paragraph(p) = &blocks[0] {- if matches!(p.kind, ParagraphKind::Normal) {- return self.render_inlines(&p.inlines);- }- }- }-- self.render_blocks(blocks)- }-- fn render_paragraph(&self, p: &Paragraph) -> Markup {- match p.kind {- ParagraphKind::Normal => html! { p { (self.render_inlines(&p.inlines)) } },- ParagraphKind::Blockquote => html! {- blockquote { p { (self.render_inlines(&p.inlines)) } }- },- ParagraphKind::Heading(level) => match level {- 1 => html! { h1 { (self.render_inlines(&p.inlines)) } },- 2 => html! { h2 { (self.render_inlines(&p.inlines)) } },- 3 => html! { h3 { (self.render_inlines(&p.inlines)) } },- 4 => html! { h4 { (self.render_inlines(&p.inlines)) } },- 5 => html! { h5 { (self.render_inlines(&p.inlines)) } },- _ => html! { h6 { (self.render_inlines(&p.inlines)) } },- },- }- }-- fn render_table(&self, t: &Table) -> Markup {- let mut head_rows = Vec::new();- let mut body_rows = Vec::new();- let mut foot_rows = Vec::new();-- for row in &t.rows {- match row.kind {- TableRowKind::Header => head_rows.push(row),- TableRowKind::Body => body_rows.push(row),- TableRowKind::Footer => foot_rows.push(row),- }- }-- html! {- table {- @if !head_rows.is_empty() {- thead { @for row in head_rows { (self.render_table_row(row, true)) } }- }- tbody { @for row in body_rows { (self.render_table_row(row, false)) } }- @if !foot_rows.is_empty() {- tfoot { @for row in foot_rows { (self.render_table_row(row, false)) } }- }- }- }- }-- fn render_table_row(&self, row: &TableRow, header: bool) -> Markup {- html! {- tr {- @for cell in &row.cells {- @let cs = cell.colspan.get();- @let rs = cell.rowspan.get();- @let cs_attr = if cs > 1 { Some(cs) } else { None };- @let rs_attr = if rs > 1 { Some(rs) } else { None };-- @if header {- @if let (Some(cs), Some(rs)) = (cs_attr, rs_attr) {- th colspan=(cs) rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }- } @else if let Some(cs) = cs_attr {- th colspan=(cs) { (self.render_blocks_inline(&cell.blocks)) }- } @else if let Some(rs) = rs_attr {- th rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }- } @else {- th { (self.render_blocks_inline(&cell.blocks)) }- }- } @else {- @if let (Some(cs), Some(rs)) = (cs_attr, rs_attr) {- td colspan=(cs) rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }- } @else if let Some(cs) = cs_attr {- td colspan=(cs) { (self.render_blocks_inline(&cell.blocks)) }- } @else if let Some(rs) = rs_attr {- td rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }- } @else {- td { (self.render_blocks_inline(&cell.blocks)) }- }- }- }- }- }- }-- fn render_list(&self, l: &List) -> Markup {- match l.list_type {- ListType::Ordered => html! {- ol { @for item in &l.items { li { (self.render_blocks_inline(&item.blocks)) } } }- },- ListType::Unordered => html! {- ul { @for item in &l.items { li { (self.render_blocks_inline(&item.blocks)) } } }- },- }- }-- fn render_image(&self, i: &Image) -> Markup {- match &i.alt {- Some(alt) => html! { img src=(i.src) alt=(alt); },- None => html! { img src=(i.src); },- }- }-- fn render_inlines(&self, inlines: &[Inline]) -> Markup {- html! { @for inline in inlines { (self.render_inline(inline)) } }- }-- fn render_inline(&self, inline: &Inline) -> Markup {- match inline {- Inline::Text(t) => html! { (t) },- Inline::LineBreak => html! { br; },-- Inline::Link { href, children } => {- html! { a href=(href) { (self.render_inlines(children)) } }- }-- Inline::Strong(children) => html! { strong { (self.render_inlines(children)) } },- Inline::Em(children) => html! { em { (self.render_inlines(children)) } },- Inline::Del(children) => html! { del { (self.render_inlines(children)) } },- Inline::Code(code) => html! { code { (code) } },- Inline::Sup(children) => html! { sup { (self.render_inlines(children)) } },- Inline::Sub(children) => html! { sub { (self.render_inlines(children)) } },-- Inline::FootnoteRef(id) => {- html! { sup { a href={ "#footnote-" (&id.0) } { (&id.0) } } }- }- Inline::EndnoteRef(id) => html! { sup { a href={ "#endnote-" (&id.0) } { (&id.0) } } },- Inline::CommentRef(id) => html! { a href={ "#comment-" (&id.0) } { "💬" } },- Inline::Bookmark(id) => html! { a id=(&id.0) {} },- }- }-}diff --git a/apps/api/native/src/document/renderers/mod.rs b/apps/api/native/src/document/renderers/mod.rsdeleted file mode 100644index 3d4613dd4..000000000--- a/apps/api/native/src/document/renderers/mod.rs+++ /dev/null@@ -1 +0,0 @@-pub mod html;diff --git a/apps/api/native/src/document/xml.rs b/apps/api/native/src/document/xml.rsdeleted file mode 100644index 45ce1cec7..000000000--- a/apps/api/native/src/document/xml.rs+++ /dev/null@@ -1,49 +0,0 @@-//! Helpers shared by the zip+XML based providers (docx, odt).--use roxmltree::Node;-use std::io::{Read, Seek};-use zip::read::ZipArchive;--pub fn read_zip_text<R: Read + Seek>(zip: &mut ZipArchive<R>, path: &str) -> Option<String> {- let mut file = zip.by_name(path).ok()?;- let mut s = String::new();- file.read_to_string(&mut s).ok()?;- Some(s)-}--pub fn strip_bom(s: &str) -> &str {- s.strip_prefix('\u{FEFF}').unwrap_or(s)-}--pub fn is_tag(node: &Node, local: &str) -> bool {- node.is_element() && node.tag_name().name() == local-}--/// Attribute value matched by local name, ignoring the namespace prefix.-pub fn attr<'a>(node: &Node<'a, 'a>, local: &str) -> Option<&'a str> {- node- .attributes()- .find(|a| {- let name = a.name();- match name.rsplit_once(':') {- Some((_, l)) => l == local,- None => name == local,- }- })- .map(|a| a.value())-}--pub fn child<'a>(node: &Node<'a, 'a>, local: &str) -> Option<Node<'a, 'a>> {- node- .children()- .find(|n| n.is_element() && n.tag_name().name() == local)-}--pub fn children<'a, 'b>(- node: &Node<'a, 'a>,- local: &'b str,-) -> impl Iterator<Item = Node<'a, 'a>> + use<'a, 'b> {- node- .children()- .filter(move |n| n.is_element() && n.tag_name().name() == local)-}diff --git a/apps/api/native/src/lib.rs b/apps/api/native/src/lib.rsindex f440be0cf..9735f28c8 100644--- a/apps/api/native/src/lib.rs+++ b/apps/api/native/src/lib.rs@@ -1,14 +1,13 @@#![deny(clippy::all)]pub use crate::crawler::*;+pub use crate::document::*;pub use crate::engpicker::*;pub use crate::html::*;pub use crate::logging::*;pub use crate::pdf::*;pub use crate::utils::*;-pub use crate::document::{DocumentConverter, DocumentType};-mod crawler;mod document;mod engpicker;diff --git a/apps/api/src/__tests__/snips/v2/document-converter.test.ts b/apps/api/src/__tests__/snips/v2/document-converter.test.tsindex 3ed176fec..1bab83848 100644--- a/apps/api/src/__tests__/snips/v2/document-converter.test.ts+++ b/apps/api/src/__tests__/snips/v2/document-converter.test.ts@@ -1,60 +1,69 @@import path from "path";import fs from "fs";-import { DocumentConverter, DocumentType } from "@mendable/firecrawl-rs";+import { convertDocumentToMarkdown } from "@mendable/firecrawl-rs";describe("Document Converter tests", () => {const samplesDir = path.join(process.cwd(), "samples");- const expectedHtmlBase = (documentText: string) =>- `<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Document</title></head><body><main><p><strong>Hello!</strong></p><p>${documentText} file to test the Firecrawl Document Converter.</p><p><em>Italic</em></p><p><strong>Bold</strong></p><p>Underlined</p><p><del>Strikethrough</del></p><table><tbody><tr><td>Header 1</td><td>Header 2</td></tr><tr><td>Value 1</td><td>Value 2</td></tr></tbody></table></main></body></html>`;+ const expectedMarkdownBase = (documentText: string) =>+ `**Hello!**\n\n${documentText} file to test the Firecrawl Document Converter.\n\n*Italic*\n\n**Bold**\n\nUnderlined\n\n~~Strikethrough~~\n\n| | |\n| --- | --- |\n| Header 1 | Header 2 |\n| Value 1 | Value 2 |\n`;const sampleFiles = [- { file: "sample.docx", type: DocumentType.Docx, name: "DOCX" },- { file: "sample.odt", type: DocumentType.Odt, name: "ODT" },- { file: "sample.rtf", type: DocumentType.Rtf, name: "RTF" },+ { file: "sample.docx", name: "DOCX" },+ { file: "sample.odt", name: "ODT" },+ { file: "sample.rtf", name: "RTF" },];- const converter = new DocumentConverter();+ describe.each(sampleFiles)("$name document conversion", ({ file, name }) => {+ const filePath = path.join(samplesDir, file);- describe.each(sampleFiles)(- "$name document conversion",- ({ file, type, name }) => {- const filePath = path.join(samplesDir, file);-- beforeAll(() => {- if (!fs.existsSync(filePath)) {- throw new Error(`Sample file ${filePath} does not exist`);- }- });+ beforeAll(() => {+ if (!fs.existsSync(filePath)) {+ throw new Error(`Sample file ${filePath} does not exist`);+ }+ });- it.concurrent(- `should convert ${name} document and return expected HTML`,- async () => {- const fileBuffer = fs.readFileSync(filePath);+ it.concurrent(+ `should convert ${name} document and return expected markdown`,+ async () => {+ const fileBuffer = fs.readFileSync(filePath);- const html = await converter.convertBufferToHtml(- new Uint8Array(fileBuffer),- type,- );+ // No extension hint: the format must be detected from the bytes+ const markdown = convertDocumentToMarkdown(new Uint8Array(fileBuffer));- expect(html).toBe(expectedHtmlBase(name));- },- 10000,- );- },- );+ expect(markdown).toBe(expectedMarkdownBase(name));+ },+ 10000,+ );+ });describe("XLSX document conversion", () => {- const expectedHtml = `<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Document</title></head><body><main><h2>Sheet1</h2><table><tbody><tr><td>sample file</td><td>test</td></tr><tr><td></td><td></td></tr><tr><td>Name</td><td>Price</td></tr><tr><td>iPhone</td><td>1000</td></tr><tr><td>iPad</td><td>800</td></tr><tr><td>Macbook</td><td>1200</td></tr></tbody></table><h2>Sheet2</h2><table><tbody><tr><td>other tab</td><td></td></tr><tr><td></td><td></td></tr><tr><td>Name</td><td>Price</td></tr><tr><td>ChatGPT</td><td>20</td></tr><tr><td>Claude</td><td>17</td></tr><tr><td>Perplexity</td><td>20</td></tr></tbody></table></main></body></html>`;+ const expectedMarkdown = `## Sheet1\n\n| | |\n| --- | --- |\n| sample file | test |\n| | |\n| Name | Price |\n| iPhone | 1000 |\n| iPad | 800 |\n| Macbook | 1200 |\n\n## Sheet2\n\n| | |\n| --- | --- |\n| other tab | |\n| | |\n| Name | Price |\n| ChatGPT | 20 |\n| Claude | 17 |\n| Perplexity | 20 |\n`;- it("should convert XLSX document and return expected HTML", async () => {+ it("should convert XLSX document and return expected markdown", () => {const filePath = path.join(samplesDir, "sample.xlsx");const fileBuffer = fs.readFileSync(filePath);- const html = await converter.convertBufferToHtml(- new Uint8Array(fileBuffer),- DocumentType.Xlsx,+ const markdown = convertDocumentToMarkdown(new Uint8Array(fileBuffer));+ expect(markdown).toBe(expectedMarkdown);+ });+ });++ describe("CSV document conversion", () => {+ it("should convert CSV to a markdown table using the extension hint", () => {+ // CSV has no magic bytes, so the extension hint selects the parser+ const csv = Buffer.from("Name,Price\niPhone,1000\niPad,800\n");+ const markdown = convertDocumentToMarkdown(new Uint8Array(csv), ".csv");+ expect(markdown).toBe(+ "| | |\n| --- | --- |\n| Name | Price |\n| iPhone | 1000 |\n| iPad | 800 |\n",);- expect(html).toBe(expectedHtml);+ });+ });++ describe("unrecognized content", () => {+ it("should throw on content that is not a supported document", () => {+ expect(() =>+ convertDocumentToMarkdown(new Uint8Array([1, 2, 3, 4, 5])),+ ).toThrow();});});});diff --git a/apps/api/src/controllers/v2/parse-upload.ts b/apps/api/src/controllers/v2/parse-upload.tsindex 9eaac0027..32a4d9e0d 100644--- a/apps/api/src/controllers/v2/parse-upload.ts+++ b/apps/api/src/controllers/v2/parse-upload.ts@@ -9,7 +9,7 @@ import { logger as _logger } from "../../lib/logger";import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";import { getRedisConnection } from "../../services/queue-service";import { RequestWithAuth, UploadedParseFile } from "./types";-import { detectUploadedFileKind } from "./parse";+import { detectUploadedFileKind, SUPPORTED_PARSE_FILE_TYPES } from "./parse";const PARSE_UPLOAD_MAX_BYTES = 50 * 1024 * 1024;const PARSE_UPLOAD_TTL_MS = 10 * 60 * 1000;@@ -273,8 +273,7 @@ export async function parseUploadUrlController(return res.status(400).json({success: false,code: "UNSUPPORTED_FILE_TYPE",- error:- "Unsupported upload type. Supported file extensions include .html, .htm, .xhtml, .pdf, .docx, .doc, .odt, .rtf, .xlsx, .xls, or matching supported MIME types.",+ error: `Unsupported upload type. Supported file extensions include ${SUPPORTED_PARSE_FILE_TYPES}, or matching supported MIME types.`,});}diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex eae34cff1..3ccc60735 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -36,19 +36,14 @@ import { projectScrapeCredits } from "../../lib/keyless-credit-projection";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";import path from "node:path";+import {+ DOCUMENT_EXTENSIONS,+ documentExtensionFromContentType,+} from "../../lib/document-formats";const AGENT_INTEROP_CONCURRENCY_BOOST = 3;-const SUPPORTED_PARSE_FILE_TYPES =- ".html, .htm, .pdf, .docx, .doc, .odt, .rtf, .xlsx, .xls";--const DOCUMENT_EXTENSIONS = new Set([- ".docx",- ".doc",- ".odt",- ".rtf",- ".xlsx",- ".xls",-]);+export const SUPPORTED_PARSE_FILE_TYPES =+ ".html, .htm, .xhtml, .pdf, .docx, .doc, .docm, .odt, .ods, .odp, .rtf, .xlsx, .xls, .xlsm, .xlsb, .pptx, .ppt, .pptm, .epub, .csv";export function detectUploadedFileKind(filename: string,@@ -68,17 +63,7 @@ export function detectUploadedFileKind(const isDocument =DOCUMENT_EXTENSIONS.has(extension) ||- normalizedType.includes(- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",- ) ||- normalizedType.includes("application/vnd.ms-excel") ||- normalizedType.includes(- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",- ) ||- normalizedType.includes("application/msword") ||- normalizedType.includes("application/vnd.oasis.opendocument.text") ||- normalizedType.includes("application/rtf") ||- normalizedType.includes("text/rtf");+ documentExtensionFromContentType(normalizedType) !== null;if (isDocument) {return "document";@@ -378,9 +363,9 @@ export async function parseController();if (!reservation.ok) {applyAgentAuthDiscoveryHeader(res);- return res.status(429).json(- await keylessLimitBody(req.auth.team_id, "v2_parse"),- );+ return res+ .status(429)+ .json(await keylessLimitBody(req.auth.team_id, "v2_parse"));}reservedKeylessCredits = projectedKeylessCredits;}diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex d9ef0bbdc..6b826f2dc 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -95,7 +95,7 @@ const booleanFlag = zvalue === undefined ? undefined : value === true || value === "true",);-export const developerSearchSchema = z.strictObject({+const developerSearchSchema = z.strictObject({query: z.string().min(1),k: kSchema(100),types: multiString,@@ -113,7 +113,7 @@ export const developerSearchSchema = z.strictObject({...commonQuery,});-export const DEVELOPER_SEARCH_QUERY_KEYS = [+const DEVELOPER_SEARCH_QUERY_KEYS = ["query","k","types",diff --git a/apps/api/src/lib/document-formats.ts b/apps/api/src/lib/document-formats.tsnew file mode 100644index 000000000..bce80f84a--- /dev/null+++ b/apps/api/src/lib/document-formats.ts@@ -0,0 +1,65 @@+// Document formats handled by the document engine (anydoc). PDF is+// intentionally absent — it has its own engine with OCR support.++const CONTENT_TYPE_TO_EXTENSION = new Map<string, string>([+ [+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ ".docx",+ ],+ ["application/vnd.ms-word.document.macroenabled.12", ".docm"],+ ["application/msword", ".doc"],+ [+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",+ ".xlsx",+ ],+ ["application/vnd.ms-excel", ".xls"],+ ["application/vnd.ms-excel.sheet.macroenabled.12", ".xlsm"],+ ["application/vnd.ms-excel.sheet.binary.macroenabled.12", ".xlsb"],+ [+ "application/vnd.openxmlformats-officedocument.presentationml.presentation",+ ".pptx",+ ],+ ["application/vnd.ms-powerpoint", ".ppt"],+ ["application/vnd.ms-powerpoint.presentation.macroenabled.12", ".pptm"],+ ["application/vnd.oasis.opendocument.text", ".odt"],+ ["application/vnd.oasis.opendocument.spreadsheet", ".ods"],+ ["application/vnd.oasis.opendocument.presentation", ".odp"],+ ["application/rtf", ".rtf"],+ ["text/rtf", ".rtf"],+ ["application/epub+zip", ".epub"],+ ["text/csv", ".csv"],+]);++export const DOCUMENT_EXTENSIONS = new Set(CONTENT_TYPE_TO_EXTENSION.values());++// Longest first so ".docx" wins over ".doc" when matching URL paths.+const EXTENSIONS_BY_LENGTH = [...DOCUMENT_EXTENSIONS].sort(+ (a, b) => b.length - a.length,+);++export function documentExtensionFromContentType(+ contentType: string | null | undefined,+): string | null {+ if (!contentType) return null;+ const mediaType = contentType.split(";")[0].trim().toLowerCase();+ return CONTENT_TYPE_TO_EXTENSION.get(mediaType) ?? null;+}++export function documentContentTypeFromExtension(+ extension: string,+): string | null {+ const ext = extension.toLowerCase();+ for (const [contentType, mapped] of CONTENT_TYPE_TO_EXTENSION) {+ if (mapped === ext) return contentType;+ }+ return null;+}++// Matches extensions at the end of the path or mid-path (e.g. file.xlsx/hash).+export function documentExtensionFromUrlPath(urlPath: string): string | null {+ const lowerPath = urlPath.toLowerCase();+ for (const ext of EXTENSIONS_BY_LENGTH) {+ if (lowerPath.endsWith(ext) || lowerPath.includes(ext + "/")) return ext;+ }+ return null;+}diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex a7aeb0543..ef8281483 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -87,7 +87,7 @@ export function isKeylessIpEligible(ip: string): boolean {const requestsKey = (ip: string) => `keyless_requests:${ip}`;const creditsKey = (ip: string) => `keyless_credits:${ip}`;-export type KeylessQuotaReason = "requests" | "credits";+type KeylessQuotaReason = "requests" | "credits";type KeylessConsumeResult = {ok: boolean;@@ -286,11 +286,14 @@ return next* consumption). Used by the hosted MCP before a keyless tool call so an* ineligible caller receives structured recovery without an OAuth challenge.*/-export async function checkKeylessEligibility(- ip: string,-): Promise<{+export async function checkKeylessEligibility(ip: string): Promise<{eligible: boolean;- reason?: KeylessQuotaReason | "disabled" | "ineligible_ip" | "suspicious" | "error";+ reason?:+ | KeylessQuotaReason+ | "disabled"+ | "ineligible_ip"+ | "suspicious"+ | "error";retryAfterSeconds?: number;}> {if (!isKeylessConfigured()) return { eligible: false, reason: "disabled" };diff --git a/apps/api/src/lib/research-upstream.ts b/apps/api/src/lib/research-upstream.tsindex b24673e23..85a398735 100644--- a/apps/api/src/lib/research-upstream.ts+++ b/apps/api/src/lib/research-upstream.ts@@ -9,7 +9,7 @@ const dispatcher = new Agent({bodyTimeout: TIMEOUT_MS,});-export function appendQuery(+function appendQuery(url: URL,params: Record<string, unknown>,allowed: string[],diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.tsindex b3ac85a68..6bde2f0bf 100644--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts@@ -1,114 +1,17 @@import { Meta } from "../..";import { EngineScrapeResult } from "..";import { fetchFileToBuffer } from "../utils/downloadFile";-import { DocumentConverter, DocumentType } from "@mendable/firecrawl-rs";+import { convertDocumentToMarkdown } from "@mendable/firecrawl-rs";+import { safeMarkdownToHtml } from "../pdf/markdownToHtml";import type { Response } from "undici";+import { DocumentAntibotError, EngineUnsuccessfulError } from "../../error";import {- DocumentAntibotError,- DocumentPrefetchFailed,- EngineUnsuccessfulError,-} from "../../error";+ documentContentTypeFromExtension,+ documentExtensionFromContentType,+ documentExtensionFromUrlPath,+} from "../../../../lib/document-formats";import { readFile, unlink } from "node:fs/promises";-const converter = new DocumentConverter();--function getDocumentTypeFromUrl(url: string): DocumentType {- const urlLower = url.toLowerCase();-- // Check for extensions at the end or in the middle (e.g., file.xlsx/hash)- // Check .docx before .doc to avoid false matches- if (urlLower.endsWith(".docx") || urlLower.includes(".docx/"))- return DocumentType.Docx;- if (urlLower.endsWith(".doc") || urlLower.includes(".doc/"))- return DocumentType.Doc;- if (urlLower.endsWith(".odt") || urlLower.includes(".odt/"))- return DocumentType.Odt;- if (urlLower.endsWith(".rtf") || urlLower.includes(".rtf/"))- return DocumentType.Rtf;- if (- urlLower.endsWith(".xlsx") ||- urlLower.endsWith(".xls") ||- urlLower.includes(".xlsx/") ||- urlLower.includes(".xls/")- )- return DocumentType.Xlsx;-- return DocumentType.Docx; // hope for the best-}--function getDocumentTypeFromContentType(- contentType: string | null,-): DocumentType | null {- if (!contentType) return null;-- const ct = contentType.toLowerCase();-- // Check for modern .docx format first (Office Open XML)- if (- ct.includes(- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",- )- ) {- return DocumentType.Docx;- }-- // Legacy .doc format (OLE2/CFB binary format)- if (ct.includes("application/msword")) {- return DocumentType.Doc;- }-- if (ct.includes("application/vnd.oasis.opendocument.text")) {- return DocumentType.Odt;- }-- if (ct.includes("application/rtf") || ct.includes("text/rtf")) {- return DocumentType.Rtf;- }-- if (- ct.includes(- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",- ) ||- ct.includes("application/vnd.ms-excel")- ) {- return DocumentType.Xlsx;- }-- return null;-}--function getContentTypeFromDocumentType(documentType: DocumentType): string {- switch (documentType) {- case DocumentType.Docx:- return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";- case DocumentType.Doc:- return "application/msword";- case DocumentType.Odt:- return "application/vnd.oasis.opendocument.text";- case DocumentType.Rtf:- return "application/rtf";- case DocumentType.Xlsx:- return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";- }-}--function isValidDocumentContentType(contentType: string | null): boolean {- if (!contentType) return false;-- const ct = contentType.toLowerCase();- const validTypes = [- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",- "application/vnd.ms-excel",- "application/msword",- "application/rtf",- "text/rtf",- "application/vnd.oasis.opendocument.text",- ];-- return validTypes.some(type => ct.includes(type));-}-export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {let response: Response;let buffer: Buffer;@@ -149,7 +52,7 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {// Validate content type only when fetching directly (not using prefetch)const ct = response.headers.get("Content-Type");- if (ct && !isValidDocumentContentType(ct)) {+ if (ct && documentExtensionFromContentType(ct) === null) {// The server declared a non-document type and nothing asked for a// document (no `document` flag means neither a document URL extension nor// a caller-requested format), so this engine is just the tail of the@@ -170,22 +73,25 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {}}- const documentType =- getDocumentTypeFromContentType(response.headers.get("content-type")) ??- getDocumentTypeFromUrl(response.url);+ const extension =+ documentExtensionFromContentType(response.headers.get("content-type")) ??+ documentExtensionFromUrlPath(new URL(response.url).pathname);- const html = await converter.convertBufferToHtml(+ const markdown = await convertDocumentToMarkdown(new Uint8Array(buffer),- documentType,+ extension ?? undefined,);+ const html = await safeMarkdownToHtml(markdown, meta.logger, meta.id);return {url: response.url,statusCode: response.status,html,+ markdown,contentType:response.headers.get("content-type") ??- getContentTypeFromDocumentType(documentType),+ (extension ? documentContentTypeFromExtension(extension) : null) ??+ "application/octet-stream",proxyUsed,};} finally {diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.tsindex 464218f12..749454c1e 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts@@ -5,6 +5,7 @@ import path from "path";import os from "os";import { writeFile } from "fs/promises";import { Meta } from "../..";+import { documentExtensionFromContentType } from "../../../../lib/document-formats";async function feResToFilePrefetch(logger: Logger,@@ -46,23 +47,8 @@ async function feResToDocumentPrefetch(contentType: string,): Promise<Meta["documentPrefetch"]> {// Determine file extension from content type- let extension = "tmp";- if (contentType.includes("wordprocessingml")) {- // Modern .docx format (Office Open XML)- extension = "docx";- } else if (contentType.includes("msword")) {- // Legacy .doc format (OLE2/CFB binary)- extension = "doc";- } else if (- contentType.includes("spreadsheetml") ||- contentType.includes("ms-excel")- ) {- extension = "xlsx";- } else if (contentType.includes("opendocument.text")) {- extension = "odt";- } else if (contentType.includes("rtf")) {- extension = "rtf";- }+ const extension =+ documentExtensionFromContentType(contentType)?.slice(1) ?? "tmp";return feResToFilePrefetch(logger, feRes, extension, "document", contentType);}@@ -83,17 +69,7 @@ export async function specialtyScrapeCheck(return;}- const documentTypes = [- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",- "application/vnd.ms-excel",- "application/msword",- "application/rtf",- "text/rtf",- "application/vnd.oasis.opendocument.text",- ];-- const isDocument = documentTypes.some(type => contentType.startsWith(type));+ const isDocument = documentExtensionFromContentType(contentType) !== null;const isPdf =contentType === "application/pdf" ||contentType.startsWith("application/pdf;");diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 029336c21..8d1a154e8 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -82,6 +82,12 @@ import {import { htmlTransform } from "./lib/removeUnwantedElements";import { postprocessors } from "./postprocessors";import { rewriteUrl } from "./lib/rewriteUrl";+import {+ DOCUMENT_EXTENSIONS,+ documentContentTypeFromExtension,+ documentExtensionFromContentType,+ documentExtensionFromUrlPath,+} from "../../lib/document-formats";import { writeFile } from "node:fs/promises";import path from "node:path";import { tmpdir } from "node:os";@@ -245,19 +251,7 @@ function buildFeatureFlags(const lowerPath = urlO.pathname.toLowerCase();// Check for document types first (they take precedence over PDF)- const isDocument =- lowerPath.endsWith(".docx") ||- lowerPath.endsWith(".odt") ||- lowerPath.endsWith(".rtf") ||- lowerPath.endsWith(".xlsx") ||- lowerPath.endsWith(".xls") ||- lowerPath.includes(".docx/") ||- lowerPath.includes(".odt/") ||- lowerPath.includes(".rtf/") ||- lowerPath.includes(".xlsx/") ||- lowerPath.includes(".xls/");-- if (isDocument) {+ if (documentExtensionFromUrlPath(lowerPath) !== null) {flags.add("document");} else if (lowerPath.endsWith(".pdf") || lowerPath.includes(".pdf/")) {// Only add PDF flag if it's not a document@@ -276,15 +270,6 @@ function buildFeatureFlags(// The meta object is usually immutable, except for the logs array, and in edge cases (e.g. a new feature is suddenly required)// Having a meta object that is treated as immutable helps the code stay clean and easily tracable,// while also retaining the benefits that WebScraper had from its OOP design.-const DOCUMENT_EXTENSIONS = new Set([- ".docx",- ".doc",- ".odt",- ".rtf",- ".xlsx",- ".xls",-]);-const HTML_EXTENSIONS = new Set([".html", ".htm", ".xhtml"]);async function writeUploadedFileToTemp(@@ -314,20 +299,9 @@ function isPdfUpload(filename: string, contentType?: string): boolean {function isDocumentUpload(filename: string, contentType?: string): boolean {const ext = path.extname(filename).toLowerCase();- const normalizedType = contentType?.toLowerCase() ?? "";return (DOCUMENT_EXTENSIONS.has(ext) ||- normalizedType.includes(- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",- ) ||- normalizedType.includes("application/vnd.ms-excel") ||- normalizedType.includes(- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",- ) ||- normalizedType.includes("application/msword") ||- normalizedType.includes("application/vnd.oasis.opendocument.text") ||- normalizedType.includes("application/rtf") ||- normalizedType.includes("text/rtf")+ documentExtensionFromContentType(contentType) !== null);}@@ -419,6 +393,7 @@ async function buildMetaObject(proxyUsed: "basic",contentType:contentType ||+ documentContentTypeFromExtension(fallbackExtension) ||"application/vnd.openxmlformats-officedocument.wordprocessingml.document",};} else if (isHtmlUpload(filename, contentType)) {