Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Custom Rust document parser (#2159)

tom443de6fcc5bb
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.toml
index 86ef34d4d..7c38f4116 100644
--- a/apps/api/native/Cargo.toml
+++ b/apps/api/native/Cargo.toml
@@ -9,9 +9,11 @@ version = "0.1.0"
crate-type = ["cdylib"]
[dependencies]
+chrono = { version = "0.4", features = ["serde"] }
kuchikiki = "0.8.2"
lol_html = "2.6.0"
lopdf = "0.38.0"
+maud = "0.27.0"
napi = { version = "3.0.0", features = ["serde-json"] }
napi-derive = "3.0.0"
nodesig = { git = "https://github.com/firecrawl/nodesig" }
@@ -22,6 +24,7 @@ serde = { version = "1.0.219", features = ["derive"] }
serde_json = "1.0.143"
texting_robots = "0.2.2"
url = "2.5.7"
+zip = "5.0.0"
[build-dependencies]
napi-build = "2"
diff --git a/apps/api/native/src/document/mod.rs b/apps/api/native/src/document/mod.rs
new file mode 100644
index 000000000..173e33613
--- /dev/null
+++ b/apps/api/native/src/document/mod.rs
@@ -0,0 +1,64 @@
+pub mod model;
+pub mod providers;
+pub mod renderers;
+
+pub use providers::factory::DocumentType;
+
+use crate::document::model::Document;
+use crate::document::providers::factory::ProviderFactory;
+use crate::document::renderers::html::HtmlRenderer;
+use napi::bindgen_prelude::*;
+use napi_derive::napi;
+
+#[napi]
+#[derive(Debug)]
+pub enum ConversionError {
+ UnsupportedFormat,
+ Provider(String),
+ Io(String),
+}
+
+impl From<std::io::Error> for ConversionError {
+ fn from(e: std::io::Error) -> Self {
+ ConversionError::Io(e.to_string())
+ }
+}
+
+#[napi]
+pub struct DocumentConverter {
+ factory: ProviderFactory,
+ html_renderer: HtmlRenderer,
+}
+
+impl Default for DocumentConverter {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+#[napi]
+impl DocumentConverter {
+ #[napi(constructor)]
+ pub fn new() -> Self {
+ Self {
+ factory: ProviderFactory::new(),
+ html_renderer: HtmlRenderer::new(),
+ }
+ }
+
+ #[napi]
+ pub fn convert_buffer_to_html(
+ &self,
+ data: &[u8],
+ doc_type: DocumentType,
+ ) -> napi::Result<String> {
+ let provider = self.factory.get_provider(doc_type);
+
+ let document: Document = provider
+ .parse_buffer(data)
+ .map_err(|e| Error::new(Status::GenericFailure, format!("Provider error: {e}")))?;
+
+ let html = self.html_renderer.render(&document);
+ Ok(html)
+ }
+}
diff --git a/apps/api/native/src/document/model/mod.rs b/apps/api/native/src/document/model/mod.rs
new file mode 100644
index 000000000..2c9777455
--- /dev/null
+++ b/apps/api/native/src/document/model/mod.rs
@@ -0,0 +1,135 @@
+use chrono::{DateTime, Utc};
+use std::num::NonZeroU32;
+
+#[derive(Debug, Clone)]
+pub struct Document {
+ pub blocks: Vec<Block>,
+ pub metadata: DocumentMetadata,
+ pub notes: Vec<Note>,
+ pub comments: Vec<Comment>,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct DocumentMetadata {
+ pub title: Option<String>,
+ pub author: Option<String>,
+ pub created: Option<DateTime<Utc>>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct NoteId(pub String);
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct CommentId(pub String);
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct BookmarkId(pub String);
+
+#[derive(Debug, Clone)]
+pub enum Block {
+ Paragraph(Paragraph),
+ Table(Table),
+ List(List),
+ Image(Image),
+}
+
+#[derive(Debug, Clone)]
+pub struct Paragraph {
+ pub kind: ParagraphKind,
+ pub inlines: Vec<Inline>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ParagraphKind {
+ Normal,
+ Heading(u8), // 1..=6 will render as <h1>.. <h6>
+ Blockquote,
+}
+
+#[derive(Debug, Clone)]
+pub enum Inline {
+ Text(String),
+ LineBreak,
+ Link { href: String, children: Vec<Inline> },
+
+ Strong(Vec<Inline>),
+ Em(Vec<Inline>),
+ Del(Vec<Inline>),
+ Code(String),
+ Sup(Vec<Inline>),
+ Sub(Vec<Inline>),
+
+ FootnoteRef(NoteId),
+ EndnoteRef(NoteId),
+ CommentRef(CommentId),
+ Bookmark(BookmarkId),
+}
+
+#[derive(Debug, Clone)]
+pub struct Image {
+ pub src: String,
+ pub alt: Option<String>,
+}
+
+#[derive(Debug, Clone)]
+pub struct Table {
+ pub rows: Vec<TableRow>,
+}
+
+#[derive(Debug, Clone)]
+pub struct TableRow {
+ pub cells: Vec<TableCell>,
+ pub kind: TableRowKind,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TableRowKind {
+ Header,
+ Body,
+ Footer,
+}
+
+#[derive(Debug, Clone)]
+pub struct TableCell {
+ pub blocks: Vec<Block>,
+ pub colspan: NonZeroU32,
+ pub rowspan: NonZeroU32,
+}
+
+#[derive(Debug, Clone)]
+pub struct List {
+ pub items: Vec<ListItem>,
+ pub list_type: ListType,
+}
+
+#[derive(Debug, Clone)]
+pub struct ListItem {
+ pub blocks: Vec<Block>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ListType {
+ Ordered,
+ Unordered,
+}
+
+#[derive(Debug, Clone)]
+pub struct Note {
+ pub id: NoteId,
+ pub kind: NoteKind,
+ pub blocks: Vec<Block>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum NoteKind {
+ Footnote,
+ Endnote,
+}
+
+#[derive(Debug, Clone)]
+pub struct Comment {
+ pub id: CommentId,
+ pub author_name: Option<String>,
+ pub author_initials: Option<String>,
+ pub blocks: Vec<Block>,
+}
diff --git a/apps/api/native/src/document/providers/docx.rs b/apps/api/native/src/document/providers/docx.rs
new file mode 100644
index 000000000..9bb168fb1
--- /dev/null
+++ b/apps/api/native/src/document/providers/docx.rs
@@ -0,0 +1,1074 @@
+use crate::document::model::*;
+use crate::document::providers::DocumentProvider;
+use chrono::{DateTime, Utc};
+use roxmltree::{Document as XmlDoc, Node};
+use std::collections::HashMap;
+use std::error::Error;
+use std::io::{Read, Seek};
+use std::num::NonZeroU32;
+use zip::read::ZipArchive;
+
+pub struct DocxProvider;
+
+impl DocxProvider {
+ pub fn new() -> Self {
+ Self
+ }
+}
+
+impl DocumentProvider for DocxProvider {
+ fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {
+ let cursor = std::io::Cursor::new(data);
+ let mut zip = ZipArchive::new(cursor)?;
+
+ let relationships = read_relationships(&mut zip, "word/_rels/document.xml.rels");
+ let styles = read_styles(&mut zip);
+ let numbering = read_numbering(&mut zip);
+
+ let document_xml = read_zip_text(&mut zip, "word/document.xml")
+ .ok_or("Missing word/document.xml in document")?;
+ let xml = XmlDoc::parse(strip_bom(&document_xml))?;
+
+ let metadata = read_core_properties(&mut zip).unwrap_or_default();
+
+ let size_buckets = compute_style_size_buckets_for_doc(&xml, &styles);
+ let mut blocks = Vec::new();
+ if let Some(body) = xml.descendants().find(|n| is_tag(n, "body")) {
+ blocks = parse_block_children(
+ &body,
+ &relationships,
+ &styles,
+ &size_buckets,
+ &numbering,
+ &mut zip,
+ );
+ }
+
+ let mut notes = Vec::new();
+ notes.extend(read_notes(
+ &mut zip,
+ "word/footnotes.xml",
+ "word/_rels/footnotes.xml.rels",
+ NoteKind::Footnote,
+ &styles,
+ &size_buckets,
+ &numbering,
+ ));
+ notes.extend(read_notes(
+ &mut zip,
+ "word/endnotes.xml",
+ "word/_rels/endnotes.xml.rels",
+ NoteKind::Endnote,
+ &styles,
+ &size_buckets,
+ &numbering,
+ ));
+
+ let comments = read_comments(
+ &mut zip,
+ "word/comments.xml",
+ "word/_rels/comments.xml.rels",
+ &styles,
+ &size_buckets,
+ &numbering,
+ );
+
+ Ok(Document {
+ blocks,
+ metadata,
+ notes,
+ comments,
+ })
+ }
+
+ fn name(&self) -> &'static str {
+ "docx"
+ }
+}
+
+fn read_zip_text<R: Read + std::io::Seek>(zip: &mut ZipArchive<R>, path: &str) -> Option<String> {
+ let mut file = zip.by_name(path).ok()?;
+ let mut s = String::new();
+ file.read_to_string(&mut s).ok()?;
+ Some(s)
+}
+
+fn strip_bom(s: &str) -> &str {
+ const BOM: char = '\u{FEFF}';
+ s.strip_prefix(BOM).unwrap_or(s)
+}
+
+#[derive(Debug, Clone, Default)]
+struct Relationships {
+ targets: HashMap<String, String>,
+}
+
+impl Relationships {
+ fn get(&self, id: &str) -> Option<&str> {
+ self.targets.get(id).map(|s| s.as_str())
+ }
+}
+
+fn read_relationships<R: Read + std::io::Seek>(
+ zip: &mut ZipArchive<R>,
+ path: &str,
+) -> Relationships {
+ let xml_text = match read_zip_text(zip, path) {
+ Some(s) => s,
+ None => return Relationships::default(),
+ };
+ let xml = match XmlDoc::parse(strip_bom(&xml_text)) {
+ Ok(d) => d,
+ Err(_) => return Relationships::default(),
+ };
+ let mut map = HashMap::new();
+ for rel in xml.descendants().filter(|n| is_tag(n, "Relationship")) {
+ if let (Some(id), Some(target)) = (get_attr_local(&rel, "Id"), get_attr_local(&rel, "Target")) {
+ map.insert(id.to_string(), target.to_string());
+ }
+ }
+ Relationships { targets: map }
+}
+
+fn read_core_properties<R: Read + std::io::Seek>(
+ zip: &mut ZipArchive<R>,
+) -> Option<DocumentMetadata> {
+ let text = read_zip_text(zip, "docProps/core.xml")?;
+ let xml = XmlDoc::parse(strip_bom(&text)).ok()?;
+ let mut meta = DocumentMetadata::default();
+
+ if let Some(title) = xml
+ .descendants()
+ .find(|n| is_tag(n, "title"))
+ .and_then(|n| n.text())
+ {
+ let trimmed = title.trim();
+ if !trimmed.is_empty() {
+ meta.title = Some(trimmed.to_string());
+ }
+ }
+ if let Some(author) = xml
+ .descendants()
+ .find(|n| is_tag(n, "creator"))
+ .and_then(|n| n.text())
+ {
+ let trimmed = author.trim();
+ if !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("unknown") {
+ meta.author = Some(trimmed.to_string());
+ }
+ }
+ if let Some(created) = xml
+ .descendants()
+ .find(|n| is_tag(n, "created"))
+ .and_then(|n| n.text())
+ {
+ if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(created) {
+ meta.created = Some(DateTime::<Utc>::from(dt));
+ }
+ }
+
+ Some(meta)
+}
+
+#[derive(Debug, Default)]
+struct StylesInfo {
+ heading_level_by_style_id: HashMap<String, u8>,
+ name_by_style_id: HashMap<String, String>,
+ default_size_by_style_id: HashMap<String, u32>,
+}
+
+fn read_styles<R: Read + Seek>(zip: &mut ZipArchive<R>) -> StylesInfo {
+ let text = match read_zip_text(zip, "word/styles.xml") {
+ Some(t) => t,
+ None => return StylesInfo::default(),
+ };
+ let doc = match XmlDoc::parse(strip_bom(&text)) {
+ Ok(d) => d,
+ Err(_) => return StylesInfo::default(),
+ };
+ let mut info = StylesInfo::default();
+
+ for style in doc.descendants().filter(|n| is_tag(n, "style")) {
+ if get_attr_local(&style, "type") != Some("paragraph") {
+ continue;
+ }
+ let Some(style_id) = get_attr_local(&style, "styleId") else {
+ continue;
+ };
+ let style_id = style_id.to_string();
+
+ if let Some(name) = child(&style, "name").and_then(|n| get_attr_local(&n, "val")) {
+ info
+ .name_by_style_id
+ .insert(style_id.clone(), name.to_string());
+ }
+
+ if let Some(rpr) = child(&style, "rPr") {
+ if let Some(sz) = child(&rpr, "sz").and_then(|n| get_attr_local(&n, "val")) {
+ if let Ok(v) = sz.parse::<u32>() {
+ info.default_size_by_style_id.insert(style_id.clone(), v);
+ }
+ }
+ }
+
+ if let Some(ppr) = child(&style, "pPr") {
+ if let Some(ol) = child(&ppr, "outlineLvl").and_then(|n| get_attr_local(&n, "val")) {
+ if let Ok(v) = ol.parse::<u8>() {
+ info
+ .heading_level_by_style_id
+ .insert(style_id.clone(), (v + 1).min(6));
+ }
+ }
+ }
+
+ if !info.heading_level_by_style_id.contains_key(&style_id) {
+ let mut assigned: Option<u8> = None;
+ if let Some(name) = info.name_by_style_id.get(&style_id) {
+ assigned = parse_heading_level(name);
+ }
+ if assigned.is_none() {
+ assigned = parse_heading_level(&style_id);
+ }
+ if assigned.is_none() {
+ let name_l = info
+ .name_by_style_id
+ .get(&style_id)
+ .map(|s| s.to_ascii_lowercase())
+ .unwrap_or_default();
+ let id_l = style_id.to_ascii_lowercase();
+ if name_l.contains("title") || id_l.contains("title") {
+ assigned = Some(1);
+ } else if name_l.contains("heading") || id_l.contains("heading") {
+ assigned = Some(2);
+ }
+ }
+ if let Some(l) = assigned {
+ info.heading_level_by_style_id.insert(style_id, l);
+ }
+ }
+ }
+ info
+}
+
+fn is_tag(node: &Node, local: &str) -> bool {
+ node.is_element() && node.tag_name().name() == local
+}
+
+fn get_attr_local<'a>(node: &Node<'a, 'a>, local: &str) -> Option<&'a str> {
+ node
+ .attributes()
+ .find(|a| {
+ let name = a.name();
+ match name.rsplit_once(':') {
+ Some((_, l)) => l == local,
+ None => name == local,
+ }
+ })
+ .map(|a| a.value())
+}
+
+fn child<'a>(node: &Node<'a, 'a>, local: &str) -> Option<Node<'a, 'a>> {
+ node
+ .children()
+ .find(|n| n.is_element() && n.tag_name().name() == local)
+}
+
+fn children<'a, 'b>(
+ node: &Node<'a, 'a>,
+ local: &'b str,
+) -> impl Iterator<Item = Node<'a, 'a>> + use<'a, 'b> {
+ node
+ .children()
+ .filter(move |n| n.is_element() && n.tag_name().name() == local)
+}
+
+fn parse_paragraph_with_listinfo(
+ node: &Node,
+ rels: &Relationships,
+ styles: &StylesInfo,
+ size_buckets: &HashMap<String, Vec<u32>>,
+ numbering: &NumberingInfo,
+) -> Option<(Paragraph, Option<ListInfo>)> {
+ let kind = paragraph_kind(node, styles, size_buckets);
+ let base_style = paragraph_run_style(node);
+ let mut inlines = Vec::new();
+
+ for child in node.children().filter(|n| n.is_element()) {
+ if is_tag(&child, "r") {
+ let run_inlines = parse_run(&child, rels, &base_style);
+ inlines.extend(run_inlines);
+ } else if is_tag(&child, "hyperlink") {
+ if let Some(link) = parse_hyperlink(&child, rels, &base_style) {
+ inlines.push(link);
+ }
+ } else if is_tag(&child, "bookmarkStart") {
+ if let Some(name) = get_attr_local(&child, "name") {
+ inlines.push(Inline::Bookmark(BookmarkId(name.to_string())));
+ }
+ } else if is_tag(&child, "br") {
+ inlines.push(Inline::LineBreak);
+ }
+ }
+
+ let list_info = paragraph_list_info(node, numbering);
+ Some((Paragraph { kind, inlines }, list_info))
+}
+
+fn paragraph_kind(
+ p: &Node,
+ styles: &StylesInfo,
+ size_buckets: &HashMap<String, Vec<u32>>,
+) -> ParagraphKind {
+ let Some(ppr) = child(p, "pPr") else {
+ return ParagraphKind::Normal;
+ };
+
+ if let Some(level) = child(&ppr, "outlineLvl").and_then(|n| get_attr_local(&n, "val")) {
+ if let Ok(v) = level.parse::<u8>() {
+ return ParagraphKind::Heading((v + 1).min(6));
+ }
+ }
+
+ if let Some(style_id) = child(&ppr, "pStyle").and_then(|n| get_attr_local(&n, "val")) {
+ let mut base_level: Option<u8> = styles.heading_level_by_style_id.get(style_id).copied();
+
+ if base_level.is_none() {
+ if let Some(name) = styles.name_by_style_id.get(style_id) {
+ if let Some(l) = parse_heading_level(name) {
+ base_level = Some(l);
+ }
+ if base_level.is_none() && name.to_ascii_lowercase().contains("quote") {
+ return ParagraphKind::Blockquote;
+ }
+ }
+ }
+ if base_level.is_none() {
+ if let Some(l) = parse_heading_level(style_id) {
+ base_level = Some(l);
+ }
+ }
+ if base_level.is_none() {
+ let id_l = style_id.to_ascii_lowercase();
+ let name_l = styles
+ .name_by_style_id
+ .get(style_id)
+ .map(|s| s.to_ascii_lowercase())
+ .unwrap_or_default();
+ if name_l.contains("title") || id_l.contains("title") {
+ base_level = Some(1);
+ } else if name_l.contains("heading") || id_l.contains("heading") {
+ base_level = Some(2);
+ } else if name_l.contains("quote") || id_l.contains("quote") {
+ return ParagraphKind::Blockquote;
+ }
+ }
+
+ if let Some(mut level) = base_level {
+ let para_size = paragraph_effective_size(p, styles, style_id);
+ if let Some(buckets) = size_buckets.get(style_id) {
+ if let Some(size) = para_size {
+ if let Some(index) = buckets.iter().position(|&s| s == size) {
+ level = (level + index as u8).min(6);
+ }
+ }
+ }
+ return ParagraphKind::Heading(level);
+ }
+ }
+
+ ParagraphKind::Normal
+}
+
+fn parse_heading_level(s: &str) -> Option<u8> {
+ let lower = s.to_ascii_lowercase();
+ let idx = lower.find("heading")?;
+ let rest = &lower[idx + "heading".len()..];
+ let digits: String = rest
+ .chars()
+ .skip_while(|c| c.is_whitespace() || *c == '-')
+ .take_while(|c| c.is_ascii_digit())
+ .collect();
+ if let Ok(n) = digits.parse::<u8>() {
+ if n >= 1 {
+ return Some(n.min(6));
+ }
+ }
+ None
+}
+
+fn paragraph_effective_size(p: &Node, styles: &StylesInfo, style_id: &str) -> Option<u32> {
+ let mut max_sz: Option<u32> = None;
+
+ if let Some(ppr) = child(p, "pPr") {
+ if let Some(rpr) = child(&ppr, "rPr") {
+ if let Some(sz) = child(&rpr, "sz").and_then(|n| get_attr_local(&n, "val")) {
+ if let Ok(v) = sz.parse::<u32>() {
+ max_sz = Some(max_sz.map_or(v, |m| m.max(v)));
+ }
+ }
+ }
+ }
+
+ for r in children(p, "r") {
+ if let Some(rpr) = child(&r, "rPr") {
+ if let Some(sz) = child(&rpr, "sz").and_then(|n| get_attr_local(&n, "val")) {
+ if let Ok(v) = sz.parse::<u32>() {
+ max_sz = Some(max_sz.map_or(v, |m| m.max(v)));
+ }
+ }
+ }
+ }
+
+ max_sz.or_else(|| styles.default_size_by_style_id.get(style_id).copied())
+}
+
+fn compute_style_size_buckets_for_doc(
+ xml: &XmlDoc,
+ styles: &StylesInfo,
+) -> HashMap<String, Vec<u32>> {
+ let mut sets: HashMap<String, std::collections::BTreeSet<u32>> = HashMap::new();
+
+ for p in xml.descendants().filter(|n| is_tag(n, "p")) {
+ let Some(style_id) = child(&p, "pPr")
+ .and_then(|n| child(&n, "pStyle"))
+ .and_then(|n| get_attr_local(&n, "val"))
+ else {
+ continue;
+ };
+
+ let id_l = style_id.to_ascii_lowercase();
+ let name_l = styles
+ .name_by_style_id
+ .get(style_id)
+ .map(|s| s.to_ascii_lowercase())
+ .unwrap_or_default();
+
+ if !(id_l.contains("heading")
+ || id_l.contains("title")
+ || name_l.contains("heading")
+ || name_l.contains("title"))
+ {
+ continue;
+ }
+
+ if let Some(sz) = paragraph_effective_size(&p, styles, style_id) {
+ sets.entry(style_id.to_string()).or_default().insert(sz);
+ }
+ }
+
+ sets
+ .into_iter()
+ .map(|(k, vset)| {
+ let mut v: Vec<u32> = vset.into_iter().collect();
+ v.sort_by(|a, b| b.cmp(a));
+ (k, v)
+ })
+ .collect()
+}
+
+#[derive(Clone, Default)]
+struct RunStyle {
+ bold: Option<bool>,
+ italic: Option<bool>,
+ strike: Option<bool>,
+ code: Option<bool>,
+ vert_align: Option<VerticalAlign>,
+}
+
+#[derive(Clone, Copy)]
+enum VerticalAlign {
+ Baseline,
+ Superscript,
+ Subscript,
+}
+
+#[derive(Clone, Copy)]
+struct ResolvedRunStyle {
+ bold: bool,
+ italic: bool,
+ strike: bool,
+ code: bool,
+ vert_align: VerticalAlign,
+}
+
+impl RunStyle {
+ fn merged(&self, overrides: &RunStyle) -> RunStyle {
+ RunStyle {
+ bold: overrides.bold.or(self.bold),
+ italic: overrides.italic.or(self.italic),
+ strike: overrides.strike.or(self.strike),
+ code: overrides.code.or(self.code),
+ vert_align: overrides.vert_align.or(self.vert_align),
+ }
+ }
+
+ fn resolve_with(&self, local: &RunStyle) -> ResolvedRunStyle {
+ ResolvedRunStyle {
+ bold: local.bold.or(self.bold).unwrap_or(false),
+ italic: local.italic.or(self.italic).unwrap_or(false),
+ strike: local.strike.or(self.strike).unwrap_or(false),
+ code: local.code.or(self.code).unwrap_or(false),
+ vert_align: local
+ .vert_align
+ .or(self.vert_align)
+ .unwrap_or(VerticalAlign::Baseline),
+ }
+ }
+}
+
+impl ResolvedRunStyle {
+ fn apply(self, mut inlines: Vec<Inline>) -> Vec<Inline> {
+ if self.strike {
+ inlines = vec![Inline::Del(inlines)];
+ }
+ if self.italic {
+ inlines = vec![Inline::Em(inlines)];
+ }
+ if self.bold {
+ inlines = vec![Inline::Strong(inlines)];
+ }
+ match self.vert_align {
+ VerticalAlign::Superscript => vec![Inline::Sup(inlines)],
+ VerticalAlign::Subscript => vec![Inline::Sub(inlines)],
+ VerticalAlign::Baseline => inlines,
+ }
+ }
+}
+
+fn read_on_off(node: &Node) -> Option<bool> {
+ let value = get_attr_local(node, "val").map(|v| v.to_ascii_lowercase());
+ match value.as_deref() {
+ None => Some(true),
+ Some("0") | Some("false") | Some("off") => Some(false),
+ Some(_) => Some(true),
+ }
+}
+
+fn run_style_from_rpr(rpr: &Node) -> RunStyle {
+ let mut style = RunStyle::default();
+
+ if let Some(b) = child(rpr, "b").and_then(|n| read_on_off(&n)) {
+ style.bold = Some(b);
+ }
+ if let Some(i) = child(rpr, "i").and_then(|n| read_on_off(&n)) {
+ style.italic = Some(i);
+ }
+ if let Some(s) = child(rpr, "strike").and_then(|n| read_on_off(&n)) {
+ style.strike = Some(s);
+ }
+ if let Some(rstyle) = child(rpr, "rStyle").and_then(|n| get_attr_local(&n, "val")) {
+ if rstyle.to_ascii_lowercase().contains("code") {
+ style.code = Some(true);
+ }
+ }
+ if let Some(va) = child(rpr, "vertAlign").and_then(|n| get_attr_local(&n, "val")) {
+ let lower = va.to_ascii_lowercase();
+ let val = if lower == "sup" || lower == "superscript" {
+ VerticalAlign::Superscript
+ } else if lower == "sub" || lower == "subscript" {
+ VerticalAlign::Subscript
+ } else {
+ VerticalAlign::Baseline
+ };
+ style.vert_align = Some(val);
+ }
+
+ style
+}
+
+fn paragraph_run_style(p: &Node) -> RunStyle {
+ child(p, "pPr")
+ .and_then(|ppr| child(&ppr, "rPr"))
+ .map(|rpr| run_style_from_rpr(&rpr))
+ .unwrap_or_default()
+}
+
+fn parse_run(run: &Node, _rels: &Relationships, base_style: &RunStyle) -> Vec<Inline> {
+ let local_style = child(run, "rPr")
+ .map(|rpr| run_style_from_rpr(&rpr))
+ .unwrap_or_default();
+ let resolved = base_style.resolve_with(&local_style);
+
+ let mut out = Vec::new();
+
+ for c in run.children().filter(|n| n.is_element()) {
+ if is_tag(&c, "t") {
+ if let Some(text) = c.text() {
+ out.push(Inline::Text(text.to_string()));
+ }
+ } else if is_tag(&c, "br") {
+ out.push(Inline::LineBreak);
+ } else if is_tag(&c, "tab") {
+ out.push(Inline::Text("\t".to_string()));
+ } else if is_tag(&c, "footnoteReference") {
+ if let Some(id) = get_attr_local(&c, "id") {
+ out.push(Inline::FootnoteRef(NoteId(id.to_string())));
+ }
+ } else if is_tag(&c, "endnoteReference") {
+ if let Some(id) = get_attr_local(&c, "id") {
+ out.push(Inline::EndnoteRef(NoteId(id.to_string())));
+ }
+ } else if is_tag(&c, "commentReference") {
+ if let Some(id) = get_attr_local(&c, "id") {
+ out.push(Inline::CommentRef(CommentId(id.to_string())));
+ }
+ }
+ }
+
+ if resolved.code {
+ let code_text: String = out
+ .iter()
+ .filter_map(|i| match i {
+ Inline::Text(s) => Some(s.as_str()),
+ _ => None,
+ })
+ .collect();
+ if !code_text.is_empty() {
+ return vec![Inline::Code(code_text)];
+ }
+ }
+
+ resolved.apply(out)
+}
+
+fn parse_hyperlink(node: &Node, rels: &Relationships, base_style: &RunStyle) -> Option<Inline> {
+ let href = if let Some(id) = get_attr_local(node, "id") {
+ rels.get(id).map(|s| s.to_string())
+ } else {
+ get_attr_local(node, "anchor").map(|anchor| format!("#{anchor}"))
+ }?;
+
+ let link_style = child(node, "rPr")
+ .map(|rpr| run_style_from_rpr(&rpr))
+ .unwrap_or_default();
+ let combined_style = base_style.merged(&link_style);
+
+ let mut children = Vec::new();
+ for child in node.children().filter(|n| n.is_element()) {
+ if is_tag(&child, "r") {
+ children.extend(parse_run(&child, rels, &combined_style));
+ }
+ }
+
+ Some(Inline::Link { href, children })
+}
+
+fn parse_table<R: Read + Seek>(
+ node: &Node,
+ rels: &Relationships,
+ styles: &StylesInfo,
+ size_buckets: &HashMap<String, Vec<u32>>,
+ numbering: &NumberingInfo,
+ zip: &mut ZipArchive<R>,
+) -> Option<Table> {
+ let mut rows = Vec::new();
+ for tr in children(node, "tr") {
+ let kind = table_row_kind(&tr);
+ let mut cells = Vec::new();
+ for tc in children(&tr, "tc") {
+ let cell_blocks = parse_block_children(&tc, rels, styles, size_buckets, numbering, zip);
+ let cell = TableCell {
+ blocks: cell_blocks,
+ colspan: NonZeroU32::new(1).unwrap(),
+ rowspan: NonZeroU32::new(1).unwrap(),
+ };
+ cells.push(cell);
+ }
+ rows.push(TableRow { cells, kind });
+ }
+
+ if !rows.is_empty() && rows.iter().any(|r| matches!(r.kind, TableRowKind::Header)) {
+ if let Some(last) = rows.last_mut() {
+ last.kind = TableRowKind::Footer;
+ }
+ }
+ Some(Table { rows })
+}
+
+fn table_row_kind(tr: &Node) -> TableRowKind {
+ if let Some(trpr) = child(tr, "trPr") {
+ if child(&trpr, "tblHeader").is_some() {
+ return TableRowKind::Header;
+ }
+ }
+ TableRowKind::Body
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct ListInfo {
+ list_type: ListType,
+ num_id: String,
+ ilvl: u32,
+}
+
+#[derive(Debug, Default)]
+struct NumberingInfo {
+ num_to_abstract: HashMap<String, String>,
+ abstract_levels: HashMap<String, HashMap<String, ListType>>,
+}
+
+impl NumberingInfo {
+ fn list_type(&self, num_id: &str, ilvl: &str) -> Option<ListType> {
+ let abs = self.num_to_abstract.get(num_id)?;
+ let levels = self.abstract_levels.get(abs)?;
+ levels.get(ilvl).copied()
+ }
+}
+
+fn read_numbering<R: Read + Seek>(zip: &mut ZipArchive<R>) -> NumberingInfo {
+ let text = match read_zip_text(zip, "word/numbering.xml") {
+ Some(t) => t,
+ None => return NumberingInfo::default(),
+ };
+ let doc = match XmlDoc::parse(strip_bom(&text)) {
+ Ok(d) => d,
+ Err(_) => return NumberingInfo::default(),
+ };
+
+ let mut info = NumberingInfo::default();
+
+ for num in doc.descendants().filter(|n| is_tag(n, "num")) {
+ if let Some(num_id) = get_attr_local(&num, "numId") {
+ if let Some(abs) = child(&num, "abstractNumId").and_then(|n| get_attr_local(&n, "val")) {
+ info
+ .num_to_abstract
+ .insert(num_id.to_string(), abs.to_string());
+ }
+ }
+ }
+
+ for abs in doc.descendants().filter(|n| is_tag(n, "abstractNum")) {
+ if let Some(abs_id) = get_attr_local(&abs, "abstractNumId") {
+ let mut levels: HashMap<String, ListType> = HashMap::new();
+ for lvl in children(&abs, "lvl") {
+ if let Some(ilvl) = get_attr_local(&lvl, "ilvl") {
+ let fmt = child(&lvl, "numFmt").and_then(|n| get_attr_local(&n, "val"));
+ let list_type = match fmt.unwrap_or("") {
+ "bullet" => ListType::Unordered,
+ _ => ListType::Ordered,
+ };
+ levels.insert(ilvl.to_string(), list_type);
+ }
+ }
+ info.abstract_levels.insert(abs_id.to_string(), levels);
+ }
+ }
+
+ info
+}
+
+fn paragraph_list_info(p: &Node, numbering: &NumberingInfo) -> Option<ListInfo> {
+ let ppr = child(p, "pPr")?;
+ let numpr = child(&ppr, "numPr")?;
+ let ilvl_str = child(&numpr, "ilvl").and_then(|n| get_attr_local(&n, "val"))?;
+ let num_id = child(&numpr, "numId").and_then(|n| get_attr_local(&n, "val"))?;
+ let ilvl: u32 = ilvl_str.parse().unwrap_or(0);
+ let list_type = numbering
+ .list_type(num_id, ilvl_str)
+ .unwrap_or(ListType::Unordered);
+ Some(ListInfo {
+ list_type,
+ num_id: num_id.to_string(),
+ ilvl,
+ })
+}
+
+fn parse_block_children<R: Read + Seek>(
+ parent: &Node,
+ rels: &Relationships,
+ styles: &StylesInfo,
+ size_buckets: &HashMap<String, Vec<u32>>,
+ numbering: &NumberingInfo,
+ zip: &mut ZipArchive<R>,
+) -> Vec<Block> {
+ let nodes: Vec<Node> = parent.children().filter(|n| n.is_element()).collect();
+ let mut out: Vec<Block> = Vec::new();
+ let mut i = 0usize;
+
+ while i < nodes.len() {
+ let node = &nodes[i];
+ if is_tag(node, "p") {
+ if paragraph_list_info(node, numbering).is_some() {
+ let (list, new_i) = parse_list(&nodes, i, rels, styles, size_buckets, numbering, zip);
+ if !list.items.is_empty() {
+ out.push(Block::List(list));
+ }
+ i = new_i;
+ continue;
+ }
+ if let Some(image) = parse_image_paragraph(node, rels, zip) {
+ out.push(Block::Image(image));
+ i += 1;
+ continue;
+ }
+ if let Some((para, _)) =
+ parse_paragraph_with_listinfo(node, rels, styles, size_buckets, numbering)
+ {
+ if paragraph_has_visible_content(&para) {
+ out.push(Block::Paragraph(para));
+ }
+ }
+ i += 1;
+ } else if is_tag(node, "tbl") {
+ if let Some(table) = parse_table(node, rels, styles, size_buckets, numbering, zip) {
+ out.push(Block::Table(table));
+ }
+ i += 1;
+ } else {
+ i += 1;
+ }
+ }
+ out
+}
+
+fn parse_list<R: Read + Seek>(
+ nodes: &[Node],
+ mut i: usize,
+ rels: &Relationships,
+ styles: &StylesInfo,
+ size_buckets: &HashMap<String, Vec<u32>>,
+ numbering: &NumberingInfo,
+ zip: &mut ZipArchive<R>,
+) -> (List, usize) {
+ let first_info =
+ paragraph_list_info(&nodes[i], numbering).expect("parse_list called at non-list paragraph");
+ let base_ilvl = first_info.ilvl;
+ let base_num_id = first_info.num_id.clone();
+ let base_type = first_info.list_type;
+
+ let mut list = List {
+ items: Vec::new(),
+ list_type: base_type,
+ };
+
+ while i < nodes.len() {
+ let node = &nodes[i];
+ if !is_tag(node, "p") {
+ break;
+ }
+ let info = match paragraph_list_info(node, numbering) {
+ Some(x) => x,
+ None => break,
+ };
+ if info.ilvl < base_ilvl {
+ break;
+ }
+ if info.ilvl == base_ilvl && (info.list_type != base_type || info.num_id != base_num_id) {
+ break;
+ }
+
+ if info.ilvl == base_ilvl {
+ let mut blocks: Vec<Block> = Vec::new();
+ if let Some(image) = parse_image_paragraph(node, rels, zip) {
+ blocks.push(Block::Image(image));
+ } else if let Some((para, _)) =
+ parse_paragraph_with_listinfo(node, rels, styles, size_buckets, numbering)
+ {
+ if paragraph_has_visible_content(&para) {
+ blocks.push(Block::Paragraph(para));
+ }
+ }
+ list.items.push(ListItem { blocks });
+ i += 1;
+
+ loop {
+ if i >= nodes.len() {
+ break;
+ }
+ let node2 = &nodes[i];
+ if !is_tag(node2, "p") {
+ break;
+ }
+ match paragraph_list_info(node2, numbering) {
+ Some(sub) if sub.ilvl > base_ilvl => {
+ let (sublist, new_i) = parse_list(nodes, i, rels, styles, size_buckets, numbering, zip);
+ if let Some(last) = list.items.last_mut() {
+ last.blocks.push(Block::List(sublist));
+ }
+ i = new_i;
+ }
+ _ => break,
+ }
+ }
+
+ if list.items.last().is_some_and(|last| last.blocks.is_empty()) {
+ list.items.pop();
+ }
+ }
+ }
+
+ (list, i)
+}
+
+fn paragraph_has_visible_content(p: &Paragraph) -> bool {
+ inlines_have_visible_content(&p.inlines)
+}
+
+fn inlines_have_visible_content(inlines: &[Inline]) -> bool {
+ inlines.iter().any(inline_is_visible)
+}
+
+fn inline_is_visible(i: &Inline) -> bool {
+ match i {
+ Inline::Text(t) => !t.trim().is_empty(),
+ Inline::LineBreak => false,
+ Inline::Link { children, .. } => inlines_have_visible_content(children),
+ Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {
+ inlines_have_visible_content(c)
+ }
+ Inline::Code(c) => !c.trim().is_empty(),
+ Inline::FootnoteRef(_) | Inline::EndnoteRef(_) | Inline::CommentRef(_) => true,
+ Inline::Bookmark(_) => false,
+ }
+}
+
+fn parse_image_paragraph<R: Read + Seek>(
+ p: &Node,
+ rels: &Relationships,
+ zip: &mut ZipArchive<R>,
+) -> Option<Image> {
+ let has_text = p
+ .descendants()
+ .filter(|n| is_tag(n, "t"))
+ .any(|t| t.text().map(|s| !s.trim().is_empty()).unwrap_or(false));
+ if has_text {
+ return None;
+ }
+
+ if let Some(drawing) = p.descendants().find(|n| is_tag(n, "drawing")) {
+ if let Some(img) = image_from_drawing(&drawing, rels, zip) {
+ return Some(img);
+ }
+ }
+
+ if let Some(pict) = p.descendants().find(|n| is_tag(n, "pict")) {
+ if let Some(img) = image_from_vml(&pict, rels, zip) {
+ return Some(img);
+ }
+ }
+ None
+}
+
+fn image_from_drawing<R: Read + Seek>(
+ drawing: &Node,
+ rels: &Relationships,
+ zip: &mut ZipArchive<R>,
+) -> Option<Image> {
+ let blip = drawing.descendants().find(|n| is_tag(n, "blip"))?;
+ let rel_id = get_attr_local(&blip, "embed").or_else(|| get_attr_local(&blip, "link"))?;
+ let alt = drawing
+ .descendants()
+ .find(|n| is_tag(n, "docPr"))
+ .and_then(|n| get_attr_local(&n, "descr").or_else(|| get_attr_local(&n, "title")))
+ .map(|s| s.to_string());
+ image_from_relationship_id(rel_id, rels, zip, alt)
+}
+
+fn image_from_vml<R: Read + Seek>(
+ pict: &Node,
+ rels: &Relationships,
+ zip: &mut ZipArchive<R>,
+) -> Option<Image> {
+ let imagedata = pict.descendants().find(|n| is_tag(n, "imagedata"))?;
+ let rel_id = get_attr_local(&imagedata, "id")?;
+ let alt = get_attr_local(&imagedata, "title").map(|s| s.to_string());
+ image_from_relationship_id(rel_id, rels, zip, alt)
+}
+
+fn image_from_relationship_id<R: Read + Seek>(
+ rid: &str,
+ rels: &Relationships,
+ _zip: &mut ZipArchive<R>,
+ alt: Option<String>,
+) -> Option<Image> {
+ let target = rels.get(rid)?;
+ // only include external images (http/https URLs)
+ if target.starts_with("http://") || target.starts_with("https://") {
+ return Some(Image {
+ src: target.to_string(),
+ alt,
+ });
+ }
+ None
+}
+
+fn read_notes<R: Read + Seek>(
+ zip: &mut ZipArchive<R>,
+ xml_path: &str,
+ rels_path: &str,
+ kind: NoteKind,
+ styles: &StylesInfo,
+ size_buckets: &HashMap<String, Vec<u32>>,
+ numbering: &NumberingInfo,
+) -> Vec<Note> {
+ let text = match read_zip_text(zip, xml_path) {
+ Some(t) => t,
+ None => return Vec::new(),
+ };
+ let doc = match XmlDoc::parse(strip_bom(&text)) {
+ Ok(d) => d,
+ Err(_) => return Vec::new(),
+ };
+ let rels = read_relationships(zip, rels_path);
+
+ let mut notes = Vec::new();
+ let root_tag = match kind {
+ NoteKind::Footnote => "footnote",
+ NoteKind::Endnote => "endnote",
+ };
+ for n in doc.descendants().filter(|n| is_tag(n, root_tag)) {
+ let Some(id) = get_attr_local(&n, "id") else {
+ continue;
+ };
+
+ if let Some(t) = get_attr_local(&n, "type") {
+ if t == "separator" || t == "continuationSeparator" {
+ continue;
+ }
+ }
+ let blocks = parse_block_children(&n, &rels, styles, size_buckets, numbering, zip);
+ notes.push(Note {
+ id: NoteId(id.to_string()),
+ kind,
+ blocks,
+ });
+ }
+ notes
+}
+
+fn read_comments<R: Read + Seek>(
+ zip: &mut ZipArchive<R>,
+ xml_path: &str,
+ rels_path: &str,
+ styles: &StylesInfo,
+ size_buckets: &HashMap<String, Vec<u32>>,
+ numbering: &NumberingInfo,
+) -> Vec<Comment> {
+ let text = match read_zip_text(zip, xml_path) {
+ Some(t) => t,
+ None => return Vec::new(),
+ };
+ let doc = match XmlDoc::parse(strip_bom(&text)) {
+ Ok(d) => d,
+ Err(_) => return Vec::new(),
+ };
+ let rels = read_relationships(zip, rels_path);
+
+ let mut out = Vec::new();
+ for c in doc.descendants().filter(|n| is_tag(n, "comment")) {
+ let Some(id) = get_attr_local(&c, "id") else {
+ continue;
+ };
+
+ let author = get_attr_local(&c, "author").map(|s| s.to_string());
+ let initials = get_attr_local(&c, "initials").map(|s| s.to_string());
+ let blocks = parse_block_children(&c, &rels, styles, size_buckets, numbering, zip);
+ out.push(Comment {
+ id: CommentId(id.to_string()),
+ author_name: author,
+ author_initials: initials,
+ blocks,
+ });
+ }
+ out
+}
diff --git a/apps/api/native/src/document/providers/factory.rs b/apps/api/native/src/document/providers/factory.rs
new file mode 100644
index 000000000..f4ee2c4c3
--- /dev/null
+++ b/apps/api/native/src/document/providers/factory.rs
@@ -0,0 +1,37 @@
+use super::docx::DocxProvider;
+use super::odt::OdtProvider;
+use super::rtf::RtfProvider;
+use super::DocumentProvider;
+use napi_derive::napi;
+
+#[napi]
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum DocumentType {
+ Docx,
+ Rtf,
+ Odt,
+}
+
+pub struct ProviderFactory {
+ docx_provider: DocxProvider,
+ rtf_provider: RtfProvider,
+ odt_provider: OdtProvider,
+}
+
+impl ProviderFactory {
+ pub fn new() -> Self {
+ Self {
+ docx_provider: DocxProvider::new(),
+ rtf_provider: RtfProvider::new(),
+ odt_provider: OdtProvider::new(),
+ }
+ }
+
+ pub fn get_provider(&self, doc_type: DocumentType) -> &dyn DocumentProvider {
+ match doc_type {
+ DocumentType::Docx => &self.docx_provider,
+ DocumentType::Rtf => &self.rtf_provider,
+ DocumentType::Odt => &self.odt_provider,
+ }
+ }
+}
diff --git a/apps/api/native/src/document/providers/mod.rs b/apps/api/native/src/document/providers/mod.rs
new file mode 100644
index 000000000..044b8b5c3
--- /dev/null
+++ b/apps/api/native/src/document/providers/mod.rs
@@ -0,0 +1,14 @@
+use crate::document::model::Document;
+use std::error::Error;
+
+pub mod docx;
+pub mod factory;
+pub mod odt;
+pub mod rtf;
+
+pub trait DocumentProvider {
+ fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>>;
+
+ #[allow(dead_code)]
+ fn name(&self) -> &'static str;
+}
diff --git a/apps/api/native/src/document/providers/odt.rs b/apps/api/native/src/document/providers/odt.rs
new file mode 100644
index 000000000..0d5cbc8c6
--- /dev/null
+++ b/apps/api/native/src/document/providers/odt.rs
@@ -0,0 +1,764 @@
+use crate::document::model::*;
+use crate::document::providers::DocumentProvider;
+use chrono::{DateTime, Utc};
+use roxmltree::{Document as XmlDoc, Node};
+use std::collections::HashMap;
+use std::error::Error;
+use std::io::{Read, Seek};
+use std::num::NonZeroU32;
+use zip::read::ZipArchive;
+
+pub struct OdtProvider;
+
+impl OdtProvider {
+ pub fn new() -> Self {
+ Self
+ }
+}
+
+impl DocumentProvider for OdtProvider {
+ fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {
+ let cursor = std::io::Cursor::new(data);
+ let mut zip = ZipArchive::new(cursor)?;
+
+ let meta = read_meta(&mut zip).unwrap_or_default();
+ let styles = read_styles(&mut zip);
+
+ let content =
+ read_zip_text(&mut zip, "content.xml").ok_or("Missing content.xml in document")?;
+ let xml = XmlDoc::parse(strip_bom(&content))?;
+
+ let mut notes: Vec<Note> = Vec::new();
+ let mut comments: Vec<Comment> = Vec::new();
+ let mut blocks: Vec<Block> = Vec::new();
+
+ let body_text = xml
+ .descendants()
+ .find(|n| is_tag(n, "text") && n.ancestors().any(|a| is_tag(&a, "body")));
+
+ if let Some(text_node) = body_text {
+ blocks = parse_block_children_odt(&text_node, &styles, &mut notes, &mut comments, &mut zip);
+ }
+
+ Ok(Document {
+ blocks,
+ metadata: meta,
+ notes,
+ comments,
+ })
+ }
+
+ fn name(&self) -> &'static str {
+ "odt"
+ }
+}
+
+fn read_zip_text<R: Read + Seek>(zip: &mut ZipArchive<R>, path: &str) -> Option<String> {
+ let mut file = zip.by_name(path).ok()?;
+ let mut s = String::new();
+ file.read_to_string(&mut s).ok()?;
+ Some(s)
+}
+
+fn strip_bom(s: &str) -> &str {
+ const BOM: char = '\u{FEFF}';
+ s.strip_prefix(BOM).unwrap_or(s)
+}
+
+#[derive(Debug, Default, Clone)]
+struct OdtStylesInfo {
+ paragraph_names: HashMap<String, String>,
+ paragraph_outline_level: HashMap<String, u8>,
+ paragraph_text_props: HashMap<String, TextStyleProps>,
+ text_props: HashMap<String, TextStyleProps>,
+ text_font_name: HashMap<String, String>,
+ list_is_ordered: HashMap<String, bool>,
+}
+
+#[derive(Debug, Default, Clone, Copy)]
+struct TextStyleProps {
+ bold: bool,
+ italic: bool,
+ strike: bool,
+ sup: bool,
+ sub: bool,
+ code: bool,
+}
+
+fn read_styles<R: Read + Seek>(zip: &mut ZipArchive<R>) -> OdtStylesInfo {
+ let mut info = OdtStylesInfo::default();
+
+ if let Some(t) = read_zip_text(zip, "styles.xml") {
+ if let Ok(doc) = XmlDoc::parse(strip_bom(&t)) {
+ harvest_styles_from_doc(&doc, &mut info);
+ }
+ }
+ if let Some(t) = read_zip_text(zip, "content.xml") {
+ if let Ok(doc) = XmlDoc::parse(strip_bom(&t)) {
+ harvest_styles_from_doc(&doc, &mut info);
+ }
+ }
+ info
+}
+
+fn harvest_styles_from_doc(doc: &XmlDoc, out: &mut OdtStylesInfo) {
+ for s in doc.descendants().filter(|n| is_tag(n, "style")) {
+ let Some(family) = get_attr_local(&s, "family") else {
+ continue;
+ };
+ let Some(name) = get_attr_local(&s, "name") else {
+ continue;
+ };
+ let lname = name.to_string();
+
+ if family == "paragraph" {
+ out.paragraph_names.insert(lname.clone(), lname.clone());
+
+ if let Some(ppr) = child(&s, "paragraph-properties") {
+ if let Some(ol) = get_attr_local(&ppr, "outline-level") {
+ if let Ok(v) = ol.parse::<u8>() {
+ out.paragraph_outline_level.insert(lname.clone(), v.min(6));
+ }
+ }
+ }
+
+ if !out.paragraph_outline_level.contains_key(&lname) {
+ if let Some(parent) = get_attr_local(&s, "parent-style-name") {
+ if let Some(lv) = parse_odt_heading_level(parent) {
+ out.paragraph_outline_level.insert(lname.clone(), lv);
+ }
+ }
+ }
+
+ if let Some(tp) = child(&s, "text-properties") {
+ let mut props = parse_text_properties(&tp);
+ if let Some(v) = get_attr_local(&tp, "font-name") {
+ if v.to_ascii_lowercase().contains("courier") || v.to_ascii_lowercase().contains("mono") {
+ props.code = true;
+ }
+ out.text_font_name.insert(lname.clone(), v.to_string());
+ }
+ out.paragraph_text_props.insert(lname.clone(), props);
+ }
+ } else if family == "text" {
+ if let Some(tp) = child(&s, "text-properties") {
+ let mut props = parse_text_properties(&tp);
+ if let Some(v) = get_attr_local(&tp, "font-name") {
+ if v.to_ascii_lowercase().contains("courier") || v.to_ascii_lowercase().contains("mono") {
+ props.code = true;
+ }
+ out.text_font_name.insert(lname.clone(), v.to_string());
+ }
+ out.text_props.insert(lname.clone(), props);
+ }
+ } else if family == "list" {
+ let is_ordered = s
+ .children()
+ .filter(|n| n.is_element())
+ .any(|child_n| is_tag(&child_n, "list-level-style-number"));
+ out.list_is_ordered.insert(lname.clone(), is_ordered);
+ }
+ }
+
+ for ls in doc.descendants().filter(|n| is_tag(n, "list-style")) {
+ if let Some(name) = get_attr_local(&ls, "name") {
+ let is_ordered = ls
+ .children()
+ .filter(|n| n.is_element())
+ .any(|c| is_tag(&c, "list-level-style-number"));
+ out.list_is_ordered.insert(name.to_string(), is_ordered);
+ }
+ }
+}
+
+fn parse_text_properties(tp: &Node) -> TextStyleProps {
+ let mut props = TextStyleProps::default();
+
+ if let Some(v) = get_attr_local(tp, "font-weight") {
+ if v.eq_ignore_ascii_case("bold") {
+ props.bold = true;
+ }
+ }
+ if let Some(v) = get_attr_local(tp, "font-style") {
+ if v.eq_ignore_ascii_case("italic") {
+ props.italic = true;
+ }
+ }
+ if let Some(v) = get_attr_local(tp, "text-line-through-type")
+ .or_else(|| get_attr_local(tp, "text-line-through-style"))
+ {
+ if v != "none" {
+ props.strike = true;
+ }
+ }
+ if let Some(v) = get_attr_local(tp, "text-position") {
+ let lv = v.to_ascii_lowercase();
+ if lv.contains("sup") || lv.contains("super") {
+ props.sup = true;
+ } else if lv.contains("sub") {
+ props.sub = true;
+ }
+ }
+ props
+}
+
+fn read_meta<R: Read + Seek>(zip: &mut ZipArchive<R>) -> Option<DocumentMetadata> {
+ let text = read_zip_text(zip, "meta.xml")?;
+ let xml = XmlDoc::parse(strip_bom(&text)).ok()?;
+ let mut meta = DocumentMetadata::default();
+
+ if let Some(title) = xml
+ .descendants()
+ .find(|n| is_tag(n, "title"))
+ .and_then(|n| n.text())
+ {
+ if !title.trim().is_empty() {
+ meta.title = Some(title.to_string());
+ }
+ }
+
+ if let Some(author) = xml
+ .descendants()
+ .find(|n| is_tag(n, "creator"))
+ .and_then(|n| n.text())
+ .or_else(|| {
+ xml
+ .descendants()
+ .find(|n| is_tag(n, "initial-creator"))
+ .and_then(|n| n.text())
+ })
+ {
+ let trimmed = author.trim();
+ if !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("unknown") {
+ meta.author = Some(trimmed.to_string());
+ }
+ }
+
+ if let Some(created) = xml
+ .descendants()
+ .find(|n| is_tag(n, "creation-date"))
+ .and_then(|n| n.text())
+ {
+ if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(created) {
+ meta.created = Some(DateTime::<Utc>::from(dt));
+ }
+ }
+
+ Some(meta)
+}
+
+fn is_tag(node: &Node, local: &str) -> bool {
+ node.is_element() && node.tag_name().name() == local
+}
+
+fn get_attr_local<'a>(node: &Node<'a, 'a>, local: &str) -> Option<&'a str> {
+ node
+ .attributes()
+ .find(|a| {
+ let name = a.name();
+ match name.rsplit_once(':') {
+ Some((_, l)) => l == local,
+ None => name == local,
+ }
+ })
+ .map(|a| a.value())
+}
+
+fn child<'a>(node: &Node<'a, 'a>, local: &str) -> Option<Node<'a, 'a>> {
+ node
+ .children()
+ .find(|n| n.is_element() && n.tag_name().name() == local)
+}
+
+fn children<'a, 'b>(
+ node: &Node<'a, 'a>,
+ local: &'b str,
+) -> impl Iterator<Item = Node<'a, 'a>> + use<'a, 'b> {
+ node
+ .children()
+ .filter(move |n| n.is_element() && n.tag_name().name() == local)
+}
+
+fn parse_block_children_odt<R: Read + Seek>(
+ node: &Node,
+ styles: &OdtStylesInfo,
+ notes: &mut Vec<Note>,
+ comments: &mut Vec<Comment>,
+ zip: &mut ZipArchive<R>,
+) -> Vec<Block> {
+ let mut blocks: Vec<Block> = Vec::new();
+
+ for child_n in node.children().filter(|n| n.is_element()) {
+ if is_tag(&child_n, "h") {
+ if let Some(p) = parse_paragraph(&child_n, styles, notes, comments) {
+ if paragraph_has_visible_content(&p) {
+ blocks.push(Block::Paragraph(p));
+ }
+ }
+ } else if is_tag(&child_n, "p") {
+ if let Some(img) = image_from_paragraph(&child_n, zip) {
+ blocks.push(Block::Image(img));
+ } else if let Some(p) = parse_paragraph(&child_n, styles, notes, comments) {
+ if paragraph_has_visible_content(&p) {
+ blocks.push(Block::Paragraph(p));
+ }
+ }
+ } else if is_tag(&child_n, "list") {
+ let mut effective = child_n;
+ let mut inherited_style_name = get_attr_local(&effective, "style-name");
+ let mut unwrapped = false;
+
+ while let Some(inner) = unwrap_single_nested_list(&effective) {
+ effective = inner;
+ unwrapped = true;
+ if inherited_style_name.is_none() {
+ inherited_style_name = get_attr_local(&effective, "style-name");
+ }
+ }
+
+ if is_heading_list(&effective) {
+ for li in children(&effective, "list-item") {
+ if let Some(h) = li.descendants().find(|n| is_tag(n, "h")) {
+ if let Some(p) = parse_paragraph(&h, styles, notes, comments) {
+ if paragraph_has_visible_content(&p) {
+ blocks.push(Block::Paragraph(p));
+ }
+ }
+ }
+ }
+ } else if let Some(l) = parse_list_with_inherit(
+ &effective,
+ styles,
+ notes,
+ comments,
+ zip,
+ inherited_style_name,
+ ) {
+ if unwrapped {
+ if let Some(Block::List(prev)) = blocks.last_mut() {
+ if let Some(last_item) = prev.items.last_mut() {
+ last_item.blocks.push(Block::List(l));
+ continue;
+ }
+ }
+ }
+ blocks.push(Block::List(l));
+ }
+ } else if is_tag(&child_n, "table") {
+ if let Some(t) = parse_table(&child_n, styles, notes, comments, zip) {
+ blocks.push(Block::Table(t));
+ }
+ } else {
+ let mut inner = parse_block_children_odt(&child_n, styles, notes, comments, zip);
+ blocks.append(&mut inner);
+ }
+ }
+
+ blocks
+}
+
+fn unwrap_single_nested_list<'a>(list: &Node<'a, 'a>) -> Option<Node<'a, 'a>> {
+ let mut li_iter = children(list, "list-item");
+ let first_li = li_iter.next()?;
+ if li_iter.next().is_some() {
+ return None;
+ }
+ let mut inner_lists = first_li.children().filter(|n| is_tag(n, "list"));
+ let inner = inner_lists.next()?;
+ if inner_lists.next().is_some() {
+ return None;
+ }
+ Some(inner)
+}
+
+fn is_heading_list(list: &Node) -> bool {
+ let mut any = false;
+ for li in children(list, "list-item") {
+ any = true;
+ if !li.descendants().any(|n| is_tag(&n, "h")) {
+ return false;
+ }
+ }
+ any
+}
+
+fn parse_paragraph(
+ node: &Node,
+ styles: &OdtStylesInfo,
+ notes: &mut Vec<Note>,
+ comments: &mut Vec<Comment>,
+) -> Option<Paragraph> {
+ let kind = paragraph_kind(node, styles);
+ let base = paragraph_text_props(node, styles);
+ let inlines = parse_inlines_with_base(node, styles, notes, comments, base);
+ Some(Paragraph { kind, inlines })
+}
+
+fn paragraph_kind(p: &Node, styles: &OdtStylesInfo) -> ParagraphKind {
+ if p.tag_name().name() == "h" {
+ if let Some(ol) = get_attr_local(p, "outline-level") {
+ if let Ok(v) = ol.parse::<u8>() {
+ return ParagraphKind::Heading(v.min(6));
+ }
+ }
+ return ParagraphKind::Heading(1);
+ }
+
+ if let Some(style_name) = get_attr_local(p, "style-name") {
+ if let Some(lvl) = styles.paragraph_outline_level.get(style_name) {
+ return ParagraphKind::Heading((*lvl).min(6));
+ }
+
+ let name = styles
+ .paragraph_names
+ .get(style_name)
+ .map(|s| s.to_ascii_lowercase())
+ .unwrap_or_default();
+ if name.contains("quote") {
+ return ParagraphKind::Blockquote;
+ }
+ }
+
+ ParagraphKind::Normal
+}
+
+fn parse_odt_heading_level(style_name: &str) -> Option<u8> {
+ let normalized = style_name.replace("_20_", " ").replace('_', " ");
+ let lower = normalized.to_ascii_lowercase();
+ if lower.contains("title") {
+ return Some(1);
+ }
+
+ if let Some(idx) = lower.find("heading") {
+ let tail = &lower[idx + "heading".len()..];
+ let num: String = tail.chars().filter(|c| c.is_ascii_digit()).collect();
+ if let Ok(n) = num.parse::<u8>() {
+ return Some(n.clamp(1, 6));
+ }
+ }
+ None
+}
+
+fn paragraph_text_props(node: &Node, styles: &OdtStylesInfo) -> TextStyleProps {
+ if let Some(style_name) = get_attr_local(node, "style-name") {
+ if let Some(p) = styles.paragraph_text_props.get(style_name) {
+ return *p;
+ }
+ }
+ TextStyleProps::default()
+}
+
+fn parse_inlines(
+ node: &Node,
+ styles: &OdtStylesInfo,
+ notes: &mut Vec<Note>,
+ comments: &mut Vec<Comment>,
+) -> Vec<Inline> {
+ let mut out: Vec<Inline> = Vec::new();
+
+ for c in node.children() {
+ if c.is_text() {
+ if let Some(t) = c.text() {
+ if !t.is_empty() {
+ out.push(Inline::Text(t.to_string()));
+ }
+ }
+ continue;
+ }
+ if !c.is_element() {
+ continue;
+ }
+
+ if is_tag(&c, "span") {
+ let mut inner = parse_inlines(&c, styles, notes, comments);
+ let sname = get_attr_local(&c, "style-name").map(|s| s.to_string());
+ inner = apply_text_style_wrappers(inner, sname.as_deref(), styles, TextStyleProps::default());
+ out.extend(inner);
+ } else if is_tag(&c, "a") {
+ if let Some(href) = get_attr_local(&c, "href") {
+ let children = parse_inlines(&c, styles, notes, comments);
+ out.push(Inline::Link {
+ href: href.to_string(),
+ children,
+ });
+ } else {
+ out.extend(parse_inlines(&c, styles, notes, comments));
+ }
+ } else if is_tag(&c, "line-break") {
+ out.push(Inline::LineBreak);
+ } else if is_tag(&c, "s") {
+ let count = get_attr_local(&c, "c")
+ .and_then(|v| v.parse::<usize>().ok())
+ .unwrap_or(1);
+ out.push(Inline::Text(" ".repeat(count)));
+ } else if is_tag(&c, "tab") {
+ out.push(Inline::Text("\t".to_string()));
+ } else if is_tag(&c, "bookmark-start") {
+ if let Some(name) = get_attr_local(&c, "name") {
+ out.push(Inline::Bookmark(BookmarkId(name.to_string())));
+ }
+ } else if is_tag(&c, "note") {
+ let kind = match get_attr_local(&c, "note-class") {
+ Some("endnote") => NoteKind::Endnote,
+ _ => NoteKind::Footnote,
+ };
+ let id = get_attr_local(&c, "id")
+ .map(|s| s.to_string())
+ .unwrap_or_else(|| format!("odt-note-{}", notes.len() + 1));
+ let body = child(&c, "note-body");
+ let mut blocks: Vec<Block> = Vec::new();
+ if let Some(b) = body {
+ blocks = parse_note_body_blocks(&b, styles, notes, comments);
+ }
+ notes.push(Note {
+ id: NoteId(id.clone()),
+ kind,
+ blocks,
+ });
+ match kind {
+ NoteKind::Footnote => out.push(Inline::FootnoteRef(NoteId(id))),
+ NoteKind::Endnote => out.push(Inline::EndnoteRef(NoteId(id))),
+ }
+ } else if is_tag(&c, "annotation") {
+ let cid = format!("odt-comment-{}", comments.len() + 1);
+ let mut author: Option<String> = None;
+ let mut initials: Option<String> = None;
+
+ if let Some(a) = c
+ .descendants()
+ .find(|n| is_tag(n, "creator"))
+ .and_then(|n| n.text())
+ {
+ if !a.trim().is_empty() {
+ author = Some(a.to_string());
+ }
+ }
+ if let Some(init) = c
+ .descendants()
+ .find(|n| is_tag(n, "initials"))
+ .and_then(|n| n.text())
+ {
+ if !init.trim().is_empty() {
+ initials = Some(init.to_string());
+ }
+ }
+
+ let mut cblocks: Vec<Block> = Vec::new();
+ for p in c.children().filter(|n| is_tag(n, "p")) {
+ let inl = parse_inlines(&p, styles, notes, comments);
+ if !inl.is_empty() {
+ cblocks.push(Block::Paragraph(Paragraph {
+ kind: ParagraphKind::Normal,
+ inlines: inl,
+ }));
+ }
+ }
+ comments.push(Comment {
+ id: CommentId(cid.clone()),
+ author_name: author,
+ author_initials: initials,
+ blocks: cblocks,
+ });
+ out.push(Inline::CommentRef(CommentId(cid)));
+ } else {
+ out.extend(parse_inlines(&c, styles, notes, comments));
+ }
+ }
+
+ out
+}
+
+fn parse_inlines_with_base(
+ node: &Node,
+ styles: &OdtStylesInfo,
+ notes: &mut Vec<Note>,
+ comments: &mut Vec<Comment>,
+ base: TextStyleProps,
+) -> Vec<Inline> {
+ let mut inlines = parse_inlines(node, styles, notes, comments);
+ inlines = apply_text_style_wrappers(inlines, None, styles, base);
+ inlines
+}
+
+fn apply_text_style_wrappers(
+ mut inlines: Vec<Inline>,
+ style_name: Option<&str>,
+ styles: &OdtStylesInfo,
+ base: TextStyleProps,
+) -> Vec<Inline> {
+ let mut props = base;
+ if let Some(name) = style_name {
+ let lower = name.to_ascii_lowercase();
+ let mut sprops = styles.text_props.get(name).copied().unwrap_or_default();
+ if lower.contains("code")
+ || styles
+ .text_font_name
+ .get(name)
+ .map(|f| {
+ f.to_ascii_lowercase().contains("courier") || f.to_ascii_lowercase().contains("mono")
+ })
+ .unwrap_or(false)
+ {
+ sprops.code = true;
+ }
+ props.bold |= sprops.bold;
+ props.italic |= sprops.italic;
+ props.strike |= sprops.strike;
+ props.sup |= sprops.sup;
+ props.sub |= sprops.sub;
+ props.code |= sprops.code;
+ }
+
+ if props.code {
+ let code_text: String = inlines
+ .iter()
+ .filter_map(|i| match i {
+ Inline::Text(s) => Some(s.as_str()),
+ _ => None,
+ })
+ .collect();
+ if !code_text.is_empty() {
+ return vec![Inline::Code(code_text)];
+ }
+ }
+
+ if props.strike {
+ inlines = vec![Inline::Del(inlines)];
+ }
+ if props.italic {
+ inlines = vec![Inline::Em(inlines)];
+ }
+ if props.bold {
+ inlines = vec![Inline::Strong(inlines)];
+ }
+ if props.sup {
+ inlines = vec![Inline::Sup(inlines)];
+ }
+ if props.sub {
+ inlines = vec![Inline::Sub(inlines)];
+ }
+ inlines
+}
+
+fn parse_note_body_blocks(
+ node: &Node,
+ styles: &OdtStylesInfo,
+ notes: &mut Vec<Note>,
+ comments: &mut Vec<Comment>,
+) -> Vec<Block> {
+ let mut blocks = Vec::new();
+ for p in node.children().filter(|n| is_tag(n, "p") || is_tag(n, "h")) {
+ let kind = paragraph_kind(&p, styles);
+ let base = paragraph_text_props(&p, styles);
+ let inl = parse_inlines_with_base(&p, styles, notes, comments, base);
+ if inlines_have_visible_content(&inl) {
+ blocks.push(Block::Paragraph(Paragraph { kind, inlines: inl }));
+ }
+ }
+ blocks
+}
+
+fn paragraph_has_visible_content(p: &Paragraph) -> bool {
+ inlines_have_visible_content(&p.inlines)
+}
+
+fn inlines_have_visible_content(inlines: &[Inline]) -> bool {
+ inlines.iter().any(inline_is_visible)
+}
+
+fn inline_is_visible(i: &Inline) -> bool {
+ match i {
+ Inline::Text(t) => !t.trim().is_empty(),
+ Inline::LineBreak => false,
+ Inline::Link { children, .. } => inlines_have_visible_content(children),
+ Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {
+ inlines_have_visible_content(c)
+ }
+ Inline::Code(c) => !c.trim().is_empty(),
+ Inline::FootnoteRef(_) | Inline::EndnoteRef(_) | Inline::CommentRef(_) => true,
+ Inline::Bookmark(_) => false,
+ }
+}
+
+fn parse_list_with_inherit<R: Read + Seek>(
+ node: &Node,
+ styles: &OdtStylesInfo,
+ notes: &mut Vec<Note>,
+ comments: &mut Vec<Comment>,
+ zip: &mut ZipArchive<R>,
+ inherit_style_name: Option<&str>,
+) -> Option<List> {
+ let style_name = get_attr_local(node, "style-name").or(inherit_style_name);
+ let list_type = match style_name
+ .and_then(|n| styles.list_is_ordered.get(n))
+ .copied()
+ {
+ Some(true) => ListType::Ordered,
+ Some(false) => ListType::Unordered,
+ None => ListType::Unordered,
+ };
+
+ let mut items: Vec<ListItem> = Vec::new();
+ for it in children(node, "list-item") {
+ let mut blocks = Vec::new();
+ let mut inner = parse_block_children_odt(&it, styles, notes, comments, zip);
+ blocks.append(&mut inner);
+ items.push(ListItem { blocks });
+ }
+ Some(List { items, list_type })
+}
+
+fn parse_table<R: Read + Seek>(
+ node: &Node,
+ styles: &OdtStylesInfo,
+ notes: &mut Vec<Note>,
+ comments: &mut Vec<Comment>,
+ zip: &mut ZipArchive<R>,
+) -> Option<Table> {
+ let mut rows: Vec<TableRow> = Vec::new();
+ for tr in children(node, "table-row") {
+ let mut cells: Vec<TableCell> = Vec::new();
+ for tc in children(&tr, "table-cell") {
+ let mut blocks = parse_block_children_odt(&tc, styles, notes, comments, zip);
+ let colspan = get_attr_local(&tc, "number-columns-spanned")
+ .and_then(|v| v.parse::<u32>().ok())
+ .and_then(NonZeroU32::new)
+ .unwrap_or_else(|| NonZeroU32::new(1).unwrap());
+ let rowspan = get_attr_local(&tc, "number-rows-spanned")
+ .and_then(|v| v.parse::<u32>().ok())
+ .and_then(NonZeroU32::new)
+ .unwrap_or_else(|| NonZeroU32::new(1).unwrap());
+ cells.push(TableCell {
+ blocks: std::mem::take(&mut blocks),
+ colspan,
+ rowspan,
+ });
+ }
+ rows.push(TableRow {
+ cells,
+ kind: TableRowKind::Body,
+ });
+ }
+ Some(Table { rows })
+}
+
+fn image_from_paragraph<R: Read + Seek>(p: &Node, zip: &mut ZipArchive<R>) -> Option<Image> {
+ let img = p.descendants().find(|n| is_tag(n, "image"))?;
+ let href = get_attr_local(&img, "href")?;
+ image_from_href(href, zip, None)
+}
+
+fn image_from_href<R: Read + Seek>(
+ href: &str,
+ _zip: &mut ZipArchive<R>,
+ alt: Option<String>,
+) -> Option<Image> {
+ // only include external images (http/https URLs)
+ if href.starts_with("http://") || href.starts_with("https://") {
+ return Some(Image {
+ src: href.to_string(),
+ alt,
+ });
+ }
+ None
+}
diff --git a/apps/api/native/src/document/providers/rtf.rs b/apps/api/native/src/document/providers/rtf.rs
new file mode 100644
index 000000000..5a49d20be
--- /dev/null
+++ b/apps/api/native/src/document/providers/rtf.rs
@@ -0,0 +1,764 @@
+use crate::document::model::*;
+use crate::document::providers::DocumentProvider;
+use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc};
+use std::error::Error;
+use std::num::NonZeroU32;
+
+pub struct RtfProvider;
+
+impl RtfProvider {
+ pub fn new() -> Self {
+ Self
+ }
+}
+
+impl DocumentProvider for RtfProvider {
+ fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {
+ let metadata = extract_metadata_from_info(data).unwrap_or_default();
+ let blocks = parse_rtf_body_to_blocks(data);
+
+ Ok(Document {
+ blocks,
+ metadata,
+ notes: Vec::new(),
+ comments: Vec::new(),
+ })
+ }
+
+ fn name(&self) -> &'static str {
+ "rtf"
+ }
+}
+
+fn extract_metadata_from_info(src: &[u8]) -> Option<DocumentMetadata> {
+ let start = find_group_start(src, b"{\\info")?;
+ let end = find_matching_brace(src, start)?;
+ let info = &src[start..end];
+
+ let mut meta = DocumentMetadata::default();
+
+ if let Some(author) = extract_simple_text_dest(info, br"{\author") {
+ if !author.eq_ignore_ascii_case("unknown") {
+ meta.author = Some(author);
+ }
+ }
+
+ if let Some(title) = extract_simple_text_dest(info, br"{\title") {
+ if !title.trim().is_empty() {
+ meta.title = Some(title);
+ }
+ }
+
+ if let Some(created) = extract_creatim(info) {
+ meta.created = Some(created);
+ }
+
+ Some(meta)
+}
+
+fn find_group_start(buf: &[u8], needle: &[u8]) -> Option<usize> {
+ buf.windows(needle.len()).position(|w| w == needle)
+}
+
+fn find_matching_brace(buf: &[u8], start: usize) -> Option<usize> {
+ let mut depth = 0usize;
+ for (i, &b) in buf[start..].iter().enumerate() {
+ match b {
+ b'{' => depth += 1,
+ b'}' => {
+ depth -= 1;
+ if depth == 0 {
+ return Some(start + i + 1);
+ }
+ }
+ _ => {}
+ }
+ }
+ None
+}
+
+fn extract_simple_text_dest(buf: &[u8], start_tag: &[u8]) -> Option<String> {
+ let s = find_group_start(buf, start_tag)?;
+ let e = find_matching_brace(buf, s)?;
+ let mut out = String::new();
+ for &b in &buf[s + start_tag.len()..e - 1] {
+ push_byte_as_text(b, &mut out);
+ }
+ if out.trim().is_empty() {
+ None
+ } else {
+ Some(out.trim().to_string())
+ }
+}
+
+fn extract_creatim(buf: &[u8]) -> Option<DateTime<Utc>> {
+ let s = find_group_start(buf, br"{\creatim")?;
+ let e = find_matching_brace(buf, s)?;
+ let g = &buf[s..e];
+
+ let mut yr: Option<i32> = None;
+ let mut mo: Option<u32> = None;
+ let mut dy: Option<u32> = None;
+ let mut hr: Option<u32> = None;
+ let mut mi: Option<u32> = None;
+
+ let mut i = 0usize;
+ while i < g.len() {
+ if g[i] == b'\\' {
+ if let Some((word, val, ni)) = read_control_word(g, i + 1) {
+ match word.as_str() {
+ "yr" => yr = val,
+ "mo" => mo = val.map(|v| v as u32),
+ "dy" => dy = val.map(|v| v as u32),
+ "hr" => hr = val.map(|v| v as u32),
+ "min" => mi = val.map(|v| v as u32),
+ _ => {}
+ }
+ i = ni;
+ continue;
+ }
+ }
+ i += 1;
+ }
+
+ let date = NaiveDate::from_ymd_opt(yr?, mo?, dy?)?;
+ let time = chrono::NaiveTime::from_hms_opt(hr.unwrap_or(0), mi.unwrap_or(0), 0)?;
+ let dt = NaiveDateTime::new(date, time);
+ Some(DateTime::<Utc>::from_naive_utc_and_offset(dt, Utc))
+}
+
+#[derive(Default)]
+struct TableBuilder {
+ rows: Vec<TableRow>,
+ current_row: Vec<TableCell>,
+ current_cell_blocks: Vec<Block>,
+}
+
+impl TableBuilder {
+ fn start_row(&mut self) {
+ self.current_cell_blocks.clear();
+ self.current_row.clear();
+ }
+
+ fn push_block(&mut self, block: Block) {
+ self.current_cell_blocks.push(block);
+ }
+
+ fn finish_cell(&mut self) {
+ if self.current_cell_blocks.is_empty() {
+ return;
+ }
+ let cell = TableCell {
+ blocks: std::mem::take(&mut self.current_cell_blocks),
+ colspan: NonZeroU32::new(1).unwrap(),
+ rowspan: NonZeroU32::new(1).unwrap(),
+ };
+ self.current_row.push(cell);
+ }
+
+ fn finish_row(&mut self) {
+ self.finish_cell();
+ if self.current_row.is_empty() {
+ return;
+ }
+ let row = TableRow {
+ cells: std::mem::take(&mut self.current_row),
+ kind: TableRowKind::Body,
+ };
+ self.rows.push(row);
+ }
+
+ fn finalize(mut self) -> Option<Block> {
+ self.finish_row();
+ if self.rows.is_empty() {
+ None
+ } else {
+ Some(Block::Table(Table { rows: self.rows }))
+ }
+ }
+}
+
+fn push_block_target(
+ block: Block,
+ blocks: &mut Vec<Block>,
+ table: &mut Option<TableBuilder>,
+ in_table_cell: bool,
+) {
+ if in_table_cell {
+ if let Some(builder) = table.as_mut() {
+ builder.push_block(block);
+ } else {
+ blocks.push(block);
+ }
+ } else {
+ if let Some(builder) = table.take() {
+ if let Some(table_block) = builder.finalize() {
+ blocks.push(table_block);
+ }
+ }
+ blocks.push(block);
+ }
+}
+
+fn flush_table(blocks: &mut Vec<Block>, table: &mut Option<TableBuilder>) {
+ if let Some(builder) = table.take() {
+ if let Some(block) = builder.finalize() {
+ blocks.push(block);
+ }
+ }
+}
+
+fn parse_rtf_body_to_blocks(src: &[u8]) -> Vec<Block> {
+ let mut p = 0usize;
+ let n = src.len();
+
+ #[derive(Clone, Default, Debug, PartialEq, Eq)]
+ struct State {
+ bold: bool,
+ italic: bool,
+ strike: bool,
+ sup: bool,
+ sub: bool,
+ }
+
+ #[derive(Clone)]
+ struct Group {
+ saved: State,
+ skip: bool,
+ name_seen: bool,
+ }
+
+ let mut state = State::default();
+ let mut stack: Vec<Group> = Vec::new();
+ let mut blocks: Vec<Block> = Vec::new();
+ let mut cur_inlines: Vec<Inline> = Vec::new();
+ let mut text_buf = String::new();
+ let mut table_builder: Option<TableBuilder> = None;
+ let mut in_table_cell = false;
+ let mut uc_skip: usize = 1;
+ let mut pending_uc_skip: usize = 0;
+
+ const SKIP_DESTS: &[&str] = &[
+ "fonttbl",
+ "colortbl",
+ "stylesheet",
+ "listtable",
+ "listoverridetable",
+ "themedata",
+ "latentstyles",
+ "rsidtbl",
+ "xmlnstbl",
+ "mmathPr",
+ "wgrffmtfilter",
+ "datastore",
+ "filetbl",
+ "colorschememapping",
+ "pnseclvl1",
+ "pnseclvl2",
+ "pnseclvl3",
+ "pnseclvl4",
+ "pnseclvl5",
+ "pnseclvl6",
+ "pnseclvl7",
+ "pnseclvl8",
+ "pnseclvl9",
+ "pict",
+ "object",
+ "info",
+ ];
+
+ fn style_wrap(mut node: Inline, st: &State) -> Inline {
+ if st.strike {
+ node = Inline::Del(vec![node]);
+ }
+ if st.italic {
+ node = Inline::Em(vec![node]);
+ }
+ if st.bold {
+ node = Inline::Strong(vec![node]);
+ }
+ if st.sup {
+ node = Inline::Sup(vec![node]);
+ } else if st.sub {
+ node = Inline::Sub(vec![node]);
+ }
+ node
+ }
+
+ fn push_text_buf(text_buf: &mut String, cur: &mut Vec<Inline>, st: &State) {
+ if !text_buf.is_empty() {
+ let node = style_wrap(Inline::Text(text_buf.clone()), st);
+ cur.push(node);
+ text_buf.clear();
+ }
+ }
+
+ fn has_visible_content(inlines: &[Inline]) -> bool {
+ inlines.iter().any(|i| match i {
+ Inline::Text(t) => !t.trim().is_empty(),
+ Inline::LineBreak => false,
+ Inline::Link { children, .. } => has_visible_content(children),
+ Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {
+ has_visible_content(c)
+ }
+ Inline::Code(t) => !t.trim().is_empty(),
+ Inline::FootnoteRef(_) | Inline::EndnoteRef(_) | Inline::CommentRef(_) => true,
+ Inline::Bookmark(_) => false,
+ })
+ }
+
+ fn flush_paragraph(
+ cur: &mut Vec<Inline>,
+ text_buf: &mut String,
+ blocks: &mut Vec<Block>,
+ table: &mut Option<TableBuilder>,
+ st: &State,
+ in_table_cell: bool,
+ ) {
+ push_text_buf(text_buf, cur, st);
+ if has_visible_content(cur) {
+ let block = Block::Paragraph(Paragraph {
+ kind: ParagraphKind::Normal,
+ inlines: std::mem::take(cur),
+ });
+ push_block_target(block, blocks, table, in_table_cell);
+ } else {
+ cur.clear();
+ if !in_table_cell {
+ flush_table(blocks, table);
+ }
+ }
+ }
+
+ let flush_before_change = |text_buf: &mut String, cur: &mut Vec<Inline>, st: &State| {
+ push_text_buf(text_buf, cur, st);
+ };
+
+ while p < n {
+ match src[p] {
+ b'{' => {
+ let inherited_skip = stack.last().map(|g| g.skip).unwrap_or(false);
+ stack.push(Group {
+ saved: state.clone(),
+ skip: inherited_skip,
+ name_seen: false,
+ });
+ p += 1;
+ }
+ b'}' => {
+ if let Some(g) = stack.last() {
+ if !g.skip {
+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);
+ }
+ }
+ if let Some(g) = stack.pop() {
+ state = g.saved;
+ }
+ p += 1;
+ }
+ b'\\' => {
+ if p + 1 >= n {
+ break;
+ }
+ let next = src[p + 1];
+
+ if next == b'\\' || next == b'{' || next == b'}' {
+ if !stack.last().map(|g| g.skip).unwrap_or(false) {
+ text_buf.push(next as char);
+ }
+ p += 2;
+ continue;
+ }
+
+ if next == b'\'' {
+ if p + 3 < n {
+ let h1 = src[p + 2];
+ let h2 = src[p + 3];
+ if !stack.last().map(|g| g.skip).unwrap_or(false) {
+ if let (Some(a), Some(b)) = (hex_val(h1), hex_val(h2)) {
+ let byte = (a << 4) | b;
+ push_byte_as_text(byte, &mut text_buf);
+ }
+ }
+ p += 4;
+ continue;
+ } else {
+ break;
+ }
+ }
+
+ let skip = stack.last().map(|g| g.skip).unwrap_or(false);
+
+ match next {
+ b'~' => {
+ if !skip {
+ text_buf.push('\u{00A0}');
+ } // non-breaking space
+ p += 2;
+ continue;
+ }
+ b'-' => {
+ if !skip {
+ text_buf.push('\u{00AD}');
+ } // soft hyphen
+ p += 2;
+ continue;
+ }
+ _ => {}
+ }
+
+ if starts_with_word(src, p + 1, b"rquote") {
+ if !skip {
+ text_buf.push('\u{2019}');
+ } // '
+ p = skip_word_and_space(src, p + 1);
+ continue;
+ }
+ if starts_with_word(src, p + 1, b"lquote") {
+ if !skip {
+ text_buf.push('\u{2018}');
+ } // '
+ p = skip_word_and_space(src, p + 1);
+ continue;
+ }
+ if starts_with_word(src, p + 1, b"rdblquote") {
+ if !skip {
+ text_buf.push('\u{201D}');
+ } // "
+ p = skip_word_and_space(src, p + 1);
+ continue;
+ }
+ if starts_with_word(src, p + 1, b"ldblquote") {
+ if !skip {
+ text_buf.push('\u{201C}');
+ } // "
+ p = skip_word_and_space(src, p + 1);
+ continue;
+ }
+ if starts_with_word(src, p + 1, b"emdash") {
+ if !skip {
+ text_buf.push('\u{2014}');
+ } // —
+ p = skip_word_and_space(src, p + 1);
+ continue;
+ }
+ if starts_with_word(src, p + 1, b"endash") {
+ if !skip {
+ text_buf.push('\u{2013}');
+ } // –
+ p = skip_word_and_space(src, p + 1);
+ continue;
+ }
+ if starts_with_word(src, p + 1, b"bullet") {
+ if !skip {
+ text_buf.push('\u{2022}');
+ } // •
+ p = skip_word_and_space(src, p + 1);
+ continue;
+ }
+ if starts_with_word(src, p + 1, b"line") {
+ if !skip {
+ push_text_buf(&mut text_buf, &mut cur_inlines, &state);
+ cur_inlines.push(Inline::LineBreak);
+ }
+ p = skip_word_and_space(src, p + 1);
+ continue;
+ }
+ if starts_with_word(src, p + 1, b"tab") {
+ if !skip {
+ text_buf.push('\t');
+ }
+ p = skip_word_and_space(src, p + 1);
+ continue;
+ }
+
+ if let Some((word, val, new_p)) = read_control_word(src, p + 1) {
+ if let Some(g) = stack.last_mut() {
+ if !g.name_seen {
+ g.name_seen = true;
+ if word == "*" || SKIP_DESTS.contains(&word.as_str()) {
+ g.skip = true;
+ }
+ }
+ }
+
+ let skipping = stack.last().map(|g| g.skip).unwrap_or(false);
+
+ if !skipping {
+ match word.as_str() {
+ "trowd" => {
+ let builder = table_builder.get_or_insert_with(TableBuilder::default);
+ builder.start_row();
+ in_table_cell = false;
+ }
+ "intbl" => {
+ in_table_cell = true;
+ }
+ "cell" => {
+ flush_paragraph(
+ &mut cur_inlines,
+ &mut text_buf,
+ &mut blocks,
+ &mut table_builder,
+ &state,
+ true,
+ );
+ if let Some(builder) = table_builder.as_mut() {
+ builder.finish_cell();
+ }
+ in_table_cell = false;
+ }
+ "row" => {
+ if let Some(builder) = table_builder.as_mut() {
+ builder.finish_row();
+ }
+ in_table_cell = false;
+ }
+ "cellx" | "clvertalb" | "clvertalc" | "clvertalt" => {}
+ "b" => {
+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);
+ state.bold = val.map(|v| v != 0).unwrap_or(true);
+ }
+ "i" => {
+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);
+ state.italic = val.map(|v| v != 0).unwrap_or(true);
+ }
+ "strike" | "striked" | "striked1" => {
+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);
+ state.strike = val.map(|v| v != 0).unwrap_or(true);
+ }
+ "super" => {
+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);
+ state.sup = val.map(|v| v != 0).unwrap_or(true);
+ if state.sup {
+ state.sub = false;
+ }
+ }
+ "sub" => {
+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);
+ state.sub = val.map(|v| v != 0).unwrap_or(true);
+ if state.sub {
+ state.sup = false;
+ }
+ }
+ "nosupersub" => {
+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);
+ state.sup = false;
+ state.sub = false;
+ }
+ "plain" => {
+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);
+ state = State::default();
+ }
+ "par" => {
+ flush_paragraph(
+ &mut cur_inlines,
+ &mut text_buf,
+ &mut blocks,
+ &mut table_builder,
+ &state,
+ in_table_cell,
+ );
+ }
+ "uc" => {
+ uc_skip = val.unwrap_or(1).max(0) as usize;
+ }
+ "u" => {
+ if let Some(mut num) = val {
+ if num < 0 {
+ num += 65536;
+ }
+ if let Some(ch) = std::char::from_u32(num as u32) {
+ text_buf.push(ch);
+ }
+ pending_uc_skip = uc_skip;
+ }
+ }
+ _ => {}
+ }
+ } else if word == "par" {
+ flush_paragraph(
+ &mut cur_inlines,
+ &mut text_buf,
+ &mut blocks,
+ &mut table_builder,
+ &state,
+ in_table_cell,
+ );
+ }
+
+ let mut final_p = new_p;
+ if pending_uc_skip > 0 {
+ let mut k = 0usize;
+ while k < pending_uc_skip && final_p < n {
+ if matches!(src[final_p], b'\\' | b'{' | b'}') {
+ break;
+ }
+ final_p += 1;
+ k += 1;
+ }
+ pending_uc_skip = 0;
+ }
+ p = final_p;
+ continue;
+ }
+ p += 1;
+ }
+ b'\r' | b'\n' => {
+ p += 1;
+ }
+ byte => {
+ if !stack.last().map(|g| g.skip).unwrap_or(false) {
+ if pending_uc_skip > 0 {
+ pending_uc_skip -= 1;
+ } else {
+ push_byte_as_text(byte, &mut text_buf);
+ }
+ }
+ p += 1;
+ }
+ }
+ }
+
+ if !text_buf.is_empty() || !cur_inlines.is_empty() {
+ flush_paragraph(
+ &mut cur_inlines,
+ &mut text_buf,
+ &mut blocks,
+ &mut table_builder,
+ &state,
+ in_table_cell,
+ );
+ }
+
+ flush_table(&mut blocks, &mut table_builder);
+
+ blocks
+}
+
+fn read_control_word(src: &[u8], mut i: usize) -> Option<(String, Option<i32>, usize)> {
+ if i >= src.len() {
+ return None;
+ }
+
+ if src[i] == b'*' {
+ i += 1;
+ if i < src.len() && src[i] == b' ' {
+ i += 1;
+ }
+ return Some(("*".to_string(), None, i));
+ }
+
+ let start = i;
+ while i < src.len() && is_alpha(src[i]) {
+ i += 1;
+ }
+ if i == start {
+ return None;
+ }
+ let word = String::from_utf8_lossy(&src[start..i]).to_string();
+
+ let mut sign = 1i32;
+ let mut val: Option<i32> = None;
+ if i < src.len() && (src[i] == b'-' || is_digit(src[i])) {
+ if src[i] == b'-' {
+ sign = -1;
+ i += 1;
+ }
+ let num_start = i;
+ while i < src.len() && is_digit(src[i]) {
+ i += 1;
+ }
+ if i > num_start {
+ let n = std::str::from_utf8(&src[num_start..i])
+ .ok()?
+ .parse::<i32>()
+ .ok()?;
+ val = Some(sign * n);
+ }
+ }
+
+ if i < src.len() && src[i] == b' ' {
+ i += 1;
+ }
+
+ Some((word, val, i))
+}
+
+#[inline]
+fn is_alpha(b: u8) -> bool {
+ b.is_ascii_uppercase() || b.is_ascii_lowercase()
+}
+
+#[inline]
+fn is_digit(b: u8) -> bool {
+ b.is_ascii_digit()
+}
+
+fn hex_val(b: u8) -> Option<u8> {
+ match b {
+ b'0'..=b'9' => Some(b - b'0'),
+ b'a'..=b'f' => Some(10 + (b - b'a')),
+ b'A'..=b'F' => Some(10 + (b - b'A')),
+ _ => None,
+ }
+}
+
+fn push_byte_as_text(byte: u8, text_buf: &mut String) {
+ let ch = decode_cp1252(byte);
+ let cp = ch as u32;
+ if ch == '\t' || ch == '\u{00A0}' || cp >= 0x20 {
+ text_buf.push(ch);
+ }
+}
+
+fn decode_cp1252(b: u8) -> char {
+ if b < 0x80 {
+ return b as char;
+ }
+ match b {
+ 0x80 => '\u{20AC}', // €
+ 0x82 => '\u{201A}', // ‚
+ 0x83 => '\u{0192}', // Æ’
+ 0x84 => '\u{201E}', // „
+ 0x85 => '\u{2026}', // …
+ 0x86 => '\u{2020}', // †
+ 0x87 => '\u{2021}', // ‡
+ 0x88 => '\u{02C6}', // ˆ
+ 0x89 => '\u{2030}', // ‰
+ 0x8A => '\u{0160}', // Å 
+ 0x8B => '\u{2039}', // ‹
+ 0x8C => '\u{0152}', // Å’
+ 0x8E => '\u{017D}', // Ž
+ 0x91 => '\u{2018}', // '
+ 0x92 => '\u{2019}', // '
+ 0x93 => '\u{201C}', // "
+ 0x94 => '\u{201D}', // "
+ 0x95 => '\u{2022}', // •
+ 0x96 => '\u{2013}', // –
+ 0x97 => '\u{2014}', // —
+ 0x98 => '\u{02DC}', // ˜
+ 0x99 => '\u{2122}', // â„¢
+ 0x9A => '\u{0161}', // Å¡
+ 0x9B => '\u{203A}', // ›
+ 0x9C => '\u{0153}', // Å“
+ 0x9E => '\u{017E}', // ž
+ 0x9F => '\u{0178}', // Ÿ
+ _ => b as char,
+ }
+}
+
+fn starts_with_word(src: &[u8], i: usize, word: &[u8]) -> bool {
+ let end = i + word.len();
+ end <= src.len() && &src[i..end] == word
+}
+
+fn skip_word_and_space(src: &[u8], mut i: usize) -> usize {
+ while i < src.len() && is_alpha(src[i]) {
+ i += 1;
+ }
+ if i < src.len() && src[i] == b' ' {
+ i += 1;
+ }
+ i
+}
diff --git a/apps/api/native/src/document/renderers/html.rs b/apps/api/native/src/document/renderers/html.rs
new file mode 100644
index 000000000..ea6582400
--- /dev/null
+++ b/apps/api/native/src/document/renderers/html.rs
@@ -0,0 +1,237 @@
+use crate::document::model::*;
+use maud::{html, Markup, DOCTYPE};
+
+pub struct HtmlRenderer;
+
+impl HtmlRenderer {
+ pub fn new() -> Self {
+ Self
+ }
+
+ pub fn render(&self, document: &Document) -> String {
+ let title = document.metadata.title.as_deref().unwrap_or("Document");
+
+ let footnotes: Vec<&Note> = document
+ .notes
+ .iter()
+ .filter(|n| matches!(n.kind, NoteKind::Footnote))
+ .collect();
+
+ let endnotes: Vec<&Note> = document
+ .notes
+ .iter()
+ .filter(|n| matches!(n.kind, NoteKind::Endnote))
+ .collect();
+
+ let author = document.metadata.author.as_deref();
+ let page: Markup = html! {
+ (DOCTYPE)
+ html lang="en" {
+ head {
+ meta charset="UTF-8";
+ meta name="viewport" content="width=device-width, initial-scale=1.0";
+ title { (title) }
+ @if let Some(author) = author {
+ meta name="author" content=(author);
+ }
+ }
+ body {
+ main { (self.render_blocks(&document.blocks)) }
+
+ @if !footnotes.is_empty() {
+ section id="footnotes" {
+ h2 { "Footnotes" }
+ @for footnote in &footnotes {
+ div id={ "footnote-" (&footnote.id.0) } {
+ (self.render_blocks(&footnote.blocks))
+ }
+ }
+ }
+ }
+
+ @if !endnotes.is_empty() {
+ section id="endnotes" {
+ h2 { "Endnotes" }
+ @for endnote in &endnotes {
+ div id={ "endnote-" (&endnote.id.0) } {
+ (self.render_blocks(&endnote.blocks))
+ }
+ }
+ }
+ }
+
+ @if !document.comments.is_empty() {
+ section id="comments" {
+ h2 { "Comments" }
+ @for comment in &document.comments {
+ article id={ "comment-" (&comment.id.0) } {
+ @if let Some(author) = &comment.author_name {
+ header {
+ (author)
+ @if let Some(initials) = &comment.author_initials {
+ " (" (initials) ")"
+ }
+ }
+ }
+ (self.render_blocks(&comment.blocks))
+ }
+ }
+ }
+ }
+ }
+ }
+ };
+
+ page.into_string()
+ }
+
+ fn render_blocks(&self, blocks: &[Block]) -> Markup {
+ html! {
+ @for b in blocks {
+ @match b {
+ Block::Paragraph(p) => { (self.render_paragraph(p)) }
+ Block::Table(t) => { (self.render_table(t)) }
+ Block::List(l) => { (self.render_list(l)) }
+ Block::Image(i) => { (self.render_image(i)) }
+ }
+ }
+ }
+ }
+
+ fn render_blocks_inline(&self, blocks: &[Block]) -> Markup {
+ if blocks.len() == 1 {
+ if let Block::Paragraph(p) = &blocks[0] {
+ if matches!(p.kind, ParagraphKind::Normal) {
+ return self.render_inlines(&p.inlines);
+ }
+ }
+ }
+
+ self.render_blocks(blocks)
+ }
+
+ fn render_paragraph(&self, p: &Paragraph) -> Markup {
+ match p.kind {
+ ParagraphKind::Normal => html! { p { (self.render_inlines(&p.inlines)) } },
+ ParagraphKind::Blockquote => html! {
+ blockquote { p { (self.render_inlines(&p.inlines)) } }
+ },
+ ParagraphKind::Heading(level) => match level {
+ 1 => html! { h1 { (self.render_inlines(&p.inlines)) } },
+ 2 => html! { h2 { (self.render_inlines(&p.inlines)) } },
+ 3 => html! { h3 { (self.render_inlines(&p.inlines)) } },
+ 4 => html! { h4 { (self.render_inlines(&p.inlines)) } },
+ 5 => html! { h5 { (self.render_inlines(&p.inlines)) } },
+ _ => html! { h6 { (self.render_inlines(&p.inlines)) } },
+ },
+ }
+ }
+
+ fn render_table(&self, t: &Table) -> Markup {
+ let mut head_rows = Vec::new();
+ let mut body_rows = Vec::new();
+ let mut foot_rows = Vec::new();
+
+ for row in &t.rows {
+ match row.kind {
+ TableRowKind::Header => head_rows.push(row),
+ TableRowKind::Body => body_rows.push(row),
+ TableRowKind::Footer => foot_rows.push(row),
+ }
+ }
+
+ html! {
+ table {
+ @if !head_rows.is_empty() {
+ thead { @for row in head_rows { (self.render_table_row(row, true)) } }
+ }
+ tbody { @for row in body_rows { (self.render_table_row(row, false)) } }
+ @if !foot_rows.is_empty() {
+ tfoot { @for row in foot_rows { (self.render_table_row(row, false)) } }
+ }
+ }
+ }
+ }
+
+ fn render_table_row(&self, row: &TableRow, header: bool) -> Markup {
+ html! {
+ tr {
+ @for cell in &row.cells {
+ @let cs = cell.colspan.get();
+ @let rs = cell.rowspan.get();
+ @let cs_attr = if cs > 1 { Some(cs) } else { None };
+ @let rs_attr = if rs > 1 { Some(rs) } else { None };
+
+ @if header {
+ @if let (Some(cs), Some(rs)) = (cs_attr, rs_attr) {
+ th colspan=(cs) rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }
+ } @else if let Some(cs) = cs_attr {
+ th colspan=(cs) { (self.render_blocks_inline(&cell.blocks)) }
+ } @else if let Some(rs) = rs_attr {
+ th rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }
+ } @else {
+ th { (self.render_blocks_inline(&cell.blocks)) }
+ }
+ } @else {
+ @if let (Some(cs), Some(rs)) = (cs_attr, rs_attr) {
+ td colspan=(cs) rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }
+ } @else if let Some(cs) = cs_attr {
+ td colspan=(cs) { (self.render_blocks_inline(&cell.blocks)) }
+ } @else if let Some(rs) = rs_attr {
+ td rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }
+ } @else {
+ td { (self.render_blocks_inline(&cell.blocks)) }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ fn render_list(&self, l: &List) -> Markup {
+ match l.list_type {
+ ListType::Ordered => html! {
+ ol { @for item in &l.items { li { (self.render_blocks_inline(&item.blocks)) } } }
+ },
+ ListType::Unordered => html! {
+ ul { @for item in &l.items { li { (self.render_blocks_inline(&item.blocks)) } } }
+ },
+ }
+ }
+
+ fn render_image(&self, i: &Image) -> Markup {
+ match &i.alt {
+ Some(alt) => html! { img src=(i.src) alt=(alt); },
+ None => html! { img src=(i.src); },
+ }
+ }
+
+ fn render_inlines(&self, inlines: &[Inline]) -> Markup {
+ html! { @for inline in inlines { (self.render_inline(inline)) } }
+ }
+
+ fn render_inline(&self, inline: &Inline) -> Markup {
+ match inline {
+ Inline::Text(t) => html! { (t) },
+ Inline::LineBreak => html! { br; },
+
+ Inline::Link { href, children } => {
+ html! { a href=(href) { (self.render_inlines(children)) } }
+ }
+
+ Inline::Strong(children) => html! { strong { (self.render_inlines(children)) } },
+ Inline::Em(children) => html! { em { (self.render_inlines(children)) } },
+ Inline::Del(children) => html! { del { (self.render_inlines(children)) } },
+ Inline::Code(code) => html! { code { (code) } },
+ Inline::Sup(children) => html! { sup { (self.render_inlines(children)) } },
+ Inline::Sub(children) => html! { sub { (self.render_inlines(children)) } },
+
+ Inline::FootnoteRef(id) => {
+ html! { sup { a href={ "#footnote-" (&id.0) } { (&id.0) } } }
+ }
+ Inline::EndnoteRef(id) => html! { sup { a href={ "#endnote-" (&id.0) } { (&id.0) } } },
+ Inline::CommentRef(id) => html! { a href={ "#comment-" (&id.0) } { "💬" } },
+ Inline::Bookmark(id) => html! { a id=(&id.0) {} },
+ }
+ }
+}
diff --git a/apps/api/native/src/document/renderers/mod.rs b/apps/api/native/src/document/renderers/mod.rs
new file mode 100644
index 000000000..3d4613dd4
--- /dev/null
+++ b/apps/api/native/src/document/renderers/mod.rs
@@ -0,0 +1 @@
+pub mod html;
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rs
index 3d2372c36..d028ae78f 100644
--- a/apps/api/native/src/html.rs
+++ b/apps/api/native/src/html.rs
@@ -216,7 +216,7 @@ fn _extract_metadata(html: &str) -> Result<HashMap<String, Value>, Box<dyn std::
if name == "description" {
out.insert(
name.to_string(),
- Value::String(format!("{}, {}", existing, content)),
+ Value::String(format!("{existing}, {content}")),
);
} else if name != "title" {
out.insert(
@@ -402,7 +402,7 @@ fn _transform_html_inner(opts: TransformHtmlOptions) -> Result<String, Box<dyn s
.collect::<HashSet<_>>();
for mode in modes {
- let matcher = format!(":{}:", Into::<String>::into(mode.clone()));
+ let matcher = format!(":{}:", Into::<String>::into(mode));
let signatures = signatures
.iter()
.filter(|x| x.contains(&matcher))
@@ -420,7 +420,7 @@ fn _transform_html_inner(opts: TransformHtmlOptions) -> Result<String, Box<dyn s
continue;
}
- let signature = get_node_signature(&node, mode.clone());
+ let signature = get_node_signature(&node, mode);
if signatures.contains(&signature) {
nodes_to_drop.push(node);
}
@@ -707,7 +707,7 @@ fn _extract_images(html: &str, base_url: &str) -> Result<Vec<String>, Box<dyn st
if let Some(srcset) = attrs.get("srcset") {
for part in srcset.split(',') {
- if let Some(url) = part.trim().split_whitespace().next() {
+ if let Some(url) = part.split_whitespace().next() {
if !url.is_empty() {
if let Ok(resolved) = resolve_image_url(url) {
images.insert(resolved);
@@ -730,7 +730,7 @@ fn _extract_images(html: &str, base_url: &str) -> Result<Vec<String>, Box<dyn st
for source in source_elements {
if let Some(srcset) = source.attributes.borrow().get("srcset") {
for part in srcset.split(',') {
- if let Some(url) = part.trim().split_whitespace().next() {
+ if let Some(url) = part.split_whitespace().next() {
if !url.is_empty() {
if let Ok(resolved) = resolve_image_url(url) {
images.insert(resolved);
diff --git a/apps/api/native/src/lib.rs b/apps/api/native/src/lib.rs
index 06cfebea6..f5f5c48e7 100644
--- a/apps/api/native/src/lib.rs
+++ b/apps/api/native/src/lib.rs
@@ -5,7 +5,10 @@ pub use crate::html::*;
pub use crate::pdf::*;
pub use crate::utils::*;
+pub use crate::document::{DocumentConverter, DocumentType};
+
mod crawler;
+mod document;
mod html;
mod pdf;
mod utils;
diff --git a/apps/api/native/src/pdf.rs b/apps/api/native/src/pdf.rs
index e5ae2b278..67f64de42 100644
--- a/apps/api/native/src/pdf.rs
+++ b/apps/api/native/src/pdf.rs
@@ -27,7 +27,7 @@ fn _get_pdf_metadata(path: &str) -> std::result::Result<PDFMetadata, String> {
info
.as_dict()
.and_then(|info| info.get(b"Title"))
- .and_then(|title| lopdf::decode_text_string(title))
+ .and_then(lopdf::decode_text_string)
})
.ok()
.or_else(|| {
@@ -35,7 +35,7 @@ fn _get_pdf_metadata(path: &str) -> std::result::Result<PDFMetadata, String> {
obj
.as_dict()
.and_then(|obj| obj.get(b"Title"))
- .and_then(|title| lopdf::decode_text_string(title))
+ .and_then(lopdf::decode_text_string)
.ok()
})
})
@@ -50,7 +50,7 @@ pub fn get_pdf_metadata(path: String) -> Result<PDFMetadata> {
_get_pdf_metadata(&path).map_err(|e| {
Error::new(
Status::GenericFailure,
- format!("Failed to get PDF metadata: {}", e),
+ format!("Failed to get PDF metadata: {e}"),
)
})
}
diff --git a/apps/api/samples/sample.docx b/apps/api/samples/sample.docx
new file mode 100644
index 000000000..de7b10e5e
Binary files /dev/null and b/apps/api/samples/sample.docx differ
diff --git a/apps/api/samples/sample.odt b/apps/api/samples/sample.odt
new file mode 100644
index 000000000..86c8281ee
Binary files /dev/null and b/apps/api/samples/sample.odt differ
diff --git a/apps/api/samples/sample.rtf b/apps/api/samples/sample.rtf
new file mode 100644
index 000000000..4d0d9b35d
--- /dev/null
+++ b/apps/api/samples/sample.rtf
@@ -0,0 +1,35 @@
+{\rtf1\ansi\deff3\adeflang1025
+{\fonttbl{\f0\froman\fprq2\fcharset0 Times New Roman;}{\f1\froman\fprq2\fcharset2 Symbol;}{\f2\fswiss\fprq2\fcharset0 Arial;}{\f3\froman\fprq2\fcharset0 Liberation Serif{\*\falt Times New Roman};}{\f4\froman\fprq2\fcharset0 Liberation Sans{\*\falt Arial};}{\f5\fnil\fprq2\fcharset0 0;}{\f6\fnil\fprq2\fcharset0 Microsoft YaHei;}{\f7\fnil\fprq2\fcharset0 Arial;}}
+{\colortbl;\red0\green0\blue0;\red0\green0\blue255;\red0\green255\blue255;\red0\green255\blue0;\red255\green0\blue255;\red255\green0\blue0;\red255\green255\blue0;\red255\green255\blue255;\red0\green0\blue128;\red0\green128\blue128;\red0\green128\blue0;\red128\green0\blue128;\red128\green0\blue0;\red128\green128\blue0;\red128\green128\blue128;\red192\green192\blue192;}
+{\stylesheet{\s0\snext0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052 Normal;}
+{\s15\sbasedon0\snext16\rtlch\af7\afs28 \ltrch\hich\af4\loch\sb240\sa120\keepn\f4\fs28\dbch\af6 Heading;}
+{\s16\sbasedon0\snext16\loch\sl276\slmult1\sb0\sa140 Body Text;}
+{\s17\sbasedon16\snext17\rtlch\af7 \ltrch\loch\sl240\slmult1\sb0\sa0 List;}
+{\s18\sbasedon0\snext18\rtlch\af7\afs24\ai \ltrch\loch\sb120\sa120\fs24\i caption;}
+{\s19\sbasedon0\snext19\rtlch\af7 \ltrch Index;}
+{\s20\sbasedon0\snext20\loch\nowidctlpar Table Contents;}
+}{\*\generator LibreOffice/25.2.5.2$Windows_X86_64 LibreOffice_project/03d19516eb2e1dd5d4ccd751a0d6f35f35e08022}{\info{\creatim\yr2025\mo9\dy16\hr16\min7}{\revtim\yr2025\mo9\dy16\hr19\min39}{\printim\yr0\mo0\dy0\hr0\min0}}{\*\userprops}\deftab709
+\hyphauto1\viewscale100\formshade\nobrkwrptbl\paperh16838\paperw11906\margl1134\margr1134\margt1134\margb1134\sectd\sbknone\sftnnar\saftnnrlc\sectunlocked1\pgwsxn11906\pghsxn16838\marglsxn1134\margrsxn1134\margtsxn1134\margbsxn1134\ftnbj\ftnstart1\ftnrstcont\ftnnar\fet\aftnrstcont\aftnstart1\aftnnrlc
+{\*\ftnsep\chftnsep}\pgndec\pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql{\rtlch\afs36\ab \ltrch\loch\fs36\b\loch
+Hello!}
+\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql{\rtlch\afs28\ab0 \ltrch\loch\fs28\b0\loch
+RTF file to test the Firecrawl Document Converter.}
+\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql\rtlch\afs28\ab0 \ltrch\loch\fs28\b0\loch
+
+\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql{\rtlch\afs28\ai\ab0 \ltrch\loch\fs28\i\b0\loch
+Italic}
+\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql{\rtlch\afs28\ai0\ab \ltrch\loch\fs28\i0\b\loch
+Bold}
+\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql{\rtlch\afs28\ai0\ab0 \ltrch\loch\fs28\i0\ul\ulc0\b0\loch
+Underlined}
+\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql{\rtlch\afs28\ai0\ab0 \ltrch\loch\strike\fs28\i0\ulnone\ulc0\b0\loch
+Strikethrough}
+\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql\rtlch\afs28\ai0\ab0 \ltrch\loch\strike\fs28\i0\ulnone\ulc0\b0\loch
+
+\par \trowd\trql\trleft10\ltrrow\trpaddft3\trpaddt0\trpaddfl3\trpaddl0\trpaddfb3\trpaddb0\trpaddfr3\trpaddr0\clbrdrt\brdrs\brdrw10\brdrcf1\clpadfl3\clpadl55\clbrdrl\brdrs\brdrw10\brdrcf1\clpadft3\clpadt55\clbrdrb\brdrs\brdrw10\brdrcf1\clpadfb3\clpadb55\clpadfr3\clpadr55\cellx4829\clbrdrt\brdrs\brdrw10\brdrcf1\clpadfl3\clpadl55\clbrdrl\brdrs\brdrw10\brdrcf1\clpadft3\clpadt55\clbrdrb\brdrs\brdrw10\brdrcf1\clpadfb3\clpadb55\clbrdrr\brdrs\brdrw10\brdrcf1\clpadfr3\clpadr55\cellx9646\pard\plain \s20\loch\nowidctlpar\intbl\ql\ltrpar{\rtlch\afs28 \ltrch\loch\fs28\loch
+Header 1}\cell\pard\plain \s20\loch\nowidctlpar\intbl\ql\ltrpar{\rtlch\afs28 \ltrch\loch\fs28\loch
+Header 2}\cell\row\pard \trowd\trql\trleft10\ltrrow\trpaddft3\trpaddt0\trpaddfl3\trpaddl0\trpaddfb3\trpaddb0\trpaddfr3\trpaddr0\clpadfl3\clpadl55\clbrdrl\brdrs\brdrw10\brdrcf1\clpadft3\clpadt55\clbrdrb\brdrs\brdrw10\brdrcf1\clpadfb3\clpadb55\clpadfr3\clpadr55\cellx4829\clpadfl3\clpadl55\clbrdrl\brdrs\brdrw10\brdrcf1\clpadft3\clpadt55\clbrdrb\brdrs\brdrw10\brdrcf1\clpadfb3\clpadb55\clbrdrr\brdrs\brdrw10\brdrcf1\clpadfr3\clpadr55\cellx9646\pard\plain \s20\loch\nowidctlpar\intbl\ql\ltrpar{\rtlch\afs28 \ltrch\loch\fs28\loch
+Value 1}\cell\pard\plain \s20\loch\nowidctlpar\intbl\ql\ltrpar{\rtlch\afs28 \ltrch\loch\fs28\loch
+Value 2}\cell\row\pard \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql\rtlch\afs28\ai0\ab0 \ltrch\loch\strike0\fs28\i0\ulnone\ulc0\b0\loch
+
+\par }
\ No newline at end of file
diff --git a/apps/api/src/__tests__/snips/v2/document-converter.test.ts b/apps/api/src/__tests__/snips/v2/document-converter.test.ts
new file mode 100644
index 000000000..c8d0d6d15
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/document-converter.test.ts
@@ -0,0 +1,46 @@
+import path from "path";
+import fs from "fs";
+import { DocumentConverter, DocumentType } from "@mendable/firecrawl-rs";
+
+describe("Document Converter tests", () => {
+ const samplesDir = path.join(process.cwd(), "samples");
+
+ const expectedHtmlBase = (documentText: string) =>
+ `<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Document</title></head><body><main><p><strong>Hello!</strong></p><p>${documentText} file to test the Firecrawl Document Converter.</p><p><em>Italic</em></p><p><strong>Bold</strong></p><p>Underlined</p><p><del>Strikethrough</del></p><table><tbody><tr><td>Header 1</td><td>Header 2</td></tr><tr><td>Value 1</td><td>Value 2</td></tr></tbody></table></main></body></html>`;
+
+ const sampleFiles = [
+ { file: "sample.docx", type: DocumentType.Docx, name: "DOCX" },
+ { file: "sample.odt", type: DocumentType.Odt, name: "ODT" },
+ { file: "sample.rtf", type: DocumentType.Rtf, name: "RTF" },
+ ];
+
+ const converter = new DocumentConverter();
+
+ describe.each(sampleFiles)(
+ "$name document conversion",
+ ({ file, type, name }) => {
+ const filePath = path.join(samplesDir, file);
+
+ beforeAll(() => {
+ if (!fs.existsSync(filePath)) {
+ throw new Error(`Sample file ${filePath} does not exist`);
+ }
+ });
+
+ it.concurrent(
+ `should convert ${name} document and return expected HTML`,
+ async () => {
+ const fileBuffer = fs.readFileSync(filePath);
+
+ const html = await converter.convertBufferToHtml(
+ new Uint8Array(fileBuffer),
+ type,
+ );
+
+ expect(html).toBe(expectedHtmlBase(name));
+ },
+ 10000,
+ );
+ },
+ );
+});
diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts
index f89eefc33..4abb8cf9b 100644
--- a/apps/api/src/lib/extract/extraction-service.ts
+++ b/apps/api/src/lib/extract/extraction-service.ts
@@ -844,7 +844,8 @@ export async function performExtraction(
if (docsMap.size == 0) {
// All urls are invalid
- const errorMessage = "All provided URLs are either invalid, unsupported or failed to be scraped.";
+ const errorMessage =
+ "All provided URLs are either invalid, unsupported or failed to be scraped.";
logger.error(errorMessage);
const tokens_billed = 300 + calculateThinkingCost(costTracking);
await billTeam(
@@ -976,11 +977,11 @@ export async function performExtraction(
let finalResult = reqSchema
? await mixSchemaObjects(
- reqSchema,
- singleAnswerResult,
- multiEntityResult,
- logger.child({ method: "mixSchemaObjects" }),
- )
+ reqSchema,
+ singleAnswerResult,
+ multiEntityResult,
+ logger.child({ method: "mixSchemaObjects" }),
+ )
: singleAnswerResult || multiEntityResult;
// Tokenize final result to get token count
diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
new file mode 100644
index 000000000..649b520c0
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
@@ -0,0 +1,72 @@
+import { Meta } from "../..";
+import { EngineScrapeResult } from "..";
+import { fetchFileToBuffer } from "../utils/downloadFile";
+import { DocumentConverter, DocumentType } from "@mendable/firecrawl-rs";
+
+const converter = new DocumentConverter();
+
+function getDocumentTypeFromUrl(url: string): DocumentType {
+ const urlLower = url.toLowerCase();
+ if (urlLower.endsWith(".docx")) return DocumentType.Docx;
+ if (urlLower.endsWith(".odt")) return DocumentType.Odt;
+ if (urlLower.endsWith(".rtf")) return DocumentType.Rtf;
+
+ return DocumentType.Docx; // hope for the best
+}
+
+function getDocumentTypeFromContentType(
+ contentType: string | null,
+): DocumentType | null {
+ if (!contentType) return null;
+
+ const ct = contentType.toLowerCase();
+
+ if (
+ ct.includes(
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ ) ||
+ ct.includes("application/msword")
+ ) {
+ return DocumentType.Docx;
+ }
+
+ if (ct.includes("application/vnd.oasis.opendocument.text")) {
+ return DocumentType.Odt;
+ }
+
+ if (ct.includes("application/rtf") || ct.includes("text/rtf")) {
+ return DocumentType.Rtf;
+ }
+
+ return null;
+}
+
+export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
+ const { response, buffer } = await fetchFileToBuffer(
+ meta.rewrittenUrl ?? meta.url,
+ {
+ headers: meta.options.headers,
+ signal: meta.abort.asSignal(),
+ },
+ );
+
+ const documentType =
+ getDocumentTypeFromContentType(response.headers.get("content-type")) ??
+ getDocumentTypeFromUrl(response.url);
+
+ const html = await converter.convertBufferToHtml(
+ new Uint8Array(buffer),
+ documentType,
+ );
+
+ return {
+ url: response.url,
+ statusCode: response.status,
+ html,
+ proxyUsed: "basic",
+ };
+}
+
+export function documentMaxReasonableTime(meta: Meta): number {
+ return 15000;
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts b/apps/api/src/scraper/scrapeURL/engines/docx/index.ts
deleted file mode 100644
index bfb199182..000000000
--- a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-import { Meta } from "../..";
-import { EngineScrapeResult } from "..";
-import { downloadFile } from "../utils/downloadFile";
-import mammoth from "mammoth";
-
-export async function scrapeDOCX(meta: Meta): Promise<EngineScrapeResult> {
- const { response, tempFilePath } = await downloadFile(
- meta.id,
- meta.rewrittenUrl ?? meta.url,
- {
- headers: meta.options.headers,
- signal: meta.abort.asSignal(),
- },
- );
-
- return {
- url: response.url,
- statusCode: response.status,
-
- html: (await mammoth.convertToHtml({ path: tempFilePath })).value,
-
- proxyUsed: "basic",
- };
-}
-
-export function docxMaxReasonableTime(meta: Meta): number {
- return 15000;
-}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 0b26bc5ca..309a7c2aa 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -1,6 +1,6 @@
import { ScrapeActionContent } from "../../../lib/entities";
import { Meta } from "..";
-import { docxMaxReasonableTime, scrapeDOCX } from "./docx";
+import { documentMaxReasonableTime, scrapeDocument } from "./document";
import {
fireEngineMaxReasonableTime,
scrapeURLWithFireEngineChromeCDP,
@@ -31,7 +31,7 @@ export type Engine =
| "playwright"
| "fetch"
| "pdf"
- | "docx"
+ | "document"
| "index"
| "index;documents";
@@ -59,7 +59,7 @@ const engines: Engine[] = [
...(usePlaywright ? ["playwright" as const] : []),
"fetch",
"pdf",
- "docx",
+ "document",
];
const featureFlags = [
@@ -68,7 +68,7 @@ const featureFlags = [
"screenshot",
"screenshot@fullScreen",
"pdf",
- "docx",
+ "document",
"atsv",
"location",
"mobile",
@@ -90,7 +90,7 @@ const featureFlagOptions: {
screenshot: { priority: 10 },
"screenshot@fullScreen": { priority: 10 },
pdf: { priority: 100 },
- docx: { priority: 100 },
+ document: { priority: 100 },
atsv: { priority: 90 }, // NOTE: should atsv force to tlsclient? adjust priority if not
useFastMode: { priority: 90 },
location: { priority: 10 },
@@ -149,7 +149,7 @@ const engineHandlers: {
playwright: scrapeURLWithPlaywright,
fetch: scrapeURLWithFetch,
pdf: scrapePDF,
- docx: scrapeDOCX,
+ document: scrapeDocument,
};
const engineMRTs: {
@@ -176,7 +176,7 @@ const engineMRTs: {
playwright: playwrightMaxReasonableTime,
fetch: fetchMaxReasonableTime,
pdf: pdfMaxReasonableTime,
- docx: docxMaxReasonableTime,
+ document: documentMaxReasonableTime,
};
const engineOptions: {
@@ -196,7 +196,7 @@ const engineOptions: {
screenshot: true,
"screenshot@fullScreen": true,
pdf: false,
- docx: false,
+ document: false,
atsv: false,
mobile: true,
location: true,
@@ -214,7 +214,7 @@ const engineOptions: {
screenshot: true, // through actions transform
"screenshot@fullScreen": true, // through actions transform
pdf: false,
- docx: false,
+ document: false,
atsv: false,
location: true,
mobile: true,
@@ -232,7 +232,7 @@ const engineOptions: {
screenshot: true, // through actions transform
"screenshot@fullScreen": true, // through actions transform
pdf: false,
- docx: false,
+ document: false,
atsv: false,
location: true,
mobile: true,
@@ -250,7 +250,7 @@ const engineOptions: {
screenshot: true,
"screenshot@fullScreen": true,
pdf: true,
- docx: true,
+ document: true,
atsv: false,
location: true,
mobile: true,
@@ -268,7 +268,7 @@ const engineOptions: {
screenshot: true, // through actions transform
"screenshot@fullScreen": true, // through actions transform
pdf: false,
- docx: false,
+ document: false,
atsv: false,
location: true,
mobile: true,
@@ -286,7 +286,7 @@ const engineOptions: {
screenshot: true, // through actions transform
"screenshot@fullScreen": true, // through actions transform
pdf: false,
- docx: false,
+ document: false,
atsv: false,
location: true,
mobile: true,
@@ -304,7 +304,7 @@ const engineOptions: {
screenshot: true,
"screenshot@fullScreen": true,
pdf: false,
- docx: false,
+ document: false,
atsv: false,
location: false,
mobile: false,
@@ -322,7 +322,7 @@ const engineOptions: {
screenshot: true,
"screenshot@fullScreen": true,
pdf: false,
- docx: false,
+ document: false,
atsv: false,
location: false,
mobile: false,
@@ -340,7 +340,7 @@ const engineOptions: {
screenshot: false,
"screenshot@fullScreen": false,
pdf: false,
- docx: false,
+ document: false,
atsv: false,
location: false,
mobile: false,
@@ -358,7 +358,7 @@ const engineOptions: {
screenshot: false,
"screenshot@fullScreen": false,
pdf: false,
- docx: false,
+ document: false,
atsv: true,
location: true,
mobile: false,
@@ -376,7 +376,7 @@ const engineOptions: {
screenshot: false,
"screenshot@fullScreen": false,
pdf: false,
- docx: false,
+ document: false,
atsv: true,
location: true,
mobile: false,
@@ -394,7 +394,7 @@ const engineOptions: {
screenshot: false,
"screenshot@fullScreen": false,
pdf: false,
- docx: false,
+ document: false,
atsv: false,
location: false,
mobile: false,
@@ -412,7 +412,7 @@ const engineOptions: {
screenshot: false,
"screenshot@fullScreen": false,
pdf: true,
- docx: false,
+ document: false,
atsv: false,
location: false,
mobile: false,
@@ -423,14 +423,14 @@ const engineOptions: {
},
quality: -20,
},
- docx: {
+ document: {
features: {
actions: false,
waitFor: false,
screenshot: false,
"screenshot@fullScreen": false,
pdf: false,
- docx: true,
+ document: true,
atsv: false,
location: false,
mobile: false,
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
index 0720f3c55..6a85c6af9 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
@@ -38,27 +38,31 @@ export async function specialtyScrapeCheck(
x => x[0].toLowerCase() === "content-type",
) ?? [])[1];
- if (contentType === undefined) {
+ if (!contentType) {
logger.warn("Failed to check contentType -- was not present in headers", {
headers,
});
- } else if (
+ return;
+ }
+
+ if (
contentType === "application/pdf" ||
contentType.startsWith("application/pdf;") ||
(contentType === "application/octet-stream" &&
(feRes?.file?.content.startsWith("JVBERi0") ||
feRes?.content.startsWith("%PDF-")))
) {
- // .pdf
throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));
- } else if (
- contentType ===
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ||
- contentType.startsWith(
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document;",
- )
- ) {
- // .docx
- throw new AddFeatureError(["docx"]);
+ }
+
+ const documentTypes = [
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ "application/rtf",
+ "text/rtf",
+ "application/vnd.oasis.opendocument.text",
+ ];
+
+ if (documentTypes.some(type => contentType.startsWith(type))) {
+ throw new AddFeatureError(["document"]);
}
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index d44f60f9a..e733c1d74 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -149,13 +149,18 @@ function buildFeatureFlags(
}
const urlO = new URL(url);
+ const lowerPath = urlO.pathname.toLowerCase();
- if (urlO.pathname.endsWith(".pdf")) {
+ if (lowerPath.endsWith(".pdf")) {
flags.add("pdf");
}
- if (urlO.pathname.endsWith(".docx")) {
- flags.add("docx");
+ if (
+ lowerPath.endsWith(".docx") ||
+ lowerPath.endsWith(".odt") ||
+ lowerPath.endsWith(".rtf")
+ ) {
+ flags.add("document");
}
if (options.blockAds === false) {