Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Custom Rust document parser (#2159)
tom443de6fcc5bb
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.tomlindex 86ef34d4d..7c38f4116 100644--- a/apps/api/native/Cargo.toml+++ b/apps/api/native/Cargo.toml@@ -9,9 +9,11 @@ version = "0.1.0"crate-type = ["cdylib"][dependencies]+chrono = { version = "0.4", features = ["serde"] }kuchikiki = "0.8.2"lol_html = "2.6.0"lopdf = "0.38.0"+maud = "0.27.0"napi = { version = "3.0.0", features = ["serde-json"] }napi-derive = "3.0.0"nodesig = { git = "https://github.com/firecrawl/nodesig" }@@ -22,6 +24,7 @@ serde = { version = "1.0.219", features = ["derive"] }serde_json = "1.0.143"texting_robots = "0.2.2"url = "2.5.7"+zip = "5.0.0"[build-dependencies]napi-build = "2"diff --git a/apps/api/native/src/document/mod.rs b/apps/api/native/src/document/mod.rsnew file mode 100644index 000000000..173e33613--- /dev/null+++ b/apps/api/native/src/document/mod.rs@@ -0,0 +1,64 @@+pub mod model;+pub mod providers;+pub mod renderers;++pub use providers::factory::DocumentType;++use crate::document::model::Document;+use crate::document::providers::factory::ProviderFactory;+use crate::document::renderers::html::HtmlRenderer;+use napi::bindgen_prelude::*;+use napi_derive::napi;++#[napi]+#[derive(Debug)]+pub enum ConversionError {+ UnsupportedFormat,+ Provider(String),+ Io(String),+}++impl From<std::io::Error> for ConversionError {+ fn from(e: std::io::Error) -> Self {+ ConversionError::Io(e.to_string())+ }+}++#[napi]+pub struct DocumentConverter {+ factory: ProviderFactory,+ html_renderer: HtmlRenderer,+}++impl Default for DocumentConverter {+ fn default() -> Self {+ Self::new()+ }+}++#[napi]+impl DocumentConverter {+ #[napi(constructor)]+ pub fn new() -> Self {+ Self {+ factory: ProviderFactory::new(),+ html_renderer: HtmlRenderer::new(),+ }+ }++ #[napi]+ pub fn convert_buffer_to_html(+ &self,+ data: &[u8],+ doc_type: DocumentType,+ ) -> napi::Result<String> {+ let provider = self.factory.get_provider(doc_type);++ let document: Document = provider+ .parse_buffer(data)+ .map_err(|e| Error::new(Status::GenericFailure, format!("Provider error: {e}")))?;++ let html = self.html_renderer.render(&document);+ Ok(html)+ }+}diff --git a/apps/api/native/src/document/model/mod.rs b/apps/api/native/src/document/model/mod.rsnew file mode 100644index 000000000..2c9777455--- /dev/null+++ b/apps/api/native/src/document/model/mod.rs@@ -0,0 +1,135 @@+use chrono::{DateTime, Utc};+use std::num::NonZeroU32;++#[derive(Debug, Clone)]+pub struct Document {+ pub blocks: Vec<Block>,+ pub metadata: DocumentMetadata,+ pub notes: Vec<Note>,+ pub comments: Vec<Comment>,+}++#[derive(Debug, Clone, Default)]+pub struct DocumentMetadata {+ pub title: Option<String>,+ pub author: Option<String>,+ pub created: Option<DateTime<Utc>>,+}++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub struct NoteId(pub String);++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub struct CommentId(pub String);++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub struct BookmarkId(pub String);++#[derive(Debug, Clone)]+pub enum Block {+ Paragraph(Paragraph),+ Table(Table),+ List(List),+ Image(Image),+}++#[derive(Debug, Clone)]+pub struct Paragraph {+ pub kind: ParagraphKind,+ pub inlines: Vec<Inline>,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum ParagraphKind {+ Normal,+ Heading(u8), // 1..=6 will render as <h1>.. <h6>+ Blockquote,+}++#[derive(Debug, Clone)]+pub enum Inline {+ Text(String),+ LineBreak,+ Link { href: String, children: Vec<Inline> },++ Strong(Vec<Inline>),+ Em(Vec<Inline>),+ Del(Vec<Inline>),+ Code(String),+ Sup(Vec<Inline>),+ Sub(Vec<Inline>),++ FootnoteRef(NoteId),+ EndnoteRef(NoteId),+ CommentRef(CommentId),+ Bookmark(BookmarkId),+}++#[derive(Debug, Clone)]+pub struct Image {+ pub src: String,+ pub alt: Option<String>,+}++#[derive(Debug, Clone)]+pub struct Table {+ pub rows: Vec<TableRow>,+}++#[derive(Debug, Clone)]+pub struct TableRow {+ pub cells: Vec<TableCell>,+ pub kind: TableRowKind,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum TableRowKind {+ Header,+ Body,+ Footer,+}++#[derive(Debug, Clone)]+pub struct TableCell {+ pub blocks: Vec<Block>,+ pub colspan: NonZeroU32,+ pub rowspan: NonZeroU32,+}++#[derive(Debug, Clone)]+pub struct List {+ pub items: Vec<ListItem>,+ pub list_type: ListType,+}++#[derive(Debug, Clone)]+pub struct ListItem {+ pub blocks: Vec<Block>,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum ListType {+ Ordered,+ Unordered,+}++#[derive(Debug, Clone)]+pub struct Note {+ pub id: NoteId,+ pub kind: NoteKind,+ pub blocks: Vec<Block>,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum NoteKind {+ Footnote,+ Endnote,+}++#[derive(Debug, Clone)]+pub struct Comment {+ pub id: CommentId,+ pub author_name: Option<String>,+ pub author_initials: Option<String>,+ pub blocks: Vec<Block>,+}diff --git a/apps/api/native/src/document/providers/docx.rs b/apps/api/native/src/document/providers/docx.rsnew file mode 100644index 000000000..9bb168fb1--- /dev/null+++ b/apps/api/native/src/document/providers/docx.rs@@ -0,0 +1,1074 @@+use crate::document::model::*;+use crate::document::providers::DocumentProvider;+use chrono::{DateTime, Utc};+use roxmltree::{Document as XmlDoc, Node};+use std::collections::HashMap;+use std::error::Error;+use std::io::{Read, Seek};+use std::num::NonZeroU32;+use zip::read::ZipArchive;++pub struct DocxProvider;++impl DocxProvider {+ pub fn new() -> Self {+ Self+ }+}++impl DocumentProvider for DocxProvider {+ fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {+ let cursor = std::io::Cursor::new(data);+ let mut zip = ZipArchive::new(cursor)?;++ let relationships = read_relationships(&mut zip, "word/_rels/document.xml.rels");+ let styles = read_styles(&mut zip);+ let numbering = read_numbering(&mut zip);++ let document_xml = read_zip_text(&mut zip, "word/document.xml")+ .ok_or("Missing word/document.xml in document")?;+ let xml = XmlDoc::parse(strip_bom(&document_xml))?;++ let metadata = read_core_properties(&mut zip).unwrap_or_default();++ let size_buckets = compute_style_size_buckets_for_doc(&xml, &styles);+ let mut blocks = Vec::new();+ if let Some(body) = xml.descendants().find(|n| is_tag(n, "body")) {+ blocks = parse_block_children(+ &body,+ &relationships,+ &styles,+ &size_buckets,+ &numbering,+ &mut zip,+ );+ }++ let mut notes = Vec::new();+ notes.extend(read_notes(+ &mut zip,+ "word/footnotes.xml",+ "word/_rels/footnotes.xml.rels",+ NoteKind::Footnote,+ &styles,+ &size_buckets,+ &numbering,+ ));+ notes.extend(read_notes(+ &mut zip,+ "word/endnotes.xml",+ "word/_rels/endnotes.xml.rels",+ NoteKind::Endnote,+ &styles,+ &size_buckets,+ &numbering,+ ));++ let comments = read_comments(+ &mut zip,+ "word/comments.xml",+ "word/_rels/comments.xml.rels",+ &styles,+ &size_buckets,+ &numbering,+ );++ Ok(Document {+ blocks,+ metadata,+ notes,+ comments,+ })+ }++ fn name(&self) -> &'static str {+ "docx"+ }+}++fn read_zip_text<R: Read + std::io::Seek>(zip: &mut ZipArchive<R>, path: &str) -> Option<String> {+ let mut file = zip.by_name(path).ok()?;+ let mut s = String::new();+ file.read_to_string(&mut s).ok()?;+ Some(s)+}++fn strip_bom(s: &str) -> &str {+ const BOM: char = '\u{FEFF}';+ s.strip_prefix(BOM).unwrap_or(s)+}++#[derive(Debug, Clone, Default)]+struct Relationships {+ targets: HashMap<String, String>,+}++impl Relationships {+ fn get(&self, id: &str) -> Option<&str> {+ self.targets.get(id).map(|s| s.as_str())+ }+}++fn read_relationships<R: Read + std::io::Seek>(+ zip: &mut ZipArchive<R>,+ path: &str,+) -> Relationships {+ let xml_text = match read_zip_text(zip, path) {+ Some(s) => s,+ None => return Relationships::default(),+ };+ let xml = match XmlDoc::parse(strip_bom(&xml_text)) {+ Ok(d) => d,+ Err(_) => return Relationships::default(),+ };+ let mut map = HashMap::new();+ for rel in xml.descendants().filter(|n| is_tag(n, "Relationship")) {+ if let (Some(id), Some(target)) = (get_attr_local(&rel, "Id"), get_attr_local(&rel, "Target")) {+ map.insert(id.to_string(), target.to_string());+ }+ }+ Relationships { targets: map }+}++fn read_core_properties<R: Read + std::io::Seek>(+ zip: &mut ZipArchive<R>,+) -> Option<DocumentMetadata> {+ let text = read_zip_text(zip, "docProps/core.xml")?;+ let xml = XmlDoc::parse(strip_bom(&text)).ok()?;+ let mut meta = DocumentMetadata::default();++ if let Some(title) = xml+ .descendants()+ .find(|n| is_tag(n, "title"))+ .and_then(|n| n.text())+ {+ let trimmed = title.trim();+ if !trimmed.is_empty() {+ meta.title = Some(trimmed.to_string());+ }+ }+ if let Some(author) = xml+ .descendants()+ .find(|n| is_tag(n, "creator"))+ .and_then(|n| n.text())+ {+ let trimmed = author.trim();+ if !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("unknown") {+ meta.author = Some(trimmed.to_string());+ }+ }+ if let Some(created) = xml+ .descendants()+ .find(|n| is_tag(n, "created"))+ .and_then(|n| n.text())+ {+ if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(created) {+ meta.created = Some(DateTime::<Utc>::from(dt));+ }+ }++ Some(meta)+}++#[derive(Debug, Default)]+struct StylesInfo {+ heading_level_by_style_id: HashMap<String, u8>,+ name_by_style_id: HashMap<String, String>,+ default_size_by_style_id: HashMap<String, u32>,+}++fn read_styles<R: Read + Seek>(zip: &mut ZipArchive<R>) -> StylesInfo {+ let text = match read_zip_text(zip, "word/styles.xml") {+ Some(t) => t,+ None => return StylesInfo::default(),+ };+ let doc = match XmlDoc::parse(strip_bom(&text)) {+ Ok(d) => d,+ Err(_) => return StylesInfo::default(),+ };+ let mut info = StylesInfo::default();++ for style in doc.descendants().filter(|n| is_tag(n, "style")) {+ if get_attr_local(&style, "type") != Some("paragraph") {+ continue;+ }+ let Some(style_id) = get_attr_local(&style, "styleId") else {+ continue;+ };+ let style_id = style_id.to_string();++ if let Some(name) = child(&style, "name").and_then(|n| get_attr_local(&n, "val")) {+ info+ .name_by_style_id+ .insert(style_id.clone(), name.to_string());+ }++ if let Some(rpr) = child(&style, "rPr") {+ if let Some(sz) = child(&rpr, "sz").and_then(|n| get_attr_local(&n, "val")) {+ if let Ok(v) = sz.parse::<u32>() {+ info.default_size_by_style_id.insert(style_id.clone(), v);+ }+ }+ }++ if let Some(ppr) = child(&style, "pPr") {+ if let Some(ol) = child(&ppr, "outlineLvl").and_then(|n| get_attr_local(&n, "val")) {+ if let Ok(v) = ol.parse::<u8>() {+ info+ .heading_level_by_style_id+ .insert(style_id.clone(), (v + 1).min(6));+ }+ }+ }++ if !info.heading_level_by_style_id.contains_key(&style_id) {+ let mut assigned: Option<u8> = None;+ if let Some(name) = info.name_by_style_id.get(&style_id) {+ assigned = parse_heading_level(name);+ }+ if assigned.is_none() {+ assigned = parse_heading_level(&style_id);+ }+ if assigned.is_none() {+ let name_l = info+ .name_by_style_id+ .get(&style_id)+ .map(|s| s.to_ascii_lowercase())+ .unwrap_or_default();+ let id_l = style_id.to_ascii_lowercase();+ if name_l.contains("title") || id_l.contains("title") {+ assigned = Some(1);+ } else if name_l.contains("heading") || id_l.contains("heading") {+ assigned = Some(2);+ }+ }+ if let Some(l) = assigned {+ info.heading_level_by_style_id.insert(style_id, l);+ }+ }+ }+ info+}++fn is_tag(node: &Node, local: &str) -> bool {+ node.is_element() && node.tag_name().name() == local+}++fn get_attr_local<'a>(node: &Node<'a, 'a>, local: &str) -> Option<&'a str> {+ node+ .attributes()+ .find(|a| {+ let name = a.name();+ match name.rsplit_once(':') {+ Some((_, l)) => l == local,+ None => name == local,+ }+ })+ .map(|a| a.value())+}++fn child<'a>(node: &Node<'a, 'a>, local: &str) -> Option<Node<'a, 'a>> {+ node+ .children()+ .find(|n| n.is_element() && n.tag_name().name() == local)+}++fn children<'a, 'b>(+ node: &Node<'a, 'a>,+ local: &'b str,+) -> impl Iterator<Item = Node<'a, 'a>> + use<'a, 'b> {+ node+ .children()+ .filter(move |n| n.is_element() && n.tag_name().name() == local)+}++fn parse_paragraph_with_listinfo(+ node: &Node,+ rels: &Relationships,+ styles: &StylesInfo,+ size_buckets: &HashMap<String, Vec<u32>>,+ numbering: &NumberingInfo,+) -> Option<(Paragraph, Option<ListInfo>)> {+ let kind = paragraph_kind(node, styles, size_buckets);+ let base_style = paragraph_run_style(node);+ let mut inlines = Vec::new();++ for child in node.children().filter(|n| n.is_element()) {+ if is_tag(&child, "r") {+ let run_inlines = parse_run(&child, rels, &base_style);+ inlines.extend(run_inlines);+ } else if is_tag(&child, "hyperlink") {+ if let Some(link) = parse_hyperlink(&child, rels, &base_style) {+ inlines.push(link);+ }+ } else if is_tag(&child, "bookmarkStart") {+ if let Some(name) = get_attr_local(&child, "name") {+ inlines.push(Inline::Bookmark(BookmarkId(name.to_string())));+ }+ } else if is_tag(&child, "br") {+ inlines.push(Inline::LineBreak);+ }+ }++ let list_info = paragraph_list_info(node, numbering);+ Some((Paragraph { kind, inlines }, list_info))+}++fn paragraph_kind(+ p: &Node,+ styles: &StylesInfo,+ size_buckets: &HashMap<String, Vec<u32>>,+) -> ParagraphKind {+ let Some(ppr) = child(p, "pPr") else {+ return ParagraphKind::Normal;+ };++ if let Some(level) = child(&ppr, "outlineLvl").and_then(|n| get_attr_local(&n, "val")) {+ if let Ok(v) = level.parse::<u8>() {+ return ParagraphKind::Heading((v + 1).min(6));+ }+ }++ if let Some(style_id) = child(&ppr, "pStyle").and_then(|n| get_attr_local(&n, "val")) {+ let mut base_level: Option<u8> = styles.heading_level_by_style_id.get(style_id).copied();++ if base_level.is_none() {+ if let Some(name) = styles.name_by_style_id.get(style_id) {+ if let Some(l) = parse_heading_level(name) {+ base_level = Some(l);+ }+ if base_level.is_none() && name.to_ascii_lowercase().contains("quote") {+ return ParagraphKind::Blockquote;+ }+ }+ }+ if base_level.is_none() {+ if let Some(l) = parse_heading_level(style_id) {+ base_level = Some(l);+ }+ }+ if base_level.is_none() {+ let id_l = style_id.to_ascii_lowercase();+ let name_l = styles+ .name_by_style_id+ .get(style_id)+ .map(|s| s.to_ascii_lowercase())+ .unwrap_or_default();+ if name_l.contains("title") || id_l.contains("title") {+ base_level = Some(1);+ } else if name_l.contains("heading") || id_l.contains("heading") {+ base_level = Some(2);+ } else if name_l.contains("quote") || id_l.contains("quote") {+ return ParagraphKind::Blockquote;+ }+ }++ if let Some(mut level) = base_level {+ let para_size = paragraph_effective_size(p, styles, style_id);+ if let Some(buckets) = size_buckets.get(style_id) {+ if let Some(size) = para_size {+ if let Some(index) = buckets.iter().position(|&s| s == size) {+ level = (level + index as u8).min(6);+ }+ }+ }+ return ParagraphKind::Heading(level);+ }+ }++ ParagraphKind::Normal+}++fn parse_heading_level(s: &str) -> Option<u8> {+ let lower = s.to_ascii_lowercase();+ let idx = lower.find("heading")?;+ let rest = &lower[idx + "heading".len()..];+ let digits: String = rest+ .chars()+ .skip_while(|c| c.is_whitespace() || *c == '-')+ .take_while(|c| c.is_ascii_digit())+ .collect();+ if let Ok(n) = digits.parse::<u8>() {+ if n >= 1 {+ return Some(n.min(6));+ }+ }+ None+}++fn paragraph_effective_size(p: &Node, styles: &StylesInfo, style_id: &str) -> Option<u32> {+ let mut max_sz: Option<u32> = None;++ if let Some(ppr) = child(p, "pPr") {+ if let Some(rpr) = child(&ppr, "rPr") {+ if let Some(sz) = child(&rpr, "sz").and_then(|n| get_attr_local(&n, "val")) {+ if let Ok(v) = sz.parse::<u32>() {+ max_sz = Some(max_sz.map_or(v, |m| m.max(v)));+ }+ }+ }+ }++ for r in children(p, "r") {+ if let Some(rpr) = child(&r, "rPr") {+ if let Some(sz) = child(&rpr, "sz").and_then(|n| get_attr_local(&n, "val")) {+ if let Ok(v) = sz.parse::<u32>() {+ max_sz = Some(max_sz.map_or(v, |m| m.max(v)));+ }+ }+ }+ }++ max_sz.or_else(|| styles.default_size_by_style_id.get(style_id).copied())+}++fn compute_style_size_buckets_for_doc(+ xml: &XmlDoc,+ styles: &StylesInfo,+) -> HashMap<String, Vec<u32>> {+ let mut sets: HashMap<String, std::collections::BTreeSet<u32>> = HashMap::new();++ for p in xml.descendants().filter(|n| is_tag(n, "p")) {+ let Some(style_id) = child(&p, "pPr")+ .and_then(|n| child(&n, "pStyle"))+ .and_then(|n| get_attr_local(&n, "val"))+ else {+ continue;+ };++ let id_l = style_id.to_ascii_lowercase();+ let name_l = styles+ .name_by_style_id+ .get(style_id)+ .map(|s| s.to_ascii_lowercase())+ .unwrap_or_default();++ if !(id_l.contains("heading")+ || id_l.contains("title")+ || name_l.contains("heading")+ || name_l.contains("title"))+ {+ continue;+ }++ if let Some(sz) = paragraph_effective_size(&p, styles, style_id) {+ sets.entry(style_id.to_string()).or_default().insert(sz);+ }+ }++ sets+ .into_iter()+ .map(|(k, vset)| {+ let mut v: Vec<u32> = vset.into_iter().collect();+ v.sort_by(|a, b| b.cmp(a));+ (k, v)+ })+ .collect()+}++#[derive(Clone, Default)]+struct RunStyle {+ bold: Option<bool>,+ italic: Option<bool>,+ strike: Option<bool>,+ code: Option<bool>,+ vert_align: Option<VerticalAlign>,+}++#[derive(Clone, Copy)]+enum VerticalAlign {+ Baseline,+ Superscript,+ Subscript,+}++#[derive(Clone, Copy)]+struct ResolvedRunStyle {+ bold: bool,+ italic: bool,+ strike: bool,+ code: bool,+ vert_align: VerticalAlign,+}++impl RunStyle {+ fn merged(&self, overrides: &RunStyle) -> RunStyle {+ RunStyle {+ bold: overrides.bold.or(self.bold),+ italic: overrides.italic.or(self.italic),+ strike: overrides.strike.or(self.strike),+ code: overrides.code.or(self.code),+ vert_align: overrides.vert_align.or(self.vert_align),+ }+ }++ fn resolve_with(&self, local: &RunStyle) -> ResolvedRunStyle {+ ResolvedRunStyle {+ bold: local.bold.or(self.bold).unwrap_or(false),+ italic: local.italic.or(self.italic).unwrap_or(false),+ strike: local.strike.or(self.strike).unwrap_or(false),+ code: local.code.or(self.code).unwrap_or(false),+ vert_align: local+ .vert_align+ .or(self.vert_align)+ .unwrap_or(VerticalAlign::Baseline),+ }+ }+}++impl ResolvedRunStyle {+ fn apply(self, mut inlines: Vec<Inline>) -> Vec<Inline> {+ if self.strike {+ inlines = vec![Inline::Del(inlines)];+ }+ if self.italic {+ inlines = vec![Inline::Em(inlines)];+ }+ if self.bold {+ inlines = vec![Inline::Strong(inlines)];+ }+ match self.vert_align {+ VerticalAlign::Superscript => vec![Inline::Sup(inlines)],+ VerticalAlign::Subscript => vec![Inline::Sub(inlines)],+ VerticalAlign::Baseline => inlines,+ }+ }+}++fn read_on_off(node: &Node) -> Option<bool> {+ let value = get_attr_local(node, "val").map(|v| v.to_ascii_lowercase());+ match value.as_deref() {+ None => Some(true),+ Some("0") | Some("false") | Some("off") => Some(false),+ Some(_) => Some(true),+ }+}++fn run_style_from_rpr(rpr: &Node) -> RunStyle {+ let mut style = RunStyle::default();++ if let Some(b) = child(rpr, "b").and_then(|n| read_on_off(&n)) {+ style.bold = Some(b);+ }+ if let Some(i) = child(rpr, "i").and_then(|n| read_on_off(&n)) {+ style.italic = Some(i);+ }+ if let Some(s) = child(rpr, "strike").and_then(|n| read_on_off(&n)) {+ style.strike = Some(s);+ }+ if let Some(rstyle) = child(rpr, "rStyle").and_then(|n| get_attr_local(&n, "val")) {+ if rstyle.to_ascii_lowercase().contains("code") {+ style.code = Some(true);+ }+ }+ if let Some(va) = child(rpr, "vertAlign").and_then(|n| get_attr_local(&n, "val")) {+ let lower = va.to_ascii_lowercase();+ let val = if lower == "sup" || lower == "superscript" {+ VerticalAlign::Superscript+ } else if lower == "sub" || lower == "subscript" {+ VerticalAlign::Subscript+ } else {+ VerticalAlign::Baseline+ };+ style.vert_align = Some(val);+ }++ style+}++fn paragraph_run_style(p: &Node) -> RunStyle {+ child(p, "pPr")+ .and_then(|ppr| child(&ppr, "rPr"))+ .map(|rpr| run_style_from_rpr(&rpr))+ .unwrap_or_default()+}++fn parse_run(run: &Node, _rels: &Relationships, base_style: &RunStyle) -> Vec<Inline> {+ let local_style = child(run, "rPr")+ .map(|rpr| run_style_from_rpr(&rpr))+ .unwrap_or_default();+ let resolved = base_style.resolve_with(&local_style);++ let mut out = Vec::new();++ for c in run.children().filter(|n| n.is_element()) {+ if is_tag(&c, "t") {+ if let Some(text) = c.text() {+ out.push(Inline::Text(text.to_string()));+ }+ } else if is_tag(&c, "br") {+ out.push(Inline::LineBreak);+ } else if is_tag(&c, "tab") {+ out.push(Inline::Text("\t".to_string()));+ } else if is_tag(&c, "footnoteReference") {+ if let Some(id) = get_attr_local(&c, "id") {+ out.push(Inline::FootnoteRef(NoteId(id.to_string())));+ }+ } else if is_tag(&c, "endnoteReference") {+ if let Some(id) = get_attr_local(&c, "id") {+ out.push(Inline::EndnoteRef(NoteId(id.to_string())));+ }+ } else if is_tag(&c, "commentReference") {+ if let Some(id) = get_attr_local(&c, "id") {+ out.push(Inline::CommentRef(CommentId(id.to_string())));+ }+ }+ }++ if resolved.code {+ let code_text: String = out+ .iter()+ .filter_map(|i| match i {+ Inline::Text(s) => Some(s.as_str()),+ _ => None,+ })+ .collect();+ if !code_text.is_empty() {+ return vec![Inline::Code(code_text)];+ }+ }++ resolved.apply(out)+}++fn parse_hyperlink(node: &Node, rels: &Relationships, base_style: &RunStyle) -> Option<Inline> {+ let href = if let Some(id) = get_attr_local(node, "id") {+ rels.get(id).map(|s| s.to_string())+ } else {+ get_attr_local(node, "anchor").map(|anchor| format!("#{anchor}"))+ }?;++ let link_style = child(node, "rPr")+ .map(|rpr| run_style_from_rpr(&rpr))+ .unwrap_or_default();+ let combined_style = base_style.merged(&link_style);++ let mut children = Vec::new();+ for child in node.children().filter(|n| n.is_element()) {+ if is_tag(&child, "r") {+ children.extend(parse_run(&child, rels, &combined_style));+ }+ }++ Some(Inline::Link { href, children })+}++fn parse_table<R: Read + Seek>(+ node: &Node,+ rels: &Relationships,+ styles: &StylesInfo,+ size_buckets: &HashMap<String, Vec<u32>>,+ numbering: &NumberingInfo,+ zip: &mut ZipArchive<R>,+) -> Option<Table> {+ let mut rows = Vec::new();+ for tr in children(node, "tr") {+ let kind = table_row_kind(&tr);+ let mut cells = Vec::new();+ for tc in children(&tr, "tc") {+ let cell_blocks = parse_block_children(&tc, rels, styles, size_buckets, numbering, zip);+ let cell = TableCell {+ blocks: cell_blocks,+ colspan: NonZeroU32::new(1).unwrap(),+ rowspan: NonZeroU32::new(1).unwrap(),+ };+ cells.push(cell);+ }+ rows.push(TableRow { cells, kind });+ }++ if !rows.is_empty() && rows.iter().any(|r| matches!(r.kind, TableRowKind::Header)) {+ if let Some(last) = rows.last_mut() {+ last.kind = TableRowKind::Footer;+ }+ }+ Some(Table { rows })+}++fn table_row_kind(tr: &Node) -> TableRowKind {+ if let Some(trpr) = child(tr, "trPr") {+ if child(&trpr, "tblHeader").is_some() {+ return TableRowKind::Header;+ }+ }+ TableRowKind::Body+}++#[derive(Debug, Clone, PartialEq, Eq)]+struct ListInfo {+ list_type: ListType,+ num_id: String,+ ilvl: u32,+}++#[derive(Debug, Default)]+struct NumberingInfo {+ num_to_abstract: HashMap<String, String>,+ abstract_levels: HashMap<String, HashMap<String, ListType>>,+}++impl NumberingInfo {+ fn list_type(&self, num_id: &str, ilvl: &str) -> Option<ListType> {+ let abs = self.num_to_abstract.get(num_id)?;+ let levels = self.abstract_levels.get(abs)?;+ levels.get(ilvl).copied()+ }+}++fn read_numbering<R: Read + Seek>(zip: &mut ZipArchive<R>) -> NumberingInfo {+ let text = match read_zip_text(zip, "word/numbering.xml") {+ Some(t) => t,+ None => return NumberingInfo::default(),+ };+ let doc = match XmlDoc::parse(strip_bom(&text)) {+ Ok(d) => d,+ Err(_) => return NumberingInfo::default(),+ };++ let mut info = NumberingInfo::default();++ for num in doc.descendants().filter(|n| is_tag(n, "num")) {+ if let Some(num_id) = get_attr_local(&num, "numId") {+ if let Some(abs) = child(&num, "abstractNumId").and_then(|n| get_attr_local(&n, "val")) {+ info+ .num_to_abstract+ .insert(num_id.to_string(), abs.to_string());+ }+ }+ }++ for abs in doc.descendants().filter(|n| is_tag(n, "abstractNum")) {+ if let Some(abs_id) = get_attr_local(&abs, "abstractNumId") {+ let mut levels: HashMap<String, ListType> = HashMap::new();+ for lvl in children(&abs, "lvl") {+ if let Some(ilvl) = get_attr_local(&lvl, "ilvl") {+ let fmt = child(&lvl, "numFmt").and_then(|n| get_attr_local(&n, "val"));+ let list_type = match fmt.unwrap_or("") {+ "bullet" => ListType::Unordered,+ _ => ListType::Ordered,+ };+ levels.insert(ilvl.to_string(), list_type);+ }+ }+ info.abstract_levels.insert(abs_id.to_string(), levels);+ }+ }++ info+}++fn paragraph_list_info(p: &Node, numbering: &NumberingInfo) -> Option<ListInfo> {+ let ppr = child(p, "pPr")?;+ let numpr = child(&ppr, "numPr")?;+ let ilvl_str = child(&numpr, "ilvl").and_then(|n| get_attr_local(&n, "val"))?;+ let num_id = child(&numpr, "numId").and_then(|n| get_attr_local(&n, "val"))?;+ let ilvl: u32 = ilvl_str.parse().unwrap_or(0);+ let list_type = numbering+ .list_type(num_id, ilvl_str)+ .unwrap_or(ListType::Unordered);+ Some(ListInfo {+ list_type,+ num_id: num_id.to_string(),+ ilvl,+ })+}++fn parse_block_children<R: Read + Seek>(+ parent: &Node,+ rels: &Relationships,+ styles: &StylesInfo,+ size_buckets: &HashMap<String, Vec<u32>>,+ numbering: &NumberingInfo,+ zip: &mut ZipArchive<R>,+) -> Vec<Block> {+ let nodes: Vec<Node> = parent.children().filter(|n| n.is_element()).collect();+ let mut out: Vec<Block> = Vec::new();+ let mut i = 0usize;++ while i < nodes.len() {+ let node = &nodes[i];+ if is_tag(node, "p") {+ if paragraph_list_info(node, numbering).is_some() {+ let (list, new_i) = parse_list(&nodes, i, rels, styles, size_buckets, numbering, zip);+ if !list.items.is_empty() {+ out.push(Block::List(list));+ }+ i = new_i;+ continue;+ }+ if let Some(image) = parse_image_paragraph(node, rels, zip) {+ out.push(Block::Image(image));+ i += 1;+ continue;+ }+ if let Some((para, _)) =+ parse_paragraph_with_listinfo(node, rels, styles, size_buckets, numbering)+ {+ if paragraph_has_visible_content(¶) {+ out.push(Block::Paragraph(para));+ }+ }+ i += 1;+ } else if is_tag(node, "tbl") {+ if let Some(table) = parse_table(node, rels, styles, size_buckets, numbering, zip) {+ out.push(Block::Table(table));+ }+ i += 1;+ } else {+ i += 1;+ }+ }+ out+}++fn parse_list<R: Read + Seek>(+ nodes: &[Node],+ mut i: usize,+ rels: &Relationships,+ styles: &StylesInfo,+ size_buckets: &HashMap<String, Vec<u32>>,+ numbering: &NumberingInfo,+ zip: &mut ZipArchive<R>,+) -> (List, usize) {+ let first_info =+ paragraph_list_info(&nodes[i], numbering).expect("parse_list called at non-list paragraph");+ let base_ilvl = first_info.ilvl;+ let base_num_id = first_info.num_id.clone();+ let base_type = first_info.list_type;++ let mut list = List {+ items: Vec::new(),+ list_type: base_type,+ };++ while i < nodes.len() {+ let node = &nodes[i];+ if !is_tag(node, "p") {+ break;+ }+ let info = match paragraph_list_info(node, numbering) {+ Some(x) => x,+ None => break,+ };+ if info.ilvl < base_ilvl {+ break;+ }+ if info.ilvl == base_ilvl && (info.list_type != base_type || info.num_id != base_num_id) {+ break;+ }++ if info.ilvl == base_ilvl {+ let mut blocks: Vec<Block> = Vec::new();+ if let Some(image) = parse_image_paragraph(node, rels, zip) {+ blocks.push(Block::Image(image));+ } else if let Some((para, _)) =+ parse_paragraph_with_listinfo(node, rels, styles, size_buckets, numbering)+ {+ if paragraph_has_visible_content(¶) {+ blocks.push(Block::Paragraph(para));+ }+ }+ list.items.push(ListItem { blocks });+ i += 1;++ loop {+ if i >= nodes.len() {+ break;+ }+ let node2 = &nodes[i];+ if !is_tag(node2, "p") {+ break;+ }+ match paragraph_list_info(node2, numbering) {+ Some(sub) if sub.ilvl > base_ilvl => {+ let (sublist, new_i) = parse_list(nodes, i, rels, styles, size_buckets, numbering, zip);+ if let Some(last) = list.items.last_mut() {+ last.blocks.push(Block::List(sublist));+ }+ i = new_i;+ }+ _ => break,+ }+ }++ if list.items.last().is_some_and(|last| last.blocks.is_empty()) {+ list.items.pop();+ }+ }+ }++ (list, i)+}++fn paragraph_has_visible_content(p: &Paragraph) -> bool {+ inlines_have_visible_content(&p.inlines)+}++fn inlines_have_visible_content(inlines: &[Inline]) -> bool {+ inlines.iter().any(inline_is_visible)+}++fn inline_is_visible(i: &Inline) -> bool {+ match i {+ Inline::Text(t) => !t.trim().is_empty(),+ Inline::LineBreak => false,+ Inline::Link { children, .. } => inlines_have_visible_content(children),+ Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {+ inlines_have_visible_content(c)+ }+ Inline::Code(c) => !c.trim().is_empty(),+ Inline::FootnoteRef(_) | Inline::EndnoteRef(_) | Inline::CommentRef(_) => true,+ Inline::Bookmark(_) => false,+ }+}++fn parse_image_paragraph<R: Read + Seek>(+ p: &Node,+ rels: &Relationships,+ zip: &mut ZipArchive<R>,+) -> Option<Image> {+ let has_text = p+ .descendants()+ .filter(|n| is_tag(n, "t"))+ .any(|t| t.text().map(|s| !s.trim().is_empty()).unwrap_or(false));+ if has_text {+ return None;+ }++ if let Some(drawing) = p.descendants().find(|n| is_tag(n, "drawing")) {+ if let Some(img) = image_from_drawing(&drawing, rels, zip) {+ return Some(img);+ }+ }++ if let Some(pict) = p.descendants().find(|n| is_tag(n, "pict")) {+ if let Some(img) = image_from_vml(&pict, rels, zip) {+ return Some(img);+ }+ }+ None+}++fn image_from_drawing<R: Read + Seek>(+ drawing: &Node,+ rels: &Relationships,+ zip: &mut ZipArchive<R>,+) -> Option<Image> {+ let blip = drawing.descendants().find(|n| is_tag(n, "blip"))?;+ let rel_id = get_attr_local(&blip, "embed").or_else(|| get_attr_local(&blip, "link"))?;+ let alt = drawing+ .descendants()+ .find(|n| is_tag(n, "docPr"))+ .and_then(|n| get_attr_local(&n, "descr").or_else(|| get_attr_local(&n, "title")))+ .map(|s| s.to_string());+ image_from_relationship_id(rel_id, rels, zip, alt)+}++fn image_from_vml<R: Read + Seek>(+ pict: &Node,+ rels: &Relationships,+ zip: &mut ZipArchive<R>,+) -> Option<Image> {+ let imagedata = pict.descendants().find(|n| is_tag(n, "imagedata"))?;+ let rel_id = get_attr_local(&imagedata, "id")?;+ let alt = get_attr_local(&imagedata, "title").map(|s| s.to_string());+ image_from_relationship_id(rel_id, rels, zip, alt)+}++fn image_from_relationship_id<R: Read + Seek>(+ rid: &str,+ rels: &Relationships,+ _zip: &mut ZipArchive<R>,+ alt: Option<String>,+) -> Option<Image> {+ let target = rels.get(rid)?;+ // only include external images (http/https URLs)+ if target.starts_with("http://") || target.starts_with("https://") {+ return Some(Image {+ src: target.to_string(),+ alt,+ });+ }+ None+}++fn read_notes<R: Read + Seek>(+ zip: &mut ZipArchive<R>,+ xml_path: &str,+ rels_path: &str,+ kind: NoteKind,+ styles: &StylesInfo,+ size_buckets: &HashMap<String, Vec<u32>>,+ numbering: &NumberingInfo,+) -> Vec<Note> {+ let text = match read_zip_text(zip, xml_path) {+ Some(t) => t,+ None => return Vec::new(),+ };+ let doc = match XmlDoc::parse(strip_bom(&text)) {+ Ok(d) => d,+ Err(_) => return Vec::new(),+ };+ let rels = read_relationships(zip, rels_path);++ let mut notes = Vec::new();+ let root_tag = match kind {+ NoteKind::Footnote => "footnote",+ NoteKind::Endnote => "endnote",+ };+ for n in doc.descendants().filter(|n| is_tag(n, root_tag)) {+ let Some(id) = get_attr_local(&n, "id") else {+ continue;+ };++ if let Some(t) = get_attr_local(&n, "type") {+ if t == "separator" || t == "continuationSeparator" {+ continue;+ }+ }+ let blocks = parse_block_children(&n, &rels, styles, size_buckets, numbering, zip);+ notes.push(Note {+ id: NoteId(id.to_string()),+ kind,+ blocks,+ });+ }+ notes+}++fn read_comments<R: Read + Seek>(+ zip: &mut ZipArchive<R>,+ xml_path: &str,+ rels_path: &str,+ styles: &StylesInfo,+ size_buckets: &HashMap<String, Vec<u32>>,+ numbering: &NumberingInfo,+) -> Vec<Comment> {+ let text = match read_zip_text(zip, xml_path) {+ Some(t) => t,+ None => return Vec::new(),+ };+ let doc = match XmlDoc::parse(strip_bom(&text)) {+ Ok(d) => d,+ Err(_) => return Vec::new(),+ };+ let rels = read_relationships(zip, rels_path);++ let mut out = Vec::new();+ for c in doc.descendants().filter(|n| is_tag(n, "comment")) {+ let Some(id) = get_attr_local(&c, "id") else {+ continue;+ };++ let author = get_attr_local(&c, "author").map(|s| s.to_string());+ let initials = get_attr_local(&c, "initials").map(|s| s.to_string());+ let blocks = parse_block_children(&c, &rels, styles, size_buckets, numbering, zip);+ out.push(Comment {+ id: CommentId(id.to_string()),+ author_name: author,+ author_initials: initials,+ blocks,+ });+ }+ out+}diff --git a/apps/api/native/src/document/providers/factory.rs b/apps/api/native/src/document/providers/factory.rsnew file mode 100644index 000000000..f4ee2c4c3--- /dev/null+++ b/apps/api/native/src/document/providers/factory.rs@@ -0,0 +1,37 @@+use super::docx::DocxProvider;+use super::odt::OdtProvider;+use super::rtf::RtfProvider;+use super::DocumentProvider;+use napi_derive::napi;++#[napi]+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum DocumentType {+ Docx,+ Rtf,+ Odt,+}++pub struct ProviderFactory {+ docx_provider: DocxProvider,+ rtf_provider: RtfProvider,+ odt_provider: OdtProvider,+}++impl ProviderFactory {+ pub fn new() -> Self {+ Self {+ docx_provider: DocxProvider::new(),+ rtf_provider: RtfProvider::new(),+ odt_provider: OdtProvider::new(),+ }+ }++ pub fn get_provider(&self, doc_type: DocumentType) -> &dyn DocumentProvider {+ match doc_type {+ DocumentType::Docx => &self.docx_provider,+ DocumentType::Rtf => &self.rtf_provider,+ DocumentType::Odt => &self.odt_provider,+ }+ }+}diff --git a/apps/api/native/src/document/providers/mod.rs b/apps/api/native/src/document/providers/mod.rsnew file mode 100644index 000000000..044b8b5c3--- /dev/null+++ b/apps/api/native/src/document/providers/mod.rs@@ -0,0 +1,14 @@+use crate::document::model::Document;+use std::error::Error;++pub mod docx;+pub mod factory;+pub mod odt;+pub mod rtf;++pub trait DocumentProvider {+ fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>>;++ #[allow(dead_code)]+ fn name(&self) -> &'static str;+}diff --git a/apps/api/native/src/document/providers/odt.rs b/apps/api/native/src/document/providers/odt.rsnew file mode 100644index 000000000..0d5cbc8c6--- /dev/null+++ b/apps/api/native/src/document/providers/odt.rs@@ -0,0 +1,764 @@+use crate::document::model::*;+use crate::document::providers::DocumentProvider;+use chrono::{DateTime, Utc};+use roxmltree::{Document as XmlDoc, Node};+use std::collections::HashMap;+use std::error::Error;+use std::io::{Read, Seek};+use std::num::NonZeroU32;+use zip::read::ZipArchive;++pub struct OdtProvider;++impl OdtProvider {+ pub fn new() -> Self {+ Self+ }+}++impl DocumentProvider for OdtProvider {+ fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {+ let cursor = std::io::Cursor::new(data);+ let mut zip = ZipArchive::new(cursor)?;++ let meta = read_meta(&mut zip).unwrap_or_default();+ let styles = read_styles(&mut zip);++ let content =+ read_zip_text(&mut zip, "content.xml").ok_or("Missing content.xml in document")?;+ let xml = XmlDoc::parse(strip_bom(&content))?;++ let mut notes: Vec<Note> = Vec::new();+ let mut comments: Vec<Comment> = Vec::new();+ let mut blocks: Vec<Block> = Vec::new();++ let body_text = xml+ .descendants()+ .find(|n| is_tag(n, "text") && n.ancestors().any(|a| is_tag(&a, "body")));++ if let Some(text_node) = body_text {+ blocks = parse_block_children_odt(&text_node, &styles, &mut notes, &mut comments, &mut zip);+ }++ Ok(Document {+ blocks,+ metadata: meta,+ notes,+ comments,+ })+ }++ fn name(&self) -> &'static str {+ "odt"+ }+}++fn read_zip_text<R: Read + Seek>(zip: &mut ZipArchive<R>, path: &str) -> Option<String> {+ let mut file = zip.by_name(path).ok()?;+ let mut s = String::new();+ file.read_to_string(&mut s).ok()?;+ Some(s)+}++fn strip_bom(s: &str) -> &str {+ const BOM: char = '\u{FEFF}';+ s.strip_prefix(BOM).unwrap_or(s)+}++#[derive(Debug, Default, Clone)]+struct OdtStylesInfo {+ paragraph_names: HashMap<String, String>,+ paragraph_outline_level: HashMap<String, u8>,+ paragraph_text_props: HashMap<String, TextStyleProps>,+ text_props: HashMap<String, TextStyleProps>,+ text_font_name: HashMap<String, String>,+ list_is_ordered: HashMap<String, bool>,+}++#[derive(Debug, Default, Clone, Copy)]+struct TextStyleProps {+ bold: bool,+ italic: bool,+ strike: bool,+ sup: bool,+ sub: bool,+ code: bool,+}++fn read_styles<R: Read + Seek>(zip: &mut ZipArchive<R>) -> OdtStylesInfo {+ let mut info = OdtStylesInfo::default();++ if let Some(t) = read_zip_text(zip, "styles.xml") {+ if let Ok(doc) = XmlDoc::parse(strip_bom(&t)) {+ harvest_styles_from_doc(&doc, &mut info);+ }+ }+ if let Some(t) = read_zip_text(zip, "content.xml") {+ if let Ok(doc) = XmlDoc::parse(strip_bom(&t)) {+ harvest_styles_from_doc(&doc, &mut info);+ }+ }+ info+}++fn harvest_styles_from_doc(doc: &XmlDoc, out: &mut OdtStylesInfo) {+ for s in doc.descendants().filter(|n| is_tag(n, "style")) {+ let Some(family) = get_attr_local(&s, "family") else {+ continue;+ };+ let Some(name) = get_attr_local(&s, "name") else {+ continue;+ };+ let lname = name.to_string();++ if family == "paragraph" {+ out.paragraph_names.insert(lname.clone(), lname.clone());++ if let Some(ppr) = child(&s, "paragraph-properties") {+ if let Some(ol) = get_attr_local(&ppr, "outline-level") {+ if let Ok(v) = ol.parse::<u8>() {+ out.paragraph_outline_level.insert(lname.clone(), v.min(6));+ }+ }+ }++ if !out.paragraph_outline_level.contains_key(&lname) {+ if let Some(parent) = get_attr_local(&s, "parent-style-name") {+ if let Some(lv) = parse_odt_heading_level(parent) {+ out.paragraph_outline_level.insert(lname.clone(), lv);+ }+ }+ }++ if let Some(tp) = child(&s, "text-properties") {+ let mut props = parse_text_properties(&tp);+ if let Some(v) = get_attr_local(&tp, "font-name") {+ if v.to_ascii_lowercase().contains("courier") || v.to_ascii_lowercase().contains("mono") {+ props.code = true;+ }+ out.text_font_name.insert(lname.clone(), v.to_string());+ }+ out.paragraph_text_props.insert(lname.clone(), props);+ }+ } else if family == "text" {+ if let Some(tp) = child(&s, "text-properties") {+ let mut props = parse_text_properties(&tp);+ if let Some(v) = get_attr_local(&tp, "font-name") {+ if v.to_ascii_lowercase().contains("courier") || v.to_ascii_lowercase().contains("mono") {+ props.code = true;+ }+ out.text_font_name.insert(lname.clone(), v.to_string());+ }+ out.text_props.insert(lname.clone(), props);+ }+ } else if family == "list" {+ let is_ordered = s+ .children()+ .filter(|n| n.is_element())+ .any(|child_n| is_tag(&child_n, "list-level-style-number"));+ out.list_is_ordered.insert(lname.clone(), is_ordered);+ }+ }++ for ls in doc.descendants().filter(|n| is_tag(n, "list-style")) {+ if let Some(name) = get_attr_local(&ls, "name") {+ let is_ordered = ls+ .children()+ .filter(|n| n.is_element())+ .any(|c| is_tag(&c, "list-level-style-number"));+ out.list_is_ordered.insert(name.to_string(), is_ordered);+ }+ }+}++fn parse_text_properties(tp: &Node) -> TextStyleProps {+ let mut props = TextStyleProps::default();++ if let Some(v) = get_attr_local(tp, "font-weight") {+ if v.eq_ignore_ascii_case("bold") {+ props.bold = true;+ }+ }+ if let Some(v) = get_attr_local(tp, "font-style") {+ if v.eq_ignore_ascii_case("italic") {+ props.italic = true;+ }+ }+ if let Some(v) = get_attr_local(tp, "text-line-through-type")+ .or_else(|| get_attr_local(tp, "text-line-through-style"))+ {+ if v != "none" {+ props.strike = true;+ }+ }+ if let Some(v) = get_attr_local(tp, "text-position") {+ let lv = v.to_ascii_lowercase();+ if lv.contains("sup") || lv.contains("super") {+ props.sup = true;+ } else if lv.contains("sub") {+ props.sub = true;+ }+ }+ props+}++fn read_meta<R: Read + Seek>(zip: &mut ZipArchive<R>) -> Option<DocumentMetadata> {+ let text = read_zip_text(zip, "meta.xml")?;+ let xml = XmlDoc::parse(strip_bom(&text)).ok()?;+ let mut meta = DocumentMetadata::default();++ if let Some(title) = xml+ .descendants()+ .find(|n| is_tag(n, "title"))+ .and_then(|n| n.text())+ {+ if !title.trim().is_empty() {+ meta.title = Some(title.to_string());+ }+ }++ if let Some(author) = xml+ .descendants()+ .find(|n| is_tag(n, "creator"))+ .and_then(|n| n.text())+ .or_else(|| {+ xml+ .descendants()+ .find(|n| is_tag(n, "initial-creator"))+ .and_then(|n| n.text())+ })+ {+ let trimmed = author.trim();+ if !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("unknown") {+ meta.author = Some(trimmed.to_string());+ }+ }++ if let Some(created) = xml+ .descendants()+ .find(|n| is_tag(n, "creation-date"))+ .and_then(|n| n.text())+ {+ if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(created) {+ meta.created = Some(DateTime::<Utc>::from(dt));+ }+ }++ Some(meta)+}++fn is_tag(node: &Node, local: &str) -> bool {+ node.is_element() && node.tag_name().name() == local+}++fn get_attr_local<'a>(node: &Node<'a, 'a>, local: &str) -> Option<&'a str> {+ node+ .attributes()+ .find(|a| {+ let name = a.name();+ match name.rsplit_once(':') {+ Some((_, l)) => l == local,+ None => name == local,+ }+ })+ .map(|a| a.value())+}++fn child<'a>(node: &Node<'a, 'a>, local: &str) -> Option<Node<'a, 'a>> {+ node+ .children()+ .find(|n| n.is_element() && n.tag_name().name() == local)+}++fn children<'a, 'b>(+ node: &Node<'a, 'a>,+ local: &'b str,+) -> impl Iterator<Item = Node<'a, 'a>> + use<'a, 'b> {+ node+ .children()+ .filter(move |n| n.is_element() && n.tag_name().name() == local)+}++fn parse_block_children_odt<R: Read + Seek>(+ node: &Node,+ styles: &OdtStylesInfo,+ notes: &mut Vec<Note>,+ comments: &mut Vec<Comment>,+ zip: &mut ZipArchive<R>,+) -> Vec<Block> {+ let mut blocks: Vec<Block> = Vec::new();++ for child_n in node.children().filter(|n| n.is_element()) {+ if is_tag(&child_n, "h") {+ if let Some(p) = parse_paragraph(&child_n, styles, notes, comments) {+ if paragraph_has_visible_content(&p) {+ blocks.push(Block::Paragraph(p));+ }+ }+ } else if is_tag(&child_n, "p") {+ if let Some(img) = image_from_paragraph(&child_n, zip) {+ blocks.push(Block::Image(img));+ } else if let Some(p) = parse_paragraph(&child_n, styles, notes, comments) {+ if paragraph_has_visible_content(&p) {+ blocks.push(Block::Paragraph(p));+ }+ }+ } else if is_tag(&child_n, "list") {+ let mut effective = child_n;+ let mut inherited_style_name = get_attr_local(&effective, "style-name");+ let mut unwrapped = false;++ while let Some(inner) = unwrap_single_nested_list(&effective) {+ effective = inner;+ unwrapped = true;+ if inherited_style_name.is_none() {+ inherited_style_name = get_attr_local(&effective, "style-name");+ }+ }++ if is_heading_list(&effective) {+ for li in children(&effective, "list-item") {+ if let Some(h) = li.descendants().find(|n| is_tag(n, "h")) {+ if let Some(p) = parse_paragraph(&h, styles, notes, comments) {+ if paragraph_has_visible_content(&p) {+ blocks.push(Block::Paragraph(p));+ }+ }+ }+ }+ } else if let Some(l) = parse_list_with_inherit(+ &effective,+ styles,+ notes,+ comments,+ zip,+ inherited_style_name,+ ) {+ if unwrapped {+ if let Some(Block::List(prev)) = blocks.last_mut() {+ if let Some(last_item) = prev.items.last_mut() {+ last_item.blocks.push(Block::List(l));+ continue;+ }+ }+ }+ blocks.push(Block::List(l));+ }+ } else if is_tag(&child_n, "table") {+ if let Some(t) = parse_table(&child_n, styles, notes, comments, zip) {+ blocks.push(Block::Table(t));+ }+ } else {+ let mut inner = parse_block_children_odt(&child_n, styles, notes, comments, zip);+ blocks.append(&mut inner);+ }+ }++ blocks+}++fn unwrap_single_nested_list<'a>(list: &Node<'a, 'a>) -> Option<Node<'a, 'a>> {+ let mut li_iter = children(list, "list-item");+ let first_li = li_iter.next()?;+ if li_iter.next().is_some() {+ return None;+ }+ let mut inner_lists = first_li.children().filter(|n| is_tag(n, "list"));+ let inner = inner_lists.next()?;+ if inner_lists.next().is_some() {+ return None;+ }+ Some(inner)+}++fn is_heading_list(list: &Node) -> bool {+ let mut any = false;+ for li in children(list, "list-item") {+ any = true;+ if !li.descendants().any(|n| is_tag(&n, "h")) {+ return false;+ }+ }+ any+}++fn parse_paragraph(+ node: &Node,+ styles: &OdtStylesInfo,+ notes: &mut Vec<Note>,+ comments: &mut Vec<Comment>,+) -> Option<Paragraph> {+ let kind = paragraph_kind(node, styles);+ let base = paragraph_text_props(node, styles);+ let inlines = parse_inlines_with_base(node, styles, notes, comments, base);+ Some(Paragraph { kind, inlines })+}++fn paragraph_kind(p: &Node, styles: &OdtStylesInfo) -> ParagraphKind {+ if p.tag_name().name() == "h" {+ if let Some(ol) = get_attr_local(p, "outline-level") {+ if let Ok(v) = ol.parse::<u8>() {+ return ParagraphKind::Heading(v.min(6));+ }+ }+ return ParagraphKind::Heading(1);+ }++ if let Some(style_name) = get_attr_local(p, "style-name") {+ if let Some(lvl) = styles.paragraph_outline_level.get(style_name) {+ return ParagraphKind::Heading((*lvl).min(6));+ }++ let name = styles+ .paragraph_names+ .get(style_name)+ .map(|s| s.to_ascii_lowercase())+ .unwrap_or_default();+ if name.contains("quote") {+ return ParagraphKind::Blockquote;+ }+ }++ ParagraphKind::Normal+}++fn parse_odt_heading_level(style_name: &str) -> Option<u8> {+ let normalized = style_name.replace("_20_", " ").replace('_', " ");+ let lower = normalized.to_ascii_lowercase();+ if lower.contains("title") {+ return Some(1);+ }++ if let Some(idx) = lower.find("heading") {+ let tail = &lower[idx + "heading".len()..];+ let num: String = tail.chars().filter(|c| c.is_ascii_digit()).collect();+ if let Ok(n) = num.parse::<u8>() {+ return Some(n.clamp(1, 6));+ }+ }+ None+}++fn paragraph_text_props(node: &Node, styles: &OdtStylesInfo) -> TextStyleProps {+ if let Some(style_name) = get_attr_local(node, "style-name") {+ if let Some(p) = styles.paragraph_text_props.get(style_name) {+ return *p;+ }+ }+ TextStyleProps::default()+}++fn parse_inlines(+ node: &Node,+ styles: &OdtStylesInfo,+ notes: &mut Vec<Note>,+ comments: &mut Vec<Comment>,+) -> Vec<Inline> {+ let mut out: Vec<Inline> = Vec::new();++ for c in node.children() {+ if c.is_text() {+ if let Some(t) = c.text() {+ if !t.is_empty() {+ out.push(Inline::Text(t.to_string()));+ }+ }+ continue;+ }+ if !c.is_element() {+ continue;+ }++ if is_tag(&c, "span") {+ let mut inner = parse_inlines(&c, styles, notes, comments);+ let sname = get_attr_local(&c, "style-name").map(|s| s.to_string());+ inner = apply_text_style_wrappers(inner, sname.as_deref(), styles, TextStyleProps::default());+ out.extend(inner);+ } else if is_tag(&c, "a") {+ if let Some(href) = get_attr_local(&c, "href") {+ let children = parse_inlines(&c, styles, notes, comments);+ out.push(Inline::Link {+ href: href.to_string(),+ children,+ });+ } else {+ out.extend(parse_inlines(&c, styles, notes, comments));+ }+ } else if is_tag(&c, "line-break") {+ out.push(Inline::LineBreak);+ } else if is_tag(&c, "s") {+ let count = get_attr_local(&c, "c")+ .and_then(|v| v.parse::<usize>().ok())+ .unwrap_or(1);+ out.push(Inline::Text(" ".repeat(count)));+ } else if is_tag(&c, "tab") {+ out.push(Inline::Text("\t".to_string()));+ } else if is_tag(&c, "bookmark-start") {+ if let Some(name) = get_attr_local(&c, "name") {+ out.push(Inline::Bookmark(BookmarkId(name.to_string())));+ }+ } else if is_tag(&c, "note") {+ let kind = match get_attr_local(&c, "note-class") {+ Some("endnote") => NoteKind::Endnote,+ _ => NoteKind::Footnote,+ };+ let id = get_attr_local(&c, "id")+ .map(|s| s.to_string())+ .unwrap_or_else(|| format!("odt-note-{}", notes.len() + 1));+ let body = child(&c, "note-body");+ let mut blocks: Vec<Block> = Vec::new();+ if let Some(b) = body {+ blocks = parse_note_body_blocks(&b, styles, notes, comments);+ }+ notes.push(Note {+ id: NoteId(id.clone()),+ kind,+ blocks,+ });+ match kind {+ NoteKind::Footnote => out.push(Inline::FootnoteRef(NoteId(id))),+ NoteKind::Endnote => out.push(Inline::EndnoteRef(NoteId(id))),+ }+ } else if is_tag(&c, "annotation") {+ let cid = format!("odt-comment-{}", comments.len() + 1);+ let mut author: Option<String> = None;+ let mut initials: Option<String> = None;++ if let Some(a) = c+ .descendants()+ .find(|n| is_tag(n, "creator"))+ .and_then(|n| n.text())+ {+ if !a.trim().is_empty() {+ author = Some(a.to_string());+ }+ }+ if let Some(init) = c+ .descendants()+ .find(|n| is_tag(n, "initials"))+ .and_then(|n| n.text())+ {+ if !init.trim().is_empty() {+ initials = Some(init.to_string());+ }+ }++ let mut cblocks: Vec<Block> = Vec::new();+ for p in c.children().filter(|n| is_tag(n, "p")) {+ let inl = parse_inlines(&p, styles, notes, comments);+ if !inl.is_empty() {+ cblocks.push(Block::Paragraph(Paragraph {+ kind: ParagraphKind::Normal,+ inlines: inl,+ }));+ }+ }+ comments.push(Comment {+ id: CommentId(cid.clone()),+ author_name: author,+ author_initials: initials,+ blocks: cblocks,+ });+ out.push(Inline::CommentRef(CommentId(cid)));+ } else {+ out.extend(parse_inlines(&c, styles, notes, comments));+ }+ }++ out+}++fn parse_inlines_with_base(+ node: &Node,+ styles: &OdtStylesInfo,+ notes: &mut Vec<Note>,+ comments: &mut Vec<Comment>,+ base: TextStyleProps,+) -> Vec<Inline> {+ let mut inlines = parse_inlines(node, styles, notes, comments);+ inlines = apply_text_style_wrappers(inlines, None, styles, base);+ inlines+}++fn apply_text_style_wrappers(+ mut inlines: Vec<Inline>,+ style_name: Option<&str>,+ styles: &OdtStylesInfo,+ base: TextStyleProps,+) -> Vec<Inline> {+ let mut props = base;+ if let Some(name) = style_name {+ let lower = name.to_ascii_lowercase();+ let mut sprops = styles.text_props.get(name).copied().unwrap_or_default();+ if lower.contains("code")+ || styles+ .text_font_name+ .get(name)+ .map(|f| {+ f.to_ascii_lowercase().contains("courier") || f.to_ascii_lowercase().contains("mono")+ })+ .unwrap_or(false)+ {+ sprops.code = true;+ }+ props.bold |= sprops.bold;+ props.italic |= sprops.italic;+ props.strike |= sprops.strike;+ props.sup |= sprops.sup;+ props.sub |= sprops.sub;+ props.code |= sprops.code;+ }++ if props.code {+ let code_text: String = inlines+ .iter()+ .filter_map(|i| match i {+ Inline::Text(s) => Some(s.as_str()),+ _ => None,+ })+ .collect();+ if !code_text.is_empty() {+ return vec![Inline::Code(code_text)];+ }+ }++ if props.strike {+ inlines = vec![Inline::Del(inlines)];+ }+ if props.italic {+ inlines = vec![Inline::Em(inlines)];+ }+ if props.bold {+ inlines = vec![Inline::Strong(inlines)];+ }+ if props.sup {+ inlines = vec![Inline::Sup(inlines)];+ }+ if props.sub {+ inlines = vec![Inline::Sub(inlines)];+ }+ inlines+}++fn parse_note_body_blocks(+ node: &Node,+ styles: &OdtStylesInfo,+ notes: &mut Vec<Note>,+ comments: &mut Vec<Comment>,+) -> Vec<Block> {+ let mut blocks = Vec::new();+ for p in node.children().filter(|n| is_tag(n, "p") || is_tag(n, "h")) {+ let kind = paragraph_kind(&p, styles);+ let base = paragraph_text_props(&p, styles);+ let inl = parse_inlines_with_base(&p, styles, notes, comments, base);+ if inlines_have_visible_content(&inl) {+ blocks.push(Block::Paragraph(Paragraph { kind, inlines: inl }));+ }+ }+ blocks+}++fn paragraph_has_visible_content(p: &Paragraph) -> bool {+ inlines_have_visible_content(&p.inlines)+}++fn inlines_have_visible_content(inlines: &[Inline]) -> bool {+ inlines.iter().any(inline_is_visible)+}++fn inline_is_visible(i: &Inline) -> bool {+ match i {+ Inline::Text(t) => !t.trim().is_empty(),+ Inline::LineBreak => false,+ Inline::Link { children, .. } => inlines_have_visible_content(children),+ Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {+ inlines_have_visible_content(c)+ }+ Inline::Code(c) => !c.trim().is_empty(),+ Inline::FootnoteRef(_) | Inline::EndnoteRef(_) | Inline::CommentRef(_) => true,+ Inline::Bookmark(_) => false,+ }+}++fn parse_list_with_inherit<R: Read + Seek>(+ node: &Node,+ styles: &OdtStylesInfo,+ notes: &mut Vec<Note>,+ comments: &mut Vec<Comment>,+ zip: &mut ZipArchive<R>,+ inherit_style_name: Option<&str>,+) -> Option<List> {+ let style_name = get_attr_local(node, "style-name").or(inherit_style_name);+ let list_type = match style_name+ .and_then(|n| styles.list_is_ordered.get(n))+ .copied()+ {+ Some(true) => ListType::Ordered,+ Some(false) => ListType::Unordered,+ None => ListType::Unordered,+ };++ let mut items: Vec<ListItem> = Vec::new();+ for it in children(node, "list-item") {+ let mut blocks = Vec::new();+ let mut inner = parse_block_children_odt(&it, styles, notes, comments, zip);+ blocks.append(&mut inner);+ items.push(ListItem { blocks });+ }+ Some(List { items, list_type })+}++fn parse_table<R: Read + Seek>(+ node: &Node,+ styles: &OdtStylesInfo,+ notes: &mut Vec<Note>,+ comments: &mut Vec<Comment>,+ zip: &mut ZipArchive<R>,+) -> Option<Table> {+ let mut rows: Vec<TableRow> = Vec::new();+ for tr in children(node, "table-row") {+ let mut cells: Vec<TableCell> = Vec::new();+ for tc in children(&tr, "table-cell") {+ let mut blocks = parse_block_children_odt(&tc, styles, notes, comments, zip);+ let colspan = get_attr_local(&tc, "number-columns-spanned")+ .and_then(|v| v.parse::<u32>().ok())+ .and_then(NonZeroU32::new)+ .unwrap_or_else(|| NonZeroU32::new(1).unwrap());+ let rowspan = get_attr_local(&tc, "number-rows-spanned")+ .and_then(|v| v.parse::<u32>().ok())+ .and_then(NonZeroU32::new)+ .unwrap_or_else(|| NonZeroU32::new(1).unwrap());+ cells.push(TableCell {+ blocks: std::mem::take(&mut blocks),+ colspan,+ rowspan,+ });+ }+ rows.push(TableRow {+ cells,+ kind: TableRowKind::Body,+ });+ }+ Some(Table { rows })+}++fn image_from_paragraph<R: Read + Seek>(p: &Node, zip: &mut ZipArchive<R>) -> Option<Image> {+ let img = p.descendants().find(|n| is_tag(n, "image"))?;+ let href = get_attr_local(&img, "href")?;+ image_from_href(href, zip, None)+}++fn image_from_href<R: Read + Seek>(+ href: &str,+ _zip: &mut ZipArchive<R>,+ alt: Option<String>,+) -> Option<Image> {+ // only include external images (http/https URLs)+ if href.starts_with("http://") || href.starts_with("https://") {+ return Some(Image {+ src: href.to_string(),+ alt,+ });+ }+ None+}diff --git a/apps/api/native/src/document/providers/rtf.rs b/apps/api/native/src/document/providers/rtf.rsnew file mode 100644index 000000000..5a49d20be--- /dev/null+++ b/apps/api/native/src/document/providers/rtf.rs@@ -0,0 +1,764 @@+use crate::document::model::*;+use crate::document::providers::DocumentProvider;+use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc};+use std::error::Error;+use std::num::NonZeroU32;++pub struct RtfProvider;++impl RtfProvider {+ pub fn new() -> Self {+ Self+ }+}++impl DocumentProvider for RtfProvider {+ fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {+ let metadata = extract_metadata_from_info(data).unwrap_or_default();+ let blocks = parse_rtf_body_to_blocks(data);++ Ok(Document {+ blocks,+ metadata,+ notes: Vec::new(),+ comments: Vec::new(),+ })+ }++ fn name(&self) -> &'static str {+ "rtf"+ }+}++fn extract_metadata_from_info(src: &[u8]) -> Option<DocumentMetadata> {+ let start = find_group_start(src, b"{\\info")?;+ let end = find_matching_brace(src, start)?;+ let info = &src[start..end];++ let mut meta = DocumentMetadata::default();++ if let Some(author) = extract_simple_text_dest(info, br"{\author") {+ if !author.eq_ignore_ascii_case("unknown") {+ meta.author = Some(author);+ }+ }++ if let Some(title) = extract_simple_text_dest(info, br"{\title") {+ if !title.trim().is_empty() {+ meta.title = Some(title);+ }+ }++ if let Some(created) = extract_creatim(info) {+ meta.created = Some(created);+ }++ Some(meta)+}++fn find_group_start(buf: &[u8], needle: &[u8]) -> Option<usize> {+ buf.windows(needle.len()).position(|w| w == needle)+}++fn find_matching_brace(buf: &[u8], start: usize) -> Option<usize> {+ let mut depth = 0usize;+ for (i, &b) in buf[start..].iter().enumerate() {+ match b {+ b'{' => depth += 1,+ b'}' => {+ depth -= 1;+ if depth == 0 {+ return Some(start + i + 1);+ }+ }+ _ => {}+ }+ }+ None+}++fn extract_simple_text_dest(buf: &[u8], start_tag: &[u8]) -> Option<String> {+ let s = find_group_start(buf, start_tag)?;+ let e = find_matching_brace(buf, s)?;+ let mut out = String::new();+ for &b in &buf[s + start_tag.len()..e - 1] {+ push_byte_as_text(b, &mut out);+ }+ if out.trim().is_empty() {+ None+ } else {+ Some(out.trim().to_string())+ }+}++fn extract_creatim(buf: &[u8]) -> Option<DateTime<Utc>> {+ let s = find_group_start(buf, br"{\creatim")?;+ let e = find_matching_brace(buf, s)?;+ let g = &buf[s..e];++ let mut yr: Option<i32> = None;+ let mut mo: Option<u32> = None;+ let mut dy: Option<u32> = None;+ let mut hr: Option<u32> = None;+ let mut mi: Option<u32> = None;++ let mut i = 0usize;+ while i < g.len() {+ if g[i] == b'\\' {+ if let Some((word, val, ni)) = read_control_word(g, i + 1) {+ match word.as_str() {+ "yr" => yr = val,+ "mo" => mo = val.map(|v| v as u32),+ "dy" => dy = val.map(|v| v as u32),+ "hr" => hr = val.map(|v| v as u32),+ "min" => mi = val.map(|v| v as u32),+ _ => {}+ }+ i = ni;+ continue;+ }+ }+ i += 1;+ }++ let date = NaiveDate::from_ymd_opt(yr?, mo?, dy?)?;+ let time = chrono::NaiveTime::from_hms_opt(hr.unwrap_or(0), mi.unwrap_or(0), 0)?;+ let dt = NaiveDateTime::new(date, time);+ Some(DateTime::<Utc>::from_naive_utc_and_offset(dt, Utc))+}++#[derive(Default)]+struct TableBuilder {+ rows: Vec<TableRow>,+ current_row: Vec<TableCell>,+ current_cell_blocks: Vec<Block>,+}++impl TableBuilder {+ fn start_row(&mut self) {+ self.current_cell_blocks.clear();+ self.current_row.clear();+ }++ fn push_block(&mut self, block: Block) {+ self.current_cell_blocks.push(block);+ }++ fn finish_cell(&mut self) {+ if self.current_cell_blocks.is_empty() {+ return;+ }+ let cell = TableCell {+ blocks: std::mem::take(&mut self.current_cell_blocks),+ colspan: NonZeroU32::new(1).unwrap(),+ rowspan: NonZeroU32::new(1).unwrap(),+ };+ self.current_row.push(cell);+ }++ fn finish_row(&mut self) {+ self.finish_cell();+ if self.current_row.is_empty() {+ return;+ }+ let row = TableRow {+ cells: std::mem::take(&mut self.current_row),+ kind: TableRowKind::Body,+ };+ self.rows.push(row);+ }++ fn finalize(mut self) -> Option<Block> {+ self.finish_row();+ if self.rows.is_empty() {+ None+ } else {+ Some(Block::Table(Table { rows: self.rows }))+ }+ }+}++fn push_block_target(+ block: Block,+ blocks: &mut Vec<Block>,+ table: &mut Option<TableBuilder>,+ in_table_cell: bool,+) {+ if in_table_cell {+ if let Some(builder) = table.as_mut() {+ builder.push_block(block);+ } else {+ blocks.push(block);+ }+ } else {+ if let Some(builder) = table.take() {+ if let Some(table_block) = builder.finalize() {+ blocks.push(table_block);+ }+ }+ blocks.push(block);+ }+}++fn flush_table(blocks: &mut Vec<Block>, table: &mut Option<TableBuilder>) {+ if let Some(builder) = table.take() {+ if let Some(block) = builder.finalize() {+ blocks.push(block);+ }+ }+}++fn parse_rtf_body_to_blocks(src: &[u8]) -> Vec<Block> {+ let mut p = 0usize;+ let n = src.len();++ #[derive(Clone, Default, Debug, PartialEq, Eq)]+ struct State {+ bold: bool,+ italic: bool,+ strike: bool,+ sup: bool,+ sub: bool,+ }++ #[derive(Clone)]+ struct Group {+ saved: State,+ skip: bool,+ name_seen: bool,+ }++ let mut state = State::default();+ let mut stack: Vec<Group> = Vec::new();+ let mut blocks: Vec<Block> = Vec::new();+ let mut cur_inlines: Vec<Inline> = Vec::new();+ let mut text_buf = String::new();+ let mut table_builder: Option<TableBuilder> = None;+ let mut in_table_cell = false;+ let mut uc_skip: usize = 1;+ let mut pending_uc_skip: usize = 0;++ const SKIP_DESTS: &[&str] = &[+ "fonttbl",+ "colortbl",+ "stylesheet",+ "listtable",+ "listoverridetable",+ "themedata",+ "latentstyles",+ "rsidtbl",+ "xmlnstbl",+ "mmathPr",+ "wgrffmtfilter",+ "datastore",+ "filetbl",+ "colorschememapping",+ "pnseclvl1",+ "pnseclvl2",+ "pnseclvl3",+ "pnseclvl4",+ "pnseclvl5",+ "pnseclvl6",+ "pnseclvl7",+ "pnseclvl8",+ "pnseclvl9",+ "pict",+ "object",+ "info",+ ];++ fn style_wrap(mut node: Inline, st: &State) -> Inline {+ if st.strike {+ node = Inline::Del(vec![node]);+ }+ if st.italic {+ node = Inline::Em(vec![node]);+ }+ if st.bold {+ node = Inline::Strong(vec![node]);+ }+ if st.sup {+ node = Inline::Sup(vec![node]);+ } else if st.sub {+ node = Inline::Sub(vec![node]);+ }+ node+ }++ fn push_text_buf(text_buf: &mut String, cur: &mut Vec<Inline>, st: &State) {+ if !text_buf.is_empty() {+ let node = style_wrap(Inline::Text(text_buf.clone()), st);+ cur.push(node);+ text_buf.clear();+ }+ }++ fn has_visible_content(inlines: &[Inline]) -> bool {+ inlines.iter().any(|i| match i {+ Inline::Text(t) => !t.trim().is_empty(),+ Inline::LineBreak => false,+ Inline::Link { children, .. } => has_visible_content(children),+ Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {+ has_visible_content(c)+ }+ Inline::Code(t) => !t.trim().is_empty(),+ Inline::FootnoteRef(_) | Inline::EndnoteRef(_) | Inline::CommentRef(_) => true,+ Inline::Bookmark(_) => false,+ })+ }++ fn flush_paragraph(+ cur: &mut Vec<Inline>,+ text_buf: &mut String,+ blocks: &mut Vec<Block>,+ table: &mut Option<TableBuilder>,+ st: &State,+ in_table_cell: bool,+ ) {+ push_text_buf(text_buf, cur, st);+ if has_visible_content(cur) {+ let block = Block::Paragraph(Paragraph {+ kind: ParagraphKind::Normal,+ inlines: std::mem::take(cur),+ });+ push_block_target(block, blocks, table, in_table_cell);+ } else {+ cur.clear();+ if !in_table_cell {+ flush_table(blocks, table);+ }+ }+ }++ let flush_before_change = |text_buf: &mut String, cur: &mut Vec<Inline>, st: &State| {+ push_text_buf(text_buf, cur, st);+ };++ while p < n {+ match src[p] {+ b'{' => {+ let inherited_skip = stack.last().map(|g| g.skip).unwrap_or(false);+ stack.push(Group {+ saved: state.clone(),+ skip: inherited_skip,+ name_seen: false,+ });+ p += 1;+ }+ b'}' => {+ if let Some(g) = stack.last() {+ if !g.skip {+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);+ }+ }+ if let Some(g) = stack.pop() {+ state = g.saved;+ }+ p += 1;+ }+ b'\\' => {+ if p + 1 >= n {+ break;+ }+ let next = src[p + 1];++ if next == b'\\' || next == b'{' || next == b'}' {+ if !stack.last().map(|g| g.skip).unwrap_or(false) {+ text_buf.push(next as char);+ }+ p += 2;+ continue;+ }++ if next == b'\'' {+ if p + 3 < n {+ let h1 = src[p + 2];+ let h2 = src[p + 3];+ if !stack.last().map(|g| g.skip).unwrap_or(false) {+ if let (Some(a), Some(b)) = (hex_val(h1), hex_val(h2)) {+ let byte = (a << 4) | b;+ push_byte_as_text(byte, &mut text_buf);+ }+ }+ p += 4;+ continue;+ } else {+ break;+ }+ }++ let skip = stack.last().map(|g| g.skip).unwrap_or(false);++ match next {+ b'~' => {+ if !skip {+ text_buf.push('\u{00A0}');+ } // non-breaking space+ p += 2;+ continue;+ }+ b'-' => {+ if !skip {+ text_buf.push('\u{00AD}');+ } // soft hyphen+ p += 2;+ continue;+ }+ _ => {}+ }++ if starts_with_word(src, p + 1, b"rquote") {+ if !skip {+ text_buf.push('\u{2019}');+ } // '+ p = skip_word_and_space(src, p + 1);+ continue;+ }+ if starts_with_word(src, p + 1, b"lquote") {+ if !skip {+ text_buf.push('\u{2018}');+ } // '+ p = skip_word_and_space(src, p + 1);+ continue;+ }+ if starts_with_word(src, p + 1, b"rdblquote") {+ if !skip {+ text_buf.push('\u{201D}');+ } // "+ p = skip_word_and_space(src, p + 1);+ continue;+ }+ if starts_with_word(src, p + 1, b"ldblquote") {+ if !skip {+ text_buf.push('\u{201C}');+ } // "+ p = skip_word_and_space(src, p + 1);+ continue;+ }+ if starts_with_word(src, p + 1, b"emdash") {+ if !skip {+ text_buf.push('\u{2014}');+ } // —+ p = skip_word_and_space(src, p + 1);+ continue;+ }+ if starts_with_word(src, p + 1, b"endash") {+ if !skip {+ text_buf.push('\u{2013}');+ } // –+ p = skip_word_and_space(src, p + 1);+ continue;+ }+ if starts_with_word(src, p + 1, b"bullet") {+ if !skip {+ text_buf.push('\u{2022}');+ } // •+ p = skip_word_and_space(src, p + 1);+ continue;+ }+ if starts_with_word(src, p + 1, b"line") {+ if !skip {+ push_text_buf(&mut text_buf, &mut cur_inlines, &state);+ cur_inlines.push(Inline::LineBreak);+ }+ p = skip_word_and_space(src, p + 1);+ continue;+ }+ if starts_with_word(src, p + 1, b"tab") {+ if !skip {+ text_buf.push('\t');+ }+ p = skip_word_and_space(src, p + 1);+ continue;+ }++ if let Some((word, val, new_p)) = read_control_word(src, p + 1) {+ if let Some(g) = stack.last_mut() {+ if !g.name_seen {+ g.name_seen = true;+ if word == "*" || SKIP_DESTS.contains(&word.as_str()) {+ g.skip = true;+ }+ }+ }++ let skipping = stack.last().map(|g| g.skip).unwrap_or(false);++ if !skipping {+ match word.as_str() {+ "trowd" => {+ let builder = table_builder.get_or_insert_with(TableBuilder::default);+ builder.start_row();+ in_table_cell = false;+ }+ "intbl" => {+ in_table_cell = true;+ }+ "cell" => {+ flush_paragraph(+ &mut cur_inlines,+ &mut text_buf,+ &mut blocks,+ &mut table_builder,+ &state,+ true,+ );+ if let Some(builder) = table_builder.as_mut() {+ builder.finish_cell();+ }+ in_table_cell = false;+ }+ "row" => {+ if let Some(builder) = table_builder.as_mut() {+ builder.finish_row();+ }+ in_table_cell = false;+ }+ "cellx" | "clvertalb" | "clvertalc" | "clvertalt" => {}+ "b" => {+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);+ state.bold = val.map(|v| v != 0).unwrap_or(true);+ }+ "i" => {+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);+ state.italic = val.map(|v| v != 0).unwrap_or(true);+ }+ "strike" | "striked" | "striked1" => {+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);+ state.strike = val.map(|v| v != 0).unwrap_or(true);+ }+ "super" => {+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);+ state.sup = val.map(|v| v != 0).unwrap_or(true);+ if state.sup {+ state.sub = false;+ }+ }+ "sub" => {+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);+ state.sub = val.map(|v| v != 0).unwrap_or(true);+ if state.sub {+ state.sup = false;+ }+ }+ "nosupersub" => {+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);+ state.sup = false;+ state.sub = false;+ }+ "plain" => {+ flush_before_change(&mut text_buf, &mut cur_inlines, &state);+ state = State::default();+ }+ "par" => {+ flush_paragraph(+ &mut cur_inlines,+ &mut text_buf,+ &mut blocks,+ &mut table_builder,+ &state,+ in_table_cell,+ );+ }+ "uc" => {+ uc_skip = val.unwrap_or(1).max(0) as usize;+ }+ "u" => {+ if let Some(mut num) = val {+ if num < 0 {+ num += 65536;+ }+ if let Some(ch) = std::char::from_u32(num as u32) {+ text_buf.push(ch);+ }+ pending_uc_skip = uc_skip;+ }+ }+ _ => {}+ }+ } else if word == "par" {+ flush_paragraph(+ &mut cur_inlines,+ &mut text_buf,+ &mut blocks,+ &mut table_builder,+ &state,+ in_table_cell,+ );+ }++ let mut final_p = new_p;+ if pending_uc_skip > 0 {+ let mut k = 0usize;+ while k < pending_uc_skip && final_p < n {+ if matches!(src[final_p], b'\\' | b'{' | b'}') {+ break;+ }+ final_p += 1;+ k += 1;+ }+ pending_uc_skip = 0;+ }+ p = final_p;+ continue;+ }+ p += 1;+ }+ b'\r' | b'\n' => {+ p += 1;+ }+ byte => {+ if !stack.last().map(|g| g.skip).unwrap_or(false) {+ if pending_uc_skip > 0 {+ pending_uc_skip -= 1;+ } else {+ push_byte_as_text(byte, &mut text_buf);+ }+ }+ p += 1;+ }+ }+ }++ if !text_buf.is_empty() || !cur_inlines.is_empty() {+ flush_paragraph(+ &mut cur_inlines,+ &mut text_buf,+ &mut blocks,+ &mut table_builder,+ &state,+ in_table_cell,+ );+ }++ flush_table(&mut blocks, &mut table_builder);++ blocks+}++fn read_control_word(src: &[u8], mut i: usize) -> Option<(String, Option<i32>, usize)> {+ if i >= src.len() {+ return None;+ }++ if src[i] == b'*' {+ i += 1;+ if i < src.len() && src[i] == b' ' {+ i += 1;+ }+ return Some(("*".to_string(), None, i));+ }++ let start = i;+ while i < src.len() && is_alpha(src[i]) {+ i += 1;+ }+ if i == start {+ return None;+ }+ let word = String::from_utf8_lossy(&src[start..i]).to_string();++ let mut sign = 1i32;+ let mut val: Option<i32> = None;+ if i < src.len() && (src[i] == b'-' || is_digit(src[i])) {+ if src[i] == b'-' {+ sign = -1;+ i += 1;+ }+ let num_start = i;+ while i < src.len() && is_digit(src[i]) {+ i += 1;+ }+ if i > num_start {+ let n = std::str::from_utf8(&src[num_start..i])+ .ok()?+ .parse::<i32>()+ .ok()?;+ val = Some(sign * n);+ }+ }++ if i < src.len() && src[i] == b' ' {+ i += 1;+ }++ Some((word, val, i))+}++#[inline]+fn is_alpha(b: u8) -> bool {+ b.is_ascii_uppercase() || b.is_ascii_lowercase()+}++#[inline]+fn is_digit(b: u8) -> bool {+ b.is_ascii_digit()+}++fn hex_val(b: u8) -> Option<u8> {+ match b {+ b'0'..=b'9' => Some(b - b'0'),+ b'a'..=b'f' => Some(10 + (b - b'a')),+ b'A'..=b'F' => Some(10 + (b - b'A')),+ _ => None,+ }+}++fn push_byte_as_text(byte: u8, text_buf: &mut String) {+ let ch = decode_cp1252(byte);+ let cp = ch as u32;+ if ch == '\t' || ch == '\u{00A0}' || cp >= 0x20 {+ text_buf.push(ch);+ }+}++fn decode_cp1252(b: u8) -> char {+ if b < 0x80 {+ return b as char;+ }+ match b {+ 0x80 => '\u{20AC}', // €+ 0x82 => '\u{201A}', // ‚+ 0x83 => '\u{0192}', // Æ’+ 0x84 => '\u{201E}', // „+ 0x85 => '\u{2026}', // …+ 0x86 => '\u{2020}', // â€+ 0x87 => '\u{2021}', // ‡+ 0x88 => '\u{02C6}', // ˆ+ 0x89 => '\u{2030}', // ‰+ 0x8A => '\u{0160}', // Å+ 0x8B => '\u{2039}', // ‹+ 0x8C => '\u{0152}', // Å’+ 0x8E => '\u{017D}', // Ž+ 0x91 => '\u{2018}', // '+ 0x92 => '\u{2019}', // '+ 0x93 => '\u{201C}', // "+ 0x94 => '\u{201D}', // "+ 0x95 => '\u{2022}', // •+ 0x96 => '\u{2013}', // –+ 0x97 => '\u{2014}', // —+ 0x98 => '\u{02DC}', // Ëœ+ 0x99 => '\u{2122}', // â„¢+ 0x9A => '\u{0161}', // Å¡+ 0x9B => '\u{203A}', // ›+ 0x9C => '\u{0153}', // Å“+ 0x9E => '\u{017E}', // ž+ 0x9F => '\u{0178}', // Ÿ+ _ => b as char,+ }+}++fn starts_with_word(src: &[u8], i: usize, word: &[u8]) -> bool {+ let end = i + word.len();+ end <= src.len() && &src[i..end] == word+}++fn skip_word_and_space(src: &[u8], mut i: usize) -> usize {+ while i < src.len() && is_alpha(src[i]) {+ i += 1;+ }+ if i < src.len() && src[i] == b' ' {+ i += 1;+ }+ i+}diff --git a/apps/api/native/src/document/renderers/html.rs b/apps/api/native/src/document/renderers/html.rsnew file mode 100644index 000000000..ea6582400--- /dev/null+++ b/apps/api/native/src/document/renderers/html.rs@@ -0,0 +1,237 @@+use crate::document::model::*;+use maud::{html, Markup, DOCTYPE};++pub struct HtmlRenderer;++impl HtmlRenderer {+ pub fn new() -> Self {+ Self+ }++ pub fn render(&self, document: &Document) -> String {+ let title = document.metadata.title.as_deref().unwrap_or("Document");++ let footnotes: Vec<&Note> = document+ .notes+ .iter()+ .filter(|n| matches!(n.kind, NoteKind::Footnote))+ .collect();++ let endnotes: Vec<&Note> = document+ .notes+ .iter()+ .filter(|n| matches!(n.kind, NoteKind::Endnote))+ .collect();++ let author = document.metadata.author.as_deref();+ let page: Markup = html! {+ (DOCTYPE)+ html lang="en" {+ head {+ meta charset="UTF-8";+ meta name="viewport" content="width=device-width, initial-scale=1.0";+ title { (title) }+ @if let Some(author) = author {+ meta name="author" content=(author);+ }+ }+ body {+ main { (self.render_blocks(&document.blocks)) }++ @if !footnotes.is_empty() {+ section id="footnotes" {+ h2 { "Footnotes" }+ @for footnote in &footnotes {+ div id={ "footnote-" (&footnote.id.0) } {+ (self.render_blocks(&footnote.blocks))+ }+ }+ }+ }++ @if !endnotes.is_empty() {+ section id="endnotes" {+ h2 { "Endnotes" }+ @for endnote in &endnotes {+ div id={ "endnote-" (&endnote.id.0) } {+ (self.render_blocks(&endnote.blocks))+ }+ }+ }+ }++ @if !document.comments.is_empty() {+ section id="comments" {+ h2 { "Comments" }+ @for comment in &document.comments {+ article id={ "comment-" (&comment.id.0) } {+ @if let Some(author) = &comment.author_name {+ header {+ (author)+ @if let Some(initials) = &comment.author_initials {+ " (" (initials) ")"+ }+ }+ }+ (self.render_blocks(&comment.blocks))+ }+ }+ }+ }+ }+ }+ };++ page.into_string()+ }++ fn render_blocks(&self, blocks: &[Block]) -> Markup {+ html! {+ @for b in blocks {+ @match b {+ Block::Paragraph(p) => { (self.render_paragraph(p)) }+ Block::Table(t) => { (self.render_table(t)) }+ Block::List(l) => { (self.render_list(l)) }+ Block::Image(i) => { (self.render_image(i)) }+ }+ }+ }+ }++ fn render_blocks_inline(&self, blocks: &[Block]) -> Markup {+ if blocks.len() == 1 {+ if let Block::Paragraph(p) = &blocks[0] {+ if matches!(p.kind, ParagraphKind::Normal) {+ return self.render_inlines(&p.inlines);+ }+ }+ }++ self.render_blocks(blocks)+ }++ fn render_paragraph(&self, p: &Paragraph) -> Markup {+ match p.kind {+ ParagraphKind::Normal => html! { p { (self.render_inlines(&p.inlines)) } },+ ParagraphKind::Blockquote => html! {+ blockquote { p { (self.render_inlines(&p.inlines)) } }+ },+ ParagraphKind::Heading(level) => match level {+ 1 => html! { h1 { (self.render_inlines(&p.inlines)) } },+ 2 => html! { h2 { (self.render_inlines(&p.inlines)) } },+ 3 => html! { h3 { (self.render_inlines(&p.inlines)) } },+ 4 => html! { h4 { (self.render_inlines(&p.inlines)) } },+ 5 => html! { h5 { (self.render_inlines(&p.inlines)) } },+ _ => html! { h6 { (self.render_inlines(&p.inlines)) } },+ },+ }+ }++ fn render_table(&self, t: &Table) -> Markup {+ let mut head_rows = Vec::new();+ let mut body_rows = Vec::new();+ let mut foot_rows = Vec::new();++ for row in &t.rows {+ match row.kind {+ TableRowKind::Header => head_rows.push(row),+ TableRowKind::Body => body_rows.push(row),+ TableRowKind::Footer => foot_rows.push(row),+ }+ }++ html! {+ table {+ @if !head_rows.is_empty() {+ thead { @for row in head_rows { (self.render_table_row(row, true)) } }+ }+ tbody { @for row in body_rows { (self.render_table_row(row, false)) } }+ @if !foot_rows.is_empty() {+ tfoot { @for row in foot_rows { (self.render_table_row(row, false)) } }+ }+ }+ }+ }++ fn render_table_row(&self, row: &TableRow, header: bool) -> Markup {+ html! {+ tr {+ @for cell in &row.cells {+ @let cs = cell.colspan.get();+ @let rs = cell.rowspan.get();+ @let cs_attr = if cs > 1 { Some(cs) } else { None };+ @let rs_attr = if rs > 1 { Some(rs) } else { None };++ @if header {+ @if let (Some(cs), Some(rs)) = (cs_attr, rs_attr) {+ th colspan=(cs) rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }+ } @else if let Some(cs) = cs_attr {+ th colspan=(cs) { (self.render_blocks_inline(&cell.blocks)) }+ } @else if let Some(rs) = rs_attr {+ th rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }+ } @else {+ th { (self.render_blocks_inline(&cell.blocks)) }+ }+ } @else {+ @if let (Some(cs), Some(rs)) = (cs_attr, rs_attr) {+ td colspan=(cs) rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }+ } @else if let Some(cs) = cs_attr {+ td colspan=(cs) { (self.render_blocks_inline(&cell.blocks)) }+ } @else if let Some(rs) = rs_attr {+ td rowspan=(rs) { (self.render_blocks_inline(&cell.blocks)) }+ } @else {+ td { (self.render_blocks_inline(&cell.blocks)) }+ }+ }+ }+ }+ }+ }++ fn render_list(&self, l: &List) -> Markup {+ match l.list_type {+ ListType::Ordered => html! {+ ol { @for item in &l.items { li { (self.render_blocks_inline(&item.blocks)) } } }+ },+ ListType::Unordered => html! {+ ul { @for item in &l.items { li { (self.render_blocks_inline(&item.blocks)) } } }+ },+ }+ }++ fn render_image(&self, i: &Image) -> Markup {+ match &i.alt {+ Some(alt) => html! { img src=(i.src) alt=(alt); },+ None => html! { img src=(i.src); },+ }+ }++ fn render_inlines(&self, inlines: &[Inline]) -> Markup {+ html! { @for inline in inlines { (self.render_inline(inline)) } }+ }++ fn render_inline(&self, inline: &Inline) -> Markup {+ match inline {+ Inline::Text(t) => html! { (t) },+ Inline::LineBreak => html! { br; },++ Inline::Link { href, children } => {+ html! { a href=(href) { (self.render_inlines(children)) } }+ }++ Inline::Strong(children) => html! { strong { (self.render_inlines(children)) } },+ Inline::Em(children) => html! { em { (self.render_inlines(children)) } },+ Inline::Del(children) => html! { del { (self.render_inlines(children)) } },+ Inline::Code(code) => html! { code { (code) } },+ Inline::Sup(children) => html! { sup { (self.render_inlines(children)) } },+ Inline::Sub(children) => html! { sub { (self.render_inlines(children)) } },++ Inline::FootnoteRef(id) => {+ html! { sup { a href={ "#footnote-" (&id.0) } { (&id.0) } } }+ }+ Inline::EndnoteRef(id) => html! { sup { a href={ "#endnote-" (&id.0) } { (&id.0) } } },+ Inline::CommentRef(id) => html! { a href={ "#comment-" (&id.0) } { "💬" } },+ Inline::Bookmark(id) => html! { a id=(&id.0) {} },+ }+ }+}diff --git a/apps/api/native/src/document/renderers/mod.rs b/apps/api/native/src/document/renderers/mod.rsnew file mode 100644index 000000000..3d4613dd4--- /dev/null+++ b/apps/api/native/src/document/renderers/mod.rs@@ -0,0 +1 @@+pub mod html;diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rsindex 3d2372c36..d028ae78f 100644--- a/apps/api/native/src/html.rs+++ b/apps/api/native/src/html.rs@@ -216,7 +216,7 @@ fn _extract_metadata(html: &str) -> Result<HashMap<String, Value>, Box<dyn std::if name == "description" {out.insert(name.to_string(),- Value::String(format!("{}, {}", existing, content)),+ Value::String(format!("{existing}, {content}")),);} else if name != "title" {out.insert(@@ -402,7 +402,7 @@ fn _transform_html_inner(opts: TransformHtmlOptions) -> Result<String, Box<dyn s.collect::<HashSet<_>>();for mode in modes {- let matcher = format!(":{}:", Into::<String>::into(mode.clone()));+ let matcher = format!(":{}:", Into::<String>::into(mode));let signatures = signatures.iter().filter(|x| x.contains(&matcher))@@ -420,7 +420,7 @@ fn _transform_html_inner(opts: TransformHtmlOptions) -> Result<String, Box<dyn scontinue;}- let signature = get_node_signature(&node, mode.clone());+ let signature = get_node_signature(&node, mode);if signatures.contains(&signature) {nodes_to_drop.push(node);}@@ -707,7 +707,7 @@ fn _extract_images(html: &str, base_url: &str) -> Result<Vec<String>, Box<dyn stif let Some(srcset) = attrs.get("srcset") {for part in srcset.split(',') {- if let Some(url) = part.trim().split_whitespace().next() {+ if let Some(url) = part.split_whitespace().next() {if !url.is_empty() {if let Ok(resolved) = resolve_image_url(url) {images.insert(resolved);@@ -730,7 +730,7 @@ fn _extract_images(html: &str, base_url: &str) -> Result<Vec<String>, Box<dyn stfor source in source_elements {if let Some(srcset) = source.attributes.borrow().get("srcset") {for part in srcset.split(',') {- if let Some(url) = part.trim().split_whitespace().next() {+ if let Some(url) = part.split_whitespace().next() {if !url.is_empty() {if let Ok(resolved) = resolve_image_url(url) {images.insert(resolved);diff --git a/apps/api/native/src/lib.rs b/apps/api/native/src/lib.rsindex 06cfebea6..f5f5c48e7 100644--- a/apps/api/native/src/lib.rs+++ b/apps/api/native/src/lib.rs@@ -5,7 +5,10 @@ pub use crate::html::*;pub use crate::pdf::*;pub use crate::utils::*;+pub use crate::document::{DocumentConverter, DocumentType};+mod crawler;+mod document;mod html;mod pdf;mod utils;diff --git a/apps/api/native/src/pdf.rs b/apps/api/native/src/pdf.rsindex e5ae2b278..67f64de42 100644--- a/apps/api/native/src/pdf.rs+++ b/apps/api/native/src/pdf.rs@@ -27,7 +27,7 @@ fn _get_pdf_metadata(path: &str) -> std::result::Result<PDFMetadata, String> {info.as_dict().and_then(|info| info.get(b"Title"))- .and_then(|title| lopdf::decode_text_string(title))+ .and_then(lopdf::decode_text_string)}).ok().or_else(|| {@@ -35,7 +35,7 @@ fn _get_pdf_metadata(path: &str) -> std::result::Result<PDFMetadata, String> {obj.as_dict().and_then(|obj| obj.get(b"Title"))- .and_then(|title| lopdf::decode_text_string(title))+ .and_then(lopdf::decode_text_string).ok()})})@@ -50,7 +50,7 @@ pub fn get_pdf_metadata(path: String) -> Result<PDFMetadata> {_get_pdf_metadata(&path).map_err(|e| {Error::new(Status::GenericFailure,- format!("Failed to get PDF metadata: {}", e),+ format!("Failed to get PDF metadata: {e}"),)})}diff --git a/apps/api/samples/sample.docx b/apps/api/samples/sample.docxnew file mode 100644index 000000000..de7b10e5eBinary files /dev/null and b/apps/api/samples/sample.docx differdiff --git a/apps/api/samples/sample.odt b/apps/api/samples/sample.odtnew file mode 100644index 000000000..86c8281eeBinary files /dev/null and b/apps/api/samples/sample.odt differdiff --git a/apps/api/samples/sample.rtf b/apps/api/samples/sample.rtfnew file mode 100644index 000000000..4d0d9b35d--- /dev/null+++ b/apps/api/samples/sample.rtf@@ -0,0 +1,35 @@+{\rtf1\ansi\deff3\adeflang1025+{\fonttbl{\f0\froman\fprq2\fcharset0 Times New Roman;}{\f1\froman\fprq2\fcharset2 Symbol;}{\f2\fswiss\fprq2\fcharset0 Arial;}{\f3\froman\fprq2\fcharset0 Liberation Serif{\*\falt Times New Roman};}{\f4\froman\fprq2\fcharset0 Liberation Sans{\*\falt Arial};}{\f5\fnil\fprq2\fcharset0 0;}{\f6\fnil\fprq2\fcharset0 Microsoft YaHei;}{\f7\fnil\fprq2\fcharset0 Arial;}}+{\colortbl;\red0\green0\blue0;\red0\green0\blue255;\red0\green255\blue255;\red0\green255\blue0;\red255\green0\blue255;\red255\green0\blue0;\red255\green255\blue0;\red255\green255\blue255;\red0\green0\blue128;\red0\green128\blue128;\red0\green128\blue0;\red128\green0\blue128;\red128\green0\blue0;\red128\green128\blue0;\red128\green128\blue128;\red192\green192\blue192;}+{\stylesheet{\s0\snext0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052 Normal;}+{\s15\sbasedon0\snext16\rtlch\af7\afs28 \ltrch\hich\af4\loch\sb240\sa120\keepn\f4\fs28\dbch\af6 Heading;}+{\s16\sbasedon0\snext16\loch\sl276\slmult1\sb0\sa140 Body Text;}+{\s17\sbasedon16\snext17\rtlch\af7 \ltrch\loch\sl240\slmult1\sb0\sa0 List;}+{\s18\sbasedon0\snext18\rtlch\af7\afs24\ai \ltrch\loch\sb120\sa120\fs24\i caption;}+{\s19\sbasedon0\snext19\rtlch\af7 \ltrch Index;}+{\s20\sbasedon0\snext20\loch\nowidctlpar Table Contents;}+}{\*\generator LibreOffice/25.2.5.2$Windows_X86_64 LibreOffice_project/03d19516eb2e1dd5d4ccd751a0d6f35f35e08022}{\info{\creatim\yr2025\mo9\dy16\hr16\min7}{\revtim\yr2025\mo9\dy16\hr19\min39}{\printim\yr0\mo0\dy0\hr0\min0}}{\*\userprops}\deftab709+\hyphauto1\viewscale100\formshade\nobrkwrptbl\paperh16838\paperw11906\margl1134\margr1134\margt1134\margb1134\sectd\sbknone\sftnnar\saftnnrlc\sectunlocked1\pgwsxn11906\pghsxn16838\marglsxn1134\margrsxn1134\margtsxn1134\margbsxn1134\ftnbj\ftnstart1\ftnrstcont\ftnnar\fet\aftnrstcont\aftnstart1\aftnnrlc+{\*\ftnsep\chftnsep}\pgndec\pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql{\rtlch\afs36\ab \ltrch\loch\fs36\b\loch+Hello!}+\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql{\rtlch\afs28\ab0 \ltrch\loch\fs28\b0\loch+RTF file to test the Firecrawl Document Converter.}+\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql\rtlch\afs28\ab0 \ltrch\loch\fs28\b0\loch++\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql{\rtlch\afs28\ai\ab0 \ltrch\loch\fs28\i\b0\loch+Italic}+\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql{\rtlch\afs28\ai0\ab \ltrch\loch\fs28\i0\b\loch+Bold}+\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql{\rtlch\afs28\ai0\ab0 \ltrch\loch\fs28\i0\ul\ulc0\b0\loch+Underlined}+\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql{\rtlch\afs28\ai0\ab0 \ltrch\loch\strike\fs28\i0\ulnone\ulc0\b0\loch+Strikethrough}+\par \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql\rtlch\afs28\ai0\ab0 \ltrch\loch\strike\fs28\i0\ulnone\ulc0\b0\loch++\par \trowd\trql\trleft10\ltrrow\trpaddft3\trpaddt0\trpaddfl3\trpaddl0\trpaddfb3\trpaddb0\trpaddfr3\trpaddr0\clbrdrt\brdrs\brdrw10\brdrcf1\clpadfl3\clpadl55\clbrdrl\brdrs\brdrw10\brdrcf1\clpadft3\clpadt55\clbrdrb\brdrs\brdrw10\brdrcf1\clpadfb3\clpadb55\clpadfr3\clpadr55\cellx4829\clbrdrt\brdrs\brdrw10\brdrcf1\clpadfl3\clpadl55\clbrdrl\brdrs\brdrw10\brdrcf1\clpadft3\clpadt55\clbrdrb\brdrs\brdrw10\brdrcf1\clpadfb3\clpadb55\clbrdrr\brdrs\brdrw10\brdrcf1\clpadfr3\clpadr55\cellx9646\pard\plain \s20\loch\nowidctlpar\intbl\ql\ltrpar{\rtlch\afs28 \ltrch\loch\fs28\loch+Header 1}\cell\pard\plain \s20\loch\nowidctlpar\intbl\ql\ltrpar{\rtlch\afs28 \ltrch\loch\fs28\loch+Header 2}\cell\row\pard \trowd\trql\trleft10\ltrrow\trpaddft3\trpaddt0\trpaddfl3\trpaddl0\trpaddfb3\trpaddb0\trpaddfr3\trpaddr0\clpadfl3\clpadl55\clbrdrl\brdrs\brdrw10\brdrcf1\clpadft3\clpadt55\clbrdrb\brdrs\brdrw10\brdrcf1\clpadfb3\clpadb55\clpadfr3\clpadr55\cellx4829\clpadfl3\clpadl55\clbrdrl\brdrs\brdrw10\brdrcf1\clpadft3\clpadt55\clbrdrb\brdrs\brdrw10\brdrcf1\clpadfb3\clpadb55\clbrdrr\brdrs\brdrw10\brdrcf1\clpadfr3\clpadr55\cellx9646\pard\plain \s20\loch\nowidctlpar\intbl\ql\ltrpar{\rtlch\afs28 \ltrch\loch\fs28\loch+Value 1}\cell\pard\plain \s20\loch\nowidctlpar\intbl\ql\ltrpar{\rtlch\afs28 \ltrch\loch\fs28\loch+Value 2}\cell\row\pard \pard\plain \s0\rtlch\af7\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\widctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af5\langfe2052\ql\rtlch\afs28\ai0\ab0 \ltrch\loch\strike0\fs28\i0\ulnone\ulc0\b0\loch++\par }\ No newline at end of filediff --git a/apps/api/src/__tests__/snips/v2/document-converter.test.ts b/apps/api/src/__tests__/snips/v2/document-converter.test.tsnew file mode 100644index 000000000..c8d0d6d15--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/document-converter.test.ts@@ -0,0 +1,46 @@+import path from "path";+import fs from "fs";+import { DocumentConverter, DocumentType } from "@mendable/firecrawl-rs";++describe("Document Converter tests", () => {+ const samplesDir = path.join(process.cwd(), "samples");++ const expectedHtmlBase = (documentText: string) =>+ `<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Document</title></head><body><main><p><strong>Hello!</strong></p><p>${documentText} file to test the Firecrawl Document Converter.</p><p><em>Italic</em></p><p><strong>Bold</strong></p><p>Underlined</p><p><del>Strikethrough</del></p><table><tbody><tr><td>Header 1</td><td>Header 2</td></tr><tr><td>Value 1</td><td>Value 2</td></tr></tbody></table></main></body></html>`;++ const sampleFiles = [+ { file: "sample.docx", type: DocumentType.Docx, name: "DOCX" },+ { file: "sample.odt", type: DocumentType.Odt, name: "ODT" },+ { file: "sample.rtf", type: DocumentType.Rtf, name: "RTF" },+ ];++ const converter = new DocumentConverter();++ describe.each(sampleFiles)(+ "$name document conversion",+ ({ file, type, name }) => {+ const filePath = path.join(samplesDir, file);++ beforeAll(() => {+ if (!fs.existsSync(filePath)) {+ throw new Error(`Sample file ${filePath} does not exist`);+ }+ });++ it.concurrent(+ `should convert ${name} document and return expected HTML`,+ async () => {+ const fileBuffer = fs.readFileSync(filePath);++ const html = await converter.convertBufferToHtml(+ new Uint8Array(fileBuffer),+ type,+ );++ expect(html).toBe(expectedHtmlBase(name));+ },+ 10000,+ );+ },+ );+});diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex f89eefc33..4abb8cf9b 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -844,7 +844,8 @@ export async function performExtraction(if (docsMap.size == 0) {// All urls are invalid- const errorMessage = "All provided URLs are either invalid, unsupported or failed to be scraped.";+ const errorMessage =+ "All provided URLs are either invalid, unsupported or failed to be scraped.";logger.error(errorMessage);const tokens_billed = 300 + calculateThinkingCost(costTracking);await billTeam(@@ -976,11 +977,11 @@ export async function performExtraction(let finalResult = reqSchema? await mixSchemaObjects(- reqSchema,- singleAnswerResult,- multiEntityResult,- logger.child({ method: "mixSchemaObjects" }),- )+ reqSchema,+ singleAnswerResult,+ multiEntityResult,+ logger.child({ method: "mixSchemaObjects" }),+ ): singleAnswerResult || multiEntityResult;// Tokenize final result to get token countdiff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.tsnew file mode 100644index 000000000..649b520c0--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts@@ -0,0 +1,72 @@+import { Meta } from "../..";+import { EngineScrapeResult } from "..";+import { fetchFileToBuffer } from "../utils/downloadFile";+import { DocumentConverter, DocumentType } from "@mendable/firecrawl-rs";++const converter = new DocumentConverter();++function getDocumentTypeFromUrl(url: string): DocumentType {+ const urlLower = url.toLowerCase();+ if (urlLower.endsWith(".docx")) return DocumentType.Docx;+ if (urlLower.endsWith(".odt")) return DocumentType.Odt;+ if (urlLower.endsWith(".rtf")) return DocumentType.Rtf;++ return DocumentType.Docx; // hope for the best+}++function getDocumentTypeFromContentType(+ contentType: string | null,+): DocumentType | null {+ if (!contentType) return null;++ const ct = contentType.toLowerCase();++ if (+ ct.includes(+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ ) ||+ ct.includes("application/msword")+ ) {+ return DocumentType.Docx;+ }++ if (ct.includes("application/vnd.oasis.opendocument.text")) {+ return DocumentType.Odt;+ }++ if (ct.includes("application/rtf") || ct.includes("text/rtf")) {+ return DocumentType.Rtf;+ }++ return null;+}++export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {+ const { response, buffer } = await fetchFileToBuffer(+ meta.rewrittenUrl ?? meta.url,+ {+ headers: meta.options.headers,+ signal: meta.abort.asSignal(),+ },+ );++ const documentType =+ getDocumentTypeFromContentType(response.headers.get("content-type")) ??+ getDocumentTypeFromUrl(response.url);++ const html = await converter.convertBufferToHtml(+ new Uint8Array(buffer),+ documentType,+ );++ return {+ url: response.url,+ statusCode: response.status,+ html,+ proxyUsed: "basic",+ };+}++export function documentMaxReasonableTime(meta: Meta): number {+ return 15000;+}diff --git a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts b/apps/api/src/scraper/scrapeURL/engines/docx/index.tsdeleted file mode 100644index bfb199182..000000000--- a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts+++ /dev/null@@ -1,28 +0,0 @@-import { Meta } from "../..";-import { EngineScrapeResult } from "..";-import { downloadFile } from "../utils/downloadFile";-import mammoth from "mammoth";--export async function scrapeDOCX(meta: Meta): Promise<EngineScrapeResult> {- const { response, tempFilePath } = await downloadFile(- meta.id,- meta.rewrittenUrl ?? meta.url,- {- headers: meta.options.headers,- signal: meta.abort.asSignal(),- },- );-- return {- url: response.url,- statusCode: response.status,-- html: (await mammoth.convertToHtml({ path: tempFilePath })).value,-- proxyUsed: "basic",- };-}--export function docxMaxReasonableTime(meta: Meta): number {- return 15000;-}diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 0b26bc5ca..309a7c2aa 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -1,6 +1,6 @@import { ScrapeActionContent } from "../../../lib/entities";import { Meta } from "..";-import { docxMaxReasonableTime, scrapeDOCX } from "./docx";+import { documentMaxReasonableTime, scrapeDocument } from "./document";import {fireEngineMaxReasonableTime,scrapeURLWithFireEngineChromeCDP,@@ -31,7 +31,7 @@ export type Engine =| "playwright"| "fetch"| "pdf"- | "docx"+ | "document"| "index"| "index;documents";@@ -59,7 +59,7 @@ const engines: Engine[] = [...(usePlaywright ? ["playwright" as const] : []),"fetch","pdf",- "docx",+ "document",];const featureFlags = [@@ -68,7 +68,7 @@ const featureFlags = ["screenshot","screenshot@fullScreen","pdf",- "docx",+ "document","atsv","location","mobile",@@ -90,7 +90,7 @@ const featureFlagOptions: {screenshot: { priority: 10 },"screenshot@fullScreen": { priority: 10 },pdf: { priority: 100 },- docx: { priority: 100 },+ document: { priority: 100 },atsv: { priority: 90 }, // NOTE: should atsv force to tlsclient? adjust priority if notuseFastMode: { priority: 90 },location: { priority: 10 },@@ -149,7 +149,7 @@ const engineHandlers: {playwright: scrapeURLWithPlaywright,fetch: scrapeURLWithFetch,pdf: scrapePDF,- docx: scrapeDOCX,+ document: scrapeDocument,};const engineMRTs: {@@ -176,7 +176,7 @@ const engineMRTs: {playwright: playwrightMaxReasonableTime,fetch: fetchMaxReasonableTime,pdf: pdfMaxReasonableTime,- docx: docxMaxReasonableTime,+ document: documentMaxReasonableTime,};const engineOptions: {@@ -196,7 +196,7 @@ const engineOptions: {screenshot: true,"screenshot@fullScreen": true,pdf: false,- docx: false,+ document: false,atsv: false,mobile: true,location: true,@@ -214,7 +214,7 @@ const engineOptions: {screenshot: true, // through actions transform"screenshot@fullScreen": true, // through actions transformpdf: false,- docx: false,+ document: false,atsv: false,location: true,mobile: true,@@ -232,7 +232,7 @@ const engineOptions: {screenshot: true, // through actions transform"screenshot@fullScreen": true, // through actions transformpdf: false,- docx: false,+ document: false,atsv: false,location: true,mobile: true,@@ -250,7 +250,7 @@ const engineOptions: {screenshot: true,"screenshot@fullScreen": true,pdf: true,- docx: true,+ document: true,atsv: false,location: true,mobile: true,@@ -268,7 +268,7 @@ const engineOptions: {screenshot: true, // through actions transform"screenshot@fullScreen": true, // through actions transformpdf: false,- docx: false,+ document: false,atsv: false,location: true,mobile: true,@@ -286,7 +286,7 @@ const engineOptions: {screenshot: true, // through actions transform"screenshot@fullScreen": true, // through actions transformpdf: false,- docx: false,+ document: false,atsv: false,location: true,mobile: true,@@ -304,7 +304,7 @@ const engineOptions: {screenshot: true,"screenshot@fullScreen": true,pdf: false,- docx: false,+ document: false,atsv: false,location: false,mobile: false,@@ -322,7 +322,7 @@ const engineOptions: {screenshot: true,"screenshot@fullScreen": true,pdf: false,- docx: false,+ document: false,atsv: false,location: false,mobile: false,@@ -340,7 +340,7 @@ const engineOptions: {screenshot: false,"screenshot@fullScreen": false,pdf: false,- docx: false,+ document: false,atsv: false,location: false,mobile: false,@@ -358,7 +358,7 @@ const engineOptions: {screenshot: false,"screenshot@fullScreen": false,pdf: false,- docx: false,+ document: false,atsv: true,location: true,mobile: false,@@ -376,7 +376,7 @@ const engineOptions: {screenshot: false,"screenshot@fullScreen": false,pdf: false,- docx: false,+ document: false,atsv: true,location: true,mobile: false,@@ -394,7 +394,7 @@ const engineOptions: {screenshot: false,"screenshot@fullScreen": false,pdf: false,- docx: false,+ document: false,atsv: false,location: false,mobile: false,@@ -412,7 +412,7 @@ const engineOptions: {screenshot: false,"screenshot@fullScreen": false,pdf: true,- docx: false,+ document: false,atsv: false,location: false,mobile: false,@@ -423,14 +423,14 @@ const engineOptions: {},quality: -20,},- docx: {+ document: {features: {actions: false,waitFor: false,screenshot: false,"screenshot@fullScreen": false,pdf: false,- docx: true,+ document: true,atsv: false,location: false,mobile: false,diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.tsindex 0720f3c55..6a85c6af9 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts@@ -38,27 +38,31 @@ export async function specialtyScrapeCheck(x => x[0].toLowerCase() === "content-type",) ?? [])[1];- if (contentType === undefined) {+ if (!contentType) {logger.warn("Failed to check contentType -- was not present in headers", {headers,});- } else if (+ return;+ }++ if (contentType === "application/pdf" ||contentType.startsWith("application/pdf;") ||(contentType === "application/octet-stream" &&(feRes?.file?.content.startsWith("JVBERi0") ||feRes?.content.startsWith("%PDF-")))) {throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));- } else if (- contentType ===- "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ||- contentType.startsWith(- "application/vnd.openxmlformats-officedocument.wordprocessingml.document;",- )- ) {- // .docx- throw new AddFeatureError(["docx"]);+ }++ const documentTypes = [+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ "application/rtf",+ "text/rtf",+ "application/vnd.oasis.opendocument.text",+ ];++ if (documentTypes.some(type => contentType.startsWith(type))) {+ throw new AddFeatureError(["document"]);}}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex d44f60f9a..e733c1d74 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -149,13 +149,18 @@ function buildFeatureFlags(}const urlO = new URL(url);+ const lowerPath = urlO.pathname.toLowerCase();- if (urlO.pathname.endsWith(".pdf")) {+ if (lowerPath.endsWith(".pdf")) {flags.add("pdf");}- if (urlO.pathname.endsWith(".docx")) {- flags.add("docx");+ if (+ lowerPath.endsWith(".docx") ||+ lowerPath.endsWith(".odt") ||+ lowerPath.endsWith(".rtf")+ ) {+ flags.add("document");}if (options.blockAds === false) {