Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): parse formatting in legacy .doc (#4175)

tomsideguide348267c30824
diff --git a/apps/api/native/src/document/encoding.rs b/apps/api/native/src/document/encoding.rs
index 858c0cb73..2b107feaa 100644
--- a/apps/api/native/src/document/encoding.rs
+++ b/apps/api/native/src/document/encoding.rs
@@ -1,7 +1,7 @@
use encoding_rs::{Encoding, WINDOWS_1252};
-/// Encoding for a Windows code page. Rejects the Unicode pseudo code pages,
-/// which never describe single-byte legacy text.
+/// Encoding for a Windows code page, excluding the Unicode pseudo code
+/// pages.
pub fn encoding_for_codepage(codepage: u16) -> Option<&'static Encoding> {
match codepage {
0 | 1200 | 1201 | 65000 | 65001 => None,
diff --git a/apps/api/native/src/document/providers/doc.rs b/apps/api/native/src/document/providers/doc.rs
index 04c7b2311..7fd10a85e 100644
--- a/apps/api/native/src/document/providers/doc.rs
+++ b/apps/api/native/src/document/providers/doc.rs
@@ -1,5 +1,5 @@
-//! Legacy Word binary (.doc) provider, following the MS-DOC piece table
-//! format.
+//! Legacy Word binary (.doc) provider, following the MS-DOC piece table and
+//! formatting bin table (FKP) formats.
use crate::document::encoding::{encoding_for_codepage, encoding_for_lid};
use crate::document::model::*;
@@ -9,14 +9,17 @@ use cfb::CompoundFile;
use encoding_rs::Encoding;
use std::error::Error;
use std::io::{Cursor, Read, Seek};
+use std::num::NonZeroU32;
const MAGIC_WORD_97: u16 = 0xA5EC;
const MAGIC_WORD_6_95: u16 = 0xA5DC;
-/// Caps text extraction so a malformed piece table cannot force
-/// multi-gigabyte output from a small file.
+/// Upper bound on text extracted from one document.
const MAX_TEXT_CHARS: usize = 10_000_000;
+const FKP_PAGE_SIZE: usize = 512;
+const MAX_FIELD_INSTRUCTION_CHARS: usize = 2048;
+
pub struct DocProvider;
impl DocProvider {
@@ -51,7 +54,15 @@ impl DocumentProvider for DocProvider {
.and_then(encoding_for_codepage)
.unwrap_or_else(|| encoding_for_lid(fib.lid));
- let text = extract_text(&word_doc, table.as_deref(), &fib, ansi);
+ let segments = extract_segments(&word_doc, table.as_deref(), &fib, ansi);
+ let formatting = if fib.legacy {
+ Formatting::default()
+ } else {
+ table
+ .as_deref()
+ .map(|table| Formatting::parse(&word_doc, table, &fib))
+ .unwrap_or_default()
+ };
let mut metadata = DocumentMetadata::default();
if let Some(summary) = summary {
@@ -61,7 +72,7 @@ impl DocumentProvider for DocProvider {
}
Ok(Document {
- blocks: text_to_blocks(&text),
+ blocks: build_blocks(&segments, &formatting),
metadata,
notes: Vec::new(),
comments: Vec::new(),
@@ -93,6 +104,10 @@ struct Fib {
ccp_text: usize,
fc_clx: usize,
lcb_clx: usize,
+ fc_plcf_bte_chpx: usize,
+ lcb_plcf_bte_chpx: usize,
+ fc_plcf_bte_papx: usize,
+ lcb_plcf_bte_papx: usize,
}
impl Fib {
@@ -112,6 +127,14 @@ impl Fib {
return Err("encrypted .doc files are not supported".into());
}
+ let u32_at = |offset: usize| {
+ if word_doc.len() >= offset + 4 {
+ read_u32(word_doc, offset) as usize
+ } else {
+ 0
+ }
+ };
+
Ok(Fib {
legacy,
lid: read_u16(word_doc, 0x06),
@@ -119,31 +142,30 @@ impl Fib {
ext_char: flags & 0x1000 != 0,
fc_min: read_u32(word_doc, 0x18) as usize,
fc_mac: read_u32(word_doc, 0x1C) as usize,
- ccp_text: if word_doc.len() >= 0x50 {
- read_u32(word_doc, 0x4C) as usize
- } else {
- 0
- },
- fc_clx: if word_doc.len() >= 0x1AA {
- read_u32(word_doc, 0x1A2) as usize
- } else {
- 0
- },
- lcb_clx: if word_doc.len() >= 0x1AA {
- read_u32(word_doc, 0x1A6) as usize
- } else {
- 0
- },
+ ccp_text: u32_at(0x4C),
+ fc_clx: u32_at(0x1A2),
+ lcb_clx: u32_at(0x1A6),
+ fc_plcf_bte_chpx: u32_at(0xFA),
+ lcb_plcf_bte_chpx: u32_at(0xFE),
+ fc_plcf_bte_papx: u32_at(0x102),
+ lcb_plcf_bte_papx: u32_at(0x106),
})
}
}
-fn extract_text(
+/// Decoded text run and the stream offset its bytes came from.
+struct TextSegment {
+ text: String,
+ fc_start: u32,
+ bytes_per_char: u8,
+}
+
+fn extract_segments(
word_doc: &[u8],
table: Option<&[u8]>,
fib: &Fib,
ansi: &'static Encoding,
-) -> String {
+) -> Vec<TextSegment> {
if !fib.legacy {
if let Some(table) = table {
if let Some(pieces) = parse_piece_table(table, fib) {
@@ -156,11 +178,16 @@ fn extract_text(
let start = fib.fc_min.min(word_doc.len());
let end = fib.fc_mac.clamp(start, word_doc.len());
let bytes = &word_doc[start..end];
- if fib.ext_char {
- decode_utf16le(bytes)
+ let (text, bytes_per_char) = if fib.ext_char {
+ (decode_utf16le(bytes), 2)
} else {
- ansi.decode_without_bom_handling(bytes).0.into_owned()
- }
+ (ansi.decode_without_bom_handling(bytes).0.into_owned(), 1)
+ };
+ vec![TextSegment {
+ text,
+ fc_start: start as u32,
+ bytes_per_char,
+ }]
}
struct Piece {
@@ -231,27 +258,32 @@ fn decode_pieces(
pieces: &[Piece],
ccp_text: usize,
ansi: &'static Encoding,
-) -> String {
- // piece CP ranges are contiguous and ascending, so clamping ccpText bounds
- // the total output no matter how many pieces the table declares
+) -> Vec<TextSegment> {
+ // piece cp ranges are contiguous, so this clamp bounds total output
let ccp_text = ccp_text.min(MAX_TEXT_CHARS);
- let mut out = String::new();
+ let mut segments = Vec::new();
for piece in pieces {
- // main document body only; headers, footnotes etc. live past ccpText
+ // body text only; cps past ccpText are headers, footnotes etc.
if piece.start_cp as usize >= ccp_text {
continue;
}
let count = (piece.end_cp as usize).min(ccp_text) - piece.start_cp as usize;
let available = word_doc.len().saturating_sub(piece.offset);
- if piece.compressed {
- let bytes = &word_doc[piece.offset.min(word_doc.len())..][..count.min(available)];
- out.push_str(&ansi.decode_without_bom_handling(bytes).0);
+ let start = piece.offset.min(word_doc.len());
+ let (text, bytes_per_char) = if piece.compressed {
+ let bytes = &word_doc[start..][..count.min(available)];
+ (ansi.decode_without_bom_handling(bytes).0.into_owned(), 1)
} else {
- let bytes = &word_doc[piece.offset.min(word_doc.len())..][..(count * 2).min(available)];
- out.push_str(&decode_utf16le(bytes));
- }
+ let bytes = &word_doc[start..][..(count * 2).min(available)];
+ (decode_utf16le(bytes), 2)
+ };
+ segments.push(TextSegment {
+ text,
+ fc_start: start as u32,
+ bytes_per_char,
+ });
}
- out
+ segments
}
fn decode_utf16le(bytes: &[u8]) -> String {
@@ -262,81 +294,574 @@ fn decode_utf16le(bytes: &[u8]) -> String {
String::from_utf16_lossy(&units)
}
-fn text_to_blocks(text: &str) -> Vec<Block> {
- let mut blocks = Vec::new();
- let mut inlines: Vec<Inline> = Vec::new();
- let mut buf = String::new();
- // per-field flag: true while inside the instruction part (before 0x14)
- let mut field_stack: Vec<bool> = Vec::new();
+#[derive(Clone, Copy, Default, PartialEq, Eq)]
+struct CharProps {
+ bold: bool,
+ italic: bool,
+ strike: bool,
+}
+
+#[derive(Clone, Copy, Default)]
+struct ParaProps {
+ in_table: bool,
+ row_end: bool,
+ heading: Option<u8>,
+}
+
+struct FcRun<T> {
+ fc_start: u32,
+ fc_end: u32,
+ props: T,
+}
+
+#[derive(Default)]
+struct Formatting {
+ char_runs: Vec<FcRun<CharProps>>,
+ para_runs: Vec<FcRun<ParaProps>>,
+}
+
+impl Formatting {
+ fn parse(word_doc: &[u8], table: &[u8], fib: &Fib) -> Self {
+ Self {
+ char_runs: parse_bin_table(
+ word_doc,
+ table,
+ fib.fc_plcf_bte_chpx,
+ fib.lcb_plcf_bte_chpx,
+ parse_chpx_fkp,
+ ),
+ para_runs: parse_bin_table(
+ word_doc,
+ table,
+ fib.fc_plcf_bte_papx,
+ fib.lcb_plcf_bte_papx,
+ parse_papx_fkp,
+ ),
+ }
+ }
+}
+
+/// Walks a bin table: a PLC of page numbers of FKPs in the WordDocument
+/// stream.
+fn parse_bin_table<T>(
+ word_doc: &[u8],
+ table: &[u8],
+ fc: usize,
+ lcb: usize,
+ parse_fkp: impl Fn(&[u8], &mut Vec<FcRun<T>>),
+) -> Vec<FcRun<T>> {
+ let mut runs = Vec::new();
+ let Some(end) = fc.checked_add(lcb) else {
+ return runs;
+ };
+ let Some(plc) = table.get(fc..end) else {
+ return runs;
+ };
+ if lcb < 12 || (lcb - 4) % 8 != 0 {
+ return runs;
+ }
+ let n = (lcb - 4) / 8;
+ for k in 0..n {
+ let pn = read_u32(plc, 4 * (n + 1) + 4 * k) as usize;
+ let Some(page_start) = pn.checked_mul(FKP_PAGE_SIZE) else {
+ continue;
+ };
+ if let Some(page) = word_doc.get(page_start..page_start + FKP_PAGE_SIZE) {
+ parse_fkp(page, &mut runs);
+ }
+ }
+ runs.sort_by_key(|run| run.fc_start);
+ runs
+}
+
+fn parse_chpx_fkp(page: &[u8], runs: &mut Vec<FcRun<CharProps>>) {
+ let crun = page[FKP_PAGE_SIZE - 1] as usize;
+ if crun == 0 || 4 * (crun + 1) + crun >= FKP_PAGE_SIZE {
+ return;
+ }
+ for i in 0..crun {
+ let fc_start = read_u32(page, 4 * i);
+ let fc_end = read_u32(page, 4 * (i + 1));
+ if fc_end <= fc_start {
+ continue;
+ }
+ // rgb entry: word offset of the Chpx within the page; 0 means default
+ let rgb = page[4 * (crun + 1) + i] as usize;
+ let mut props = CharProps::default();
+ if rgb != 0 {
+ let offset = rgb * 2;
+ if let Some(&cb) = page.get(offset) {
+ if let Some(grpprl) = page.get(offset + 1..offset + 1 + cb as usize) {
+ props = char_props_from_grpprl(grpprl);
+ }
+ }
+ }
+ runs.push(FcRun {
+ fc_start,
+ fc_end,
+ props,
+ });
+ }
+}
- fn flush_text(buf: &mut String, inlines: &mut Vec<Inline>) {
- if !buf.is_empty() {
- inlines.push(Inline::Text(std::mem::take(buf)));
+fn parse_papx_fkp(page: &[u8], runs: &mut Vec<FcRun<ParaProps>>) {
+ let cpara = page[FKP_PAGE_SIZE - 1] as usize;
+ if cpara == 0 || 4 * (cpara + 1) + 13 * cpara >= FKP_PAGE_SIZE {
+ return;
+ }
+ for i in 0..cpara {
+ let fc_start = read_u32(page, 4 * i);
+ let fc_end = read_u32(page, 4 * (i + 1));
+ if fc_end <= fc_start {
+ continue;
+ }
+ // BxPap: first byte is the word offset of the PapxInFkp; 0 means default
+ let bx = page[4 * (cpara + 1) + 13 * i] as usize;
+ let mut props = ParaProps::default();
+ if bx != 0 {
+ let offset = bx * 2;
+ // PapxInFkp: grpprl length is cb*2 - 1, or cb2*2 when cb is 0
+ if let Some(&cb) = page.get(offset) {
+ let (start, len) = if cb == 0 {
+ match page.get(offset + 1) {
+ Some(&cb2) => (offset + 2, cb2 as usize * 2),
+ None => (offset, 0),
+ }
+ } else {
+ (offset + 1, (cb as usize) * 2 - 1)
+ };
+ if let Some(grpprl) = page.get(start..start + len) {
+ props = para_props_from_grpprl(grpprl);
+ }
+ }
}
+ runs.push(FcRun {
+ fc_start,
+ fc_end,
+ props,
+ });
}
+}
- fn flush_paragraph(buf: &mut String, inlines: &mut Vec<Inline>, blocks: &mut Vec<Block>) {
- flush_text(buf, inlines);
- while matches!(inlines.last(), Some(Inline::LineBreak)) {
- inlines.pop();
+fn char_props_from_grpprl(grpprl: &[u8]) -> CharProps {
+ let mut props = CharProps::default();
+ walk_sprms(grpprl, |sprm, operand| {
+ // toggle operand: 1 = on, 0x81 = invert style (assume on)
+ let on = matches!(operand.first(), Some(1) | Some(0x81));
+ match sprm {
+ 0x0835 => props.bold = on,
+ 0x0836 => props.italic = on,
+ 0x0837 => props.strike = on,
+ _ => {}
}
- if has_visible_content(inlines) {
- if let Some(Inline::Text(first)) = inlines.first_mut() {
- *first = first.trim_start().to_string();
+ });
+ props
+}
+
+fn para_props_from_grpprl(grpprl: &[u8]) -> ParaProps {
+ let mut props = ParaProps::default();
+ if grpprl.len() < 2 {
+ return props;
+ }
+ props.heading = heading_from_istd(read_u16(grpprl, 0));
+ walk_sprms(&grpprl[2..], |sprm, operand| match sprm {
+ // sprmPFInTable, sprmPFTtp, sprmPOutLvl, sprmPIstd
+ 0x2416 => props.in_table = operand.first().is_some_and(|&v| v != 0),
+ 0x2417 => props.row_end = operand.first().is_some_and(|&v| v != 0),
+ 0x2640 => {
+ if let Some(&level) = operand.first() {
+ if level < 9 {
+ props.heading = Some((level + 1).min(6));
+ }
}
- if let Some(Inline::Text(last)) = inlines.last_mut() {
- *last = last.trim_end().to_string();
+ }
+ 0x4600 => {
+ if operand.len() >= 2 {
+ if let Some(level) = heading_from_istd(read_u16(operand, 0)) {
+ props.heading = Some(level);
+ }
}
- blocks.push(Block::Paragraph(Paragraph {
- kind: ParagraphKind::Normal,
- inlines: std::mem::take(inlines),
- }));
- } else {
- inlines.clear();
}
+ _ => {}
+ });
+ props
+}
+
+/// Built-in style indexes 1..=9 are "heading 1".."heading 9".
+fn heading_from_istd(istd: u16) -> Option<u8> {
+ if (1..=9).contains(&istd) {
+ Some((istd as u8).min(6))
+ } else {
+ None
+ }
+}
+
+/// Visits each sprm in a grpprl; operand size comes from the top 3 bits.
+fn walk_sprms(grpprl: &[u8], mut visit: impl FnMut(u16, &[u8])) {
+ let mut i = 0;
+ while i + 2 <= grpprl.len() {
+ let sprm = read_u16(grpprl, i);
+ i += 2;
+ let size = match sprm >> 13 {
+ 0 | 1 => 1,
+ 2 | 4 | 5 => 2,
+ 7 => 3,
+ 3 => 4,
+ _ => {
+ // variable size: length byte first (u16 + 1 for sprmTDefTable)
+ if sprm == 0xD608 {
+ if i + 2 > grpprl.len() {
+ return;
+ }
+ read_u16(grpprl, i) as usize + 1
+ } else {
+ match grpprl.get(i) {
+ Some(&cb) => cb as usize + 1,
+ None => return,
+ }
+ }
+ }
+ };
+ let Some(operand) = grpprl.get(i..i + size) else {
+ return;
+ };
+ visit(sprm, operand);
+ i += size;
+ }
+}
+
+/// Props for the run containing `fc`, with a cursor for sequential lookups.
+fn run_lookup<T: Copy + Default>(runs: &[FcRun<T>], fc: u32, cursor: &mut usize) -> T {
+ if runs.is_empty() {
+ return T::default();
+ }
+ if *cursor >= runs.len() || fc < runs[*cursor].fc_start {
+ *cursor = runs.partition_point(|run| run.fc_end <= fc);
+ } else {
+ while *cursor < runs.len() && runs[*cursor].fc_end <= fc {
+ *cursor += 1;
+ }
+ }
+ match runs.get(*cursor) {
+ Some(run) if fc >= run.fc_start && fc < run.fc_end => run.props,
+ _ => T::default(),
}
+}
+
+fn build_blocks(segments: &[TextSegment], formatting: &Formatting) -> Vec<Block> {
+ let mut builder = BlockBuilder {
+ formatting,
+ blocks: Vec::new(),
+ inlines: Vec::new(),
+ run_text: String::new(),
+ run_props: CharProps::default(),
+ char_cursor: 0,
+ para_cursor: 0,
+ fields: Vec::new(),
+ table_rows: Vec::new(),
+ row_cells: Vec::new(),
+ cell_blocks: Vec::new(),
+ };
+
+ for segment in segments {
+ let mut fc = segment.fc_start;
+ for ch in segment.text.chars() {
+ builder.handle_char(ch, fc);
+ fc = fc.saturating_add(match segment.bytes_per_char {
+ 1 => 1,
+ _ => 2 * ch.len_utf16() as u32,
+ });
+ }
+ }
+ builder.finish()
+}
+
+struct FieldFrame {
+ in_instruction: bool,
+ instruction: String,
+ result_inline_start: usize,
+}
+
+struct BlockBuilder<'a> {
+ formatting: &'a Formatting,
+ blocks: Vec<Block>,
+ inlines: Vec<Inline>,
+ run_text: String,
+ run_props: CharProps,
+ char_cursor: usize,
+ para_cursor: usize,
+ fields: Vec<FieldFrame>,
+ table_rows: Vec<TableRow>,
+ row_cells: Vec<TableCell>,
+ cell_blocks: Vec<Block>,
+}
- for ch in text.chars() {
+impl BlockBuilder<'_> {
+ fn handle_char(&mut self, ch: char, fc: u32) {
match ch {
'\u{13}' => {
- field_stack.push(true);
- continue;
+ self.fields.push(FieldFrame {
+ in_instruction: true,
+ instruction: String::new(),
+ result_inline_start: usize::MAX,
+ });
+ return;
}
'\u{14}' => {
- if let Some(top) = field_stack.last_mut() {
- *top = false;
+ self.flush_run();
+ let start = self.inlines.len();
+ if let Some(frame) = self.fields.last_mut() {
+ frame.in_instruction = false;
+ frame.result_inline_start = start;
}
- continue;
+ return;
}
'\u{15}' => {
- field_stack.pop();
- continue;
+ self.flush_run();
+ if let Some(frame) = self.fields.pop() {
+ self.close_field(frame);
+ }
+ return;
}
_ => {}
}
- if field_stack.iter().any(|&in_instruction| in_instruction) {
- continue;
+
+ if let Some(frame) = self.fields.iter_mut().rev().find(|f| f.in_instruction) {
+ if frame.instruction.len() < MAX_FIELD_INSTRUCTION_CHARS {
+ frame.instruction.push(ch);
+ }
+ return;
}
+
match ch {
- // paragraph mark, cell/row mark, page break, column break
- '\r' | '\n' | '\u{7}' | '\u{c}' | '\u{e}' => {
- flush_paragraph(&mut buf, &mut inlines, &mut blocks)
+ '\r' | '\n' | '\u{c}' | '\u{e}' => {
+ let props = self.para_props(fc);
+ self.end_paragraph(props);
+ }
+ '\u{7}' => {
+ let props = self.para_props(fc);
+ if props.row_end {
+ self.end_row();
+ } else if props.in_table {
+ self.end_cell(props);
+ } else {
+ // no table context: behave like a paragraph mark
+ self.end_paragraph(props);
+ }
}
'\u{b}' => {
- flush_text(&mut buf, &mut inlines);
- if !inlines.is_empty() {
- inlines.push(Inline::LineBreak);
+ self.flush_run();
+ if !self.inlines.is_empty() {
+ self.inlines.push(Inline::LineBreak);
}
}
- '\t' => buf.push('\t'),
- '\u{1e}' => buf.push('-'),
+ '\t' => self.push_char('\t', fc),
+ '\u{1e}' => self.push_char('-', fc),
c if c.is_control() || c == '\u{1f}' => {}
- c => buf.push(c),
+ c => self.push_char(c, fc),
+ }
+ }
+
+ fn push_char(&mut self, ch: char, fc: u32) {
+ let props = run_lookup(&self.formatting.char_runs, fc, &mut self.char_cursor);
+ if props != self.run_props {
+ self.flush_run();
+ self.run_props = props;
+ }
+ self.run_text.push(ch);
+ }
+
+ fn para_props(&mut self, fc: u32) -> ParaProps {
+ run_lookup(&self.formatting.para_runs, fc, &mut self.para_cursor)
+ }
+
+ fn flush_run(&mut self) {
+ if self.run_text.is_empty() {
+ return;
+ }
+ let mut node = Inline::Text(std::mem::take(&mut self.run_text));
+ if self.run_props.strike {
+ node = Inline::Del(vec![node]);
+ }
+ if self.run_props.italic {
+ node = Inline::Em(vec![node]);
+ }
+ if self.run_props.bold {
+ node = Inline::Strong(vec![node]);
+ }
+ self.inlines.push(node);
+ }
+
+ fn close_field(&mut self, frame: FieldFrame) {
+ let Some(href) = hyperlink_target(&frame.instruction) else {
+ return;
+ };
+ let start = frame.result_inline_start.min(self.inlines.len());
+ let children = self.inlines.split_off(start);
+ if has_visible_content(&children) {
+ self.inlines.push(Inline::Link { href, children });
+ } else {
+ self.inlines.extend(children);
+ }
+ }
+
+ /// Takes the pending inlines as a paragraph, or `None` when nothing
+ /// visible accumulated.
+ fn take_paragraph(&mut self, props: ParaProps) -> Option<Block> {
+ self.flush_run();
+ // links cannot span paragraphs
+ for frame in &mut self.fields {
+ frame.result_inline_start = 0;
+ }
+ while matches!(self.inlines.last(), Some(Inline::LineBreak)) {
+ self.inlines.pop();
+ }
+ if !has_visible_content(&self.inlines) {
+ self.inlines.clear();
+ return None;
+ }
+ if let Some(Inline::Text(first)) = self.inlines.first_mut() {
+ *first = first.trim_start().to_string();
+ }
+ if let Some(Inline::Text(last)) = self.inlines.last_mut() {
+ *last = last.trim_end().to_string();
+ }
+ let kind = match props.heading {
+ Some(level) => ParagraphKind::Heading(level),
+ None => ParagraphKind::Normal,
+ };
+ Some(Block::Paragraph(Paragraph {
+ kind,
+ inlines: std::mem::take(&mut self.inlines),
+ }))
+ }
+
+ fn end_paragraph(&mut self, props: ParaProps) {
+ let paragraph = self.take_paragraph(props);
+ if props.in_table {
+ if let Some(paragraph) = paragraph {
+ self.cell_blocks.push(paragraph);
+ }
+ } else {
+ self.flush_table();
+ if let Some(paragraph) = paragraph {
+ self.blocks.push(paragraph);
+ }
+ }
+ }
+
+ fn end_cell(&mut self, props: ParaProps) {
+ if let Some(paragraph) = self.take_paragraph(props) {
+ self.cell_blocks.push(paragraph);
+ }
+ self.row_cells.push(TableCell {
+ blocks: std::mem::take(&mut self.cell_blocks),
+ colspan: NonZeroU32::MIN,
+ rowspan: NonZeroU32::MIN,
+ });
+ }
+
+ /// Row terminator: the TTP mark's own paragraph carries no cell content.
+ fn end_row(&mut self) {
+ self.flush_run();
+ self.inlines.clear();
+ if !self.cell_blocks.is_empty() {
+ self.row_cells.push(TableCell {
+ blocks: std::mem::take(&mut self.cell_blocks),
+ colspan: NonZeroU32::MIN,
+ rowspan: NonZeroU32::MIN,
+ });
+ }
+ if !self.row_cells.is_empty() {
+ self.table_rows.push(TableRow {
+ cells: std::mem::take(&mut self.row_cells),
+ kind: TableRowKind::Body,
+ });
+ }
+ }
+
+ fn flush_table(&mut self) {
+ if !self.cell_blocks.is_empty() || !self.row_cells.is_empty() {
+ self.end_row();
+ }
+ if !self.table_rows.is_empty() {
+ self.blocks.push(Block::Table(Table {
+ rows: std::mem::take(&mut self.table_rows),
+ }));
+ }
+ }
+
+ fn finish(mut self) -> Vec<Block> {
+ self.end_paragraph(ParaProps::default());
+ self.flush_table();
+ self.blocks
+ }
+}
+
+/// Target of a HYPERLINK field instruction.
+fn hyperlink_target(instruction: &str) -> Option<String> {
+ let rest = instruction.trim_start();
+ if !rest
+ .get(..9)
+ .is_some_and(|p| p.eq_ignore_ascii_case("HYPERLINK"))
+ {
+ return None;
+ }
+ let mut tokens = field_tokens(&rest[9..]);
+ let mut anchor = false;
+ while let Some(token) = tokens.next() {
+ match token.as_str() {
+ "\\l" => anchor = true,
+ // switches whose argument is not the target
+ "\\o" | "\\t" => {
+ tokens.next();
+ }
+ t if t.starts_with('\\') => {}
+ t => {
+ return Some(if anchor {
+ format!("#{t}")
+ } else {
+ t.to_string()
+ });
+ }
}
}
- flush_paragraph(&mut buf, &mut inlines, &mut blocks);
+ None
+}
- blocks
+/// Whitespace-separated tokens; quoted strings stay together, unquoted.
+fn field_tokens(input: &str) -> impl Iterator<Item = String> + '_ {
+ let mut chars = input.chars().peekable();
+ std::iter::from_fn(move || {
+ while chars.peek().is_some_and(|c| c.is_whitespace()) {
+ chars.next();
+ }
+ match chars.peek() {
+ None => None,
+ Some('"') => {
+ chars.next();
+ let mut token = String::new();
+ for c in chars.by_ref() {
+ if c == '"' {
+ break;
+ }
+ token.push(c);
+ }
+ Some(token)
+ }
+ Some(_) => {
+ let mut token = String::new();
+ while let Some(&c) = chars.peek() {
+ if c.is_whitespace() {
+ break;
+ }
+ token.push(c);
+ chars.next();
+ }
+ Some(token)
+ }
+ }
+ })
}
fn read_u16(data: &[u8], offset: usize) -> u16 {
@@ -362,13 +887,103 @@ mod tests {
Unicode(&'static str),
}
+ /// Contiguous formatting runs: (fc_start, fc_end, grpprl). An empty grpprl
+ /// becomes a default (offset 0) FKP entry.
+ #[derive(Default)]
+ struct TestFmt {
+ char_runs: Vec<(u32, u32, Vec<u8>)>,
+ para_runs: Vec<(u32, u32, Vec<u8>)>,
+ }
+
fn utf16_bytes(text: &str) -> Vec<u8> {
text.encode_utf16().flat_map(u16::to_le_bytes).collect()
}
const TEXT_START: usize = 0x200;
+ /// FC of the i-th UTF-16 unit of a single Unicode test piece.
+ fn ufc(i: usize) -> u32 {
+ (TEXT_START + 2 * i) as u32
+ }
+
+ fn sprm1(code: u16, value: u8) -> Vec<u8> {
+ let mut out = code.to_le_bytes().to_vec();
+ out.push(value);
+ out
+ }
+
+ fn papx(istd: u16, sprms: &[Vec<u8>]) -> Vec<u8> {
+ let mut out = istd.to_le_bytes().to_vec();
+ for sprm in sprms {
+ out.extend_from_slice(sprm);
+ }
+ out
+ }
+
+ fn build_chpx_fkp(runs: &[(u32, u32, Vec<u8>)]) -> Vec<u8> {
+ let mut page = vec![0u8; FKP_PAGE_SIZE];
+ let crun = runs.len();
+ for (i, (fc_start, fc_end, _)) in runs.iter().enumerate() {
+ page[4 * i..4 * i + 4].copy_from_slice(&fc_start.to_le_bytes());
+ page[4 * (i + 1)..4 * (i + 1) + 4].copy_from_slice(&fc_end.to_le_bytes());
+ }
+ let mut blob_end = FKP_PAGE_SIZE - 1;
+ for (i, (_, _, grpprl)) in runs.iter().enumerate() {
+ if grpprl.is_empty() {
+ continue;
+ }
+ let blob_len = 1 + grpprl.len();
+ let mut pos = blob_end - blob_len;
+ pos &= !1;
+ page[pos] = grpprl.len() as u8;
+ page[pos + 1..pos + 1 + grpprl.len()].copy_from_slice(grpprl);
+ page[4 * (crun + 1) + i] = (pos / 2) as u8;
+ blob_end = pos;
+ }
+ page[FKP_PAGE_SIZE - 1] = crun as u8;
+ page
+ }
+
+ fn build_papx_fkp(runs: &[(u32, u32, Vec<u8>)]) -> Vec<u8> {
+ let mut page = vec![0u8; FKP_PAGE_SIZE];
+ let cpara = runs.len();
+ for (i, (fc_start, fc_end, _)) in runs.iter().enumerate() {
+ page[4 * i..4 * i + 4].copy_from_slice(&fc_start.to_le_bytes());
+ page[4 * (i + 1)..4 * (i + 1) + 4].copy_from_slice(&fc_end.to_le_bytes());
+ }
+ let mut blob_end = FKP_PAGE_SIZE - 1;
+ for (i, (_, _, grpprl)) in runs.iter().enumerate() {
+ if grpprl.is_empty() {
+ continue;
+ }
+ let blob = if grpprl.len() % 2 == 1 {
+ let mut b = vec![((grpprl.len() + 1) / 2) as u8];
+ b.extend_from_slice(grpprl);
+ b
+ } else {
+ let mut b = vec![0, (grpprl.len() / 2) as u8];
+ b.extend_from_slice(grpprl);
+ b
+ };
+ let mut pos = blob_end - blob.len();
+ pos &= !1;
+ page[pos..pos + blob.len()].copy_from_slice(&blob);
+ page[4 * (cpara + 1) + 13 * i] = (pos / 2) as u8;
+ blob_end = pos;
+ }
+ page[FKP_PAGE_SIZE - 1] = cpara as u8;
+ page
+ }
+
fn build_word_doc_streams(pieces: &[TestPiece], lid: u16) -> (Vec<u8>, Vec<u8>) {
+ build_word_doc_streams_fmt(pieces, lid, &TestFmt::default())
+ }
+
+ fn build_word_doc_streams_fmt(
+ pieces: &[TestPiece],
+ lid: u16,
+ fmt: &TestFmt,
+ ) -> (Vec<u8>, Vec<u8>) {
let mut word_doc = vec![0u8; TEXT_START];
let mut cps = vec![0u32];
let mut pcds = Vec::new();
@@ -403,6 +1018,30 @@ mod tests {
let mut table = vec![0x02];
table.extend_from_slice(&(plc.len() as u32).to_le_bytes());
table.extend_from_slice(&plc);
+ let lcb_clx = table.len() as u32;
+
+ let append_fkp =
+ |word_doc: &mut Vec<u8>, table: &mut Vec<u8>, page: Vec<u8>, runs: &[(u32, u32, Vec<u8>)]| {
+ while word_doc.len() % FKP_PAGE_SIZE != 0 {
+ word_doc.push(0);
+ }
+ let pn = (word_doc.len() / FKP_PAGE_SIZE) as u32;
+ word_doc.extend_from_slice(&page);
+ let bin_table_offset = table.len() as u32;
+ table.extend_from_slice(&runs[0].0.to_le_bytes());
+ table.extend_from_slice(&runs.last().unwrap().1.to_le_bytes());
+ table.extend_from_slice(&pn.to_le_bytes());
+ (bin_table_offset, 12u32)
+ };
+
+ let chpx_bt = (!fmt.char_runs.is_empty()).then(|| {
+ let page = build_chpx_fkp(&fmt.char_runs);
+ append_fkp(&mut word_doc, &mut table, page, &fmt.char_runs)
+ });
+ let papx_bt = (!fmt.para_runs.is_empty()).then(|| {
+ let page = build_papx_fkp(&fmt.para_runs);
+ append_fkp(&mut word_doc, &mut table, page, &fmt.para_runs)
+ });
word_doc[0x00..0x02].copy_from_slice(&MAGIC_WORD_97.to_le_bytes());
word_doc[0x02..0x04].copy_from_slice(&0x00C1u16.to_le_bytes());
@@ -410,7 +1049,15 @@ mod tests {
word_doc[0x0A..0x0C].copy_from_slice(&0x0200u16.to_le_bytes());
word_doc[0x4C..0x50].copy_from_slice(&ccp_text.to_le_bytes());
word_doc[0x1A2..0x1A6].copy_from_slice(&0u32.to_le_bytes());
- word_doc[0x1A6..0x1AA].copy_from_slice(&(table.len() as u32).to_le_bytes());
+ word_doc[0x1A6..0x1AA].copy_from_slice(&lcb_clx.to_le_bytes());
+ if let Some((fc, lcb)) = chpx_bt {
+ word_doc[0xFA..0xFE].copy_from_slice(&fc.to_le_bytes());
+ word_doc[0xFE..0x102].copy_from_slice(&lcb.to_le_bytes());
+ }
+ if let Some((fc, lcb)) = papx_bt {
+ word_doc[0x102..0x106].copy_from_slice(&fc.to_le_bytes());
+ word_doc[0x106..0x10A].copy_from_slice(&lcb.to_le_bytes());
+ }
(word_doc, table)
}
@@ -425,7 +1072,16 @@ mod tests {
}
fn build_doc(pieces: &[TestPiece], lid: u16, summary: Option<&[u8]>) -> Vec<u8> {
- let (word_doc, table) = build_word_doc_streams(pieces, lid);
+ build_doc_fmt(pieces, lid, summary, &TestFmt::default())
+ }
+
+ fn build_doc_fmt(
+ pieces: &[TestPiece],
+ lid: u16,
+ summary: Option<&[u8]>,
+ fmt: &TestFmt,
+ ) -> Vec<u8> {
+ let (word_doc, table) = build_word_doc_streams_fmt(pieces, lid, fmt);
let mut streams: Vec<(&str, &[u8])> = vec![("WordDocument", &word_doc), ("1Table", &table)];
if let Some(summary) = summary {
streams.push(("\x05SummaryInformation", summary));
@@ -470,20 +1126,45 @@ mod tests {
data
}
+ /// Readable markup for inline assertions, e.g. `a <b>x</b> <a href=u>y</a>`.
+ fn inline_markup(inlines: &[Inline]) -> String {
+ inlines
+ .iter()
+ .map(|inline| match inline {
+ Inline::Text(t) => t.clone(),
+ Inline::LineBreak => "\n".to_string(),
+ Inline::Strong(c) => format!("<b>{}</b>", inline_markup(c)),
+ Inline::Em(c) => format!("<i>{}</i>", inline_markup(c)),
+ Inline::Del(c) => format!("<del>{}</del>", inline_markup(c)),
+ Inline::Link { href, children } => {
+ format!("<a href={}>{}</a>", href, inline_markup(children))
+ }
+ _ => String::new(),
+ })
+ .collect()
+ }
+
+ fn inline_text(inlines: &[Inline]) -> String {
+ inlines
+ .iter()
+ .map(|inline| match inline {
+ Inline::Text(t) => t.clone(),
+ Inline::LineBreak => "\n".to_string(),
+ Inline::Link { children, .. } => inline_text(children),
+ Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {
+ inline_text(c)
+ }
+ _ => String::new(),
+ })
+ .collect()
+ }
+
fn paragraph_texts(document: &Document) -> Vec<String> {
document
.blocks
.iter()
.map(|block| match block {
- Block::Paragraph(p) => p
- .inlines
- .iter()
- .map(|inline| match inline {
- Inline::Text(t) => t.as_str(),
- Inline::LineBreak => "\n",
- _ => "",
- })
- .collect::<String>(),
+ Block::Paragraph(p) => inline_text(&p.inlines),
_ => String::new(),
})
.collect()
@@ -541,7 +1222,10 @@ mod tests {
0x0409,
None,
);
- assert_eq!(paragraph_texts(&parse(&data)), vec!["Café \u{201C}quoted\u{201D}"]);
+ assert_eq!(
+ paragraph_texts(&parse(&data)),
+ vec!["Café \u{201C}quoted\u{201D}"]
+ );
}
#[test]
@@ -572,10 +1256,61 @@ mod tests {
assert_eq!(paragraph_texts(&parse(&data)), vec!["See the site now"]);
}
+ #[test]
+ fn hyperlink_fields_become_links() {
+ let data = build_doc(
+ &[TestPiece::Unicode(
+ "See \u{13} HYPERLINK \"https://example.com\" \u{14}the site\u{15} now\r",
+ )],
+ 0x0409,
+ None,
+ );
+ let document = parse(&data);
+ let Block::Paragraph(p) = &document.blocks[0] else {
+ panic!("expected paragraph");
+ };
+ assert_eq!(
+ inline_markup(&p.inlines),
+ "See <a href=https://example.com>the site</a> now"
+ );
+ }
+
+ #[test]
+ fn anchor_hyperlink_targets_bookmark() {
+ let data = build_doc(
+ &[TestPiece::Unicode(
+ "Go \u{13} HYPERLINK \\l \"section1\" \u{14}there\u{15}\r",
+ )],
+ 0x0409,
+ None,
+ );
+ let document = parse(&data);
+ let Block::Paragraph(p) = &document.blocks[0] else {
+ panic!("expected paragraph");
+ };
+ assert_eq!(inline_markup(&p.inlines), "Go <a href=#section1>there</a>");
+ }
+
+ #[test]
+ fn non_hyperlink_fields_keep_result_text_only() {
+ let data = build_doc(
+ &[TestPiece::Unicode("Page \u{13} PAGE \u{14}7\u{15} of 9\r")],
+ 0x0409,
+ None,
+ );
+ let document = parse(&data);
+ let Block::Paragraph(p) = &document.blocks[0] else {
+ panic!("expected paragraph");
+ };
+ assert_eq!(inline_markup(&p.inlines), "Page 7 of 9");
+ }
+
#[test]
fn cell_marks_and_line_breaks() {
let data = build_doc(
- &[TestPiece::Unicode("cell one\u{7}cell two\u{7}line\u{b}break\r")],
+ &[TestPiece::Unicode(
+ "cell one\u{7}cell two\u{7}line\u{b}break\r",
+ )],
0x0409,
None,
);
@@ -585,6 +1320,112 @@ mod tests {
);
}
+ #[test]
+ fn bold_italic_strike_runs() {
+ let text = "plain bold and style\r";
+ let fmt = TestFmt {
+ char_runs: vec![
+ (ufc(0), ufc(6), vec![]),
+ (ufc(6), ufc(10), sprm1(0x0835, 1)),
+ (ufc(10), ufc(15), vec![]),
+ (
+ ufc(15),
+ ufc(20),
+ [sprm1(0x0836, 1), sprm1(0x0837, 1)].concat(),
+ ),
+ (ufc(20), ufc(21), vec![]),
+ ],
+ ..TestFmt::default()
+ };
+ let data = build_doc_fmt(&[TestPiece::Unicode(text)], 0x0409, None, &fmt);
+ let document = parse(&data);
+ let Block::Paragraph(p) = &document.blocks[0] else {
+ panic!("expected paragraph");
+ };
+ assert_eq!(
+ inline_markup(&p.inlines),
+ "plain <b>bold</b> and <i><del>style</del></i>"
+ );
+ }
+
+ #[test]
+ fn headings_from_istd_and_outline_level() {
+ let text = "Title\rSection\rBody\r";
+ let fmt = TestFmt {
+ para_runs: vec![
+ (ufc(0), ufc(6), papx(1, &[])),
+ (ufc(6), ufc(14), papx(0, &[sprm1(0x2640, 1)])),
+ (ufc(14), ufc(19), papx(0, &[])),
+ ],
+ ..TestFmt::default()
+ };
+ let data = build_doc_fmt(&[TestPiece::Unicode(text)], 0x0409, None, &fmt);
+ let document = parse(&data);
+ let kinds: Vec<ParagraphKind> = document
+ .blocks
+ .iter()
+ .filter_map(|b| match b {
+ Block::Paragraph(p) => Some(p.kind),
+ _ => None,
+ })
+ .collect();
+ assert_eq!(
+ kinds,
+ vec![
+ ParagraphKind::Heading(1),
+ ParagraphKind::Heading(2),
+ ParagraphKind::Normal
+ ]
+ );
+ }
+
+ #[test]
+ fn table_from_cell_and_row_marks() {
+ let text = "A1\u{7}B1\u{7}\u{7}A2\u{7}B2\u{7}\u{7}after\r";
+ let in_table = papx(0, &[sprm1(0x2416, 1)]);
+ let row_end = papx(0, &[sprm1(0x2416, 1), sprm1(0x2417, 1)]);
+ let fmt = TestFmt {
+ para_runs: vec![
+ (ufc(0), ufc(3), in_table.clone()),
+ (ufc(3), ufc(6), in_table.clone()),
+ (ufc(6), ufc(7), row_end.clone()),
+ (ufc(7), ufc(10), in_table.clone()),
+ (ufc(10), ufc(13), in_table),
+ (ufc(13), ufc(14), row_end),
+ (ufc(14), ufc(20), papx(0, &[])),
+ ],
+ ..TestFmt::default()
+ };
+ let data = build_doc_fmt(&[TestPiece::Unicode(text)], 0x0409, None, &fmt);
+ let document = parse(&data);
+
+ let Block::Table(table) = &document.blocks[0] else {
+ panic!("expected table, got {:?}", document.blocks[0]);
+ };
+ let cells: Vec<Vec<String>> = table
+ .rows
+ .iter()
+ .map(|row| {
+ row
+ .cells
+ .iter()
+ .map(|cell| {
+ cell
+ .blocks
+ .iter()
+ .map(|b| match b {
+ Block::Paragraph(p) => inline_text(&p.inlines),
+ _ => String::new(),
+ })
+ .collect::<String>()
+ })
+ .collect()
+ })
+ .collect();
+ assert_eq!(cells, vec![vec!["A1", "B1"], vec!["A2", "B2"]]);
+ assert_eq!(paragraph_texts(&parse(&data)).last().unwrap(), "after");
+ }
+
#[test]
fn legacy_word_95_contiguous_text() {
let text_bytes = cp1251("Стаття перша.\rСтаття друга.\r");
@@ -647,7 +1488,8 @@ mod tests {
#[test]
fn corrupt_piece_table_falls_back_to_fc_range() {
- let (mut word_doc, _) = build_word_doc_streams(&[TestPiece::Unicode("good text here\r")], 0x0409);
+ let (mut word_doc, _) =
+ build_word_doc_streams(&[TestPiece::Unicode("good text here\r")], 0x0409);
word_doc[0x0A..0x0C].copy_from_slice(&0x1200u16.to_le_bytes());
let fc_mac = word_doc.len() as u32;
word_doc[0x18..0x1C].copy_from_slice(&(TEXT_START as u32).to_le_bytes());
diff --git a/apps/api/native/src/document/providers/docx.rs b/apps/api/native/src/document/providers/docx.rs
index 866f0bf6d..2e2b60c8b 100644
--- a/apps/api/native/src/document/providers/docx.rs
+++ b/apps/api/native/src/document/providers/docx.rs
@@ -102,8 +102,12 @@ fn read_relationships<R: Read + Seek>(zip: &mut ZipArchive<R>, path: &str) -> Re
fn read_core_properties<R: Read + Seek>(zip: &mut ZipArchive<R>) -> Option<DocumentMetadata> {
let text = read_zip_text(zip, "docProps/core.xml")?;
let xml = XmlDoc::parse(strip_bom(&text)).ok()?;
- let element_text =
- |local: &str| xml.descendants().find(|n| is_tag(n, local)).and_then(|n| n.text());
+ let element_text = |local: &str| {
+ xml
+ .descendants()
+ .find(|n| is_tag(n, local))
+ .and_then(|n| n.text())
+ };
let mut meta = DocumentMetadata::default();
if let Some(title) = element_text("title") {
@@ -643,7 +647,10 @@ fn parse_list(nodes: &[Node], mut i: usize, ctx: Ctx) -> (List, usize) {
items: Vec::new(),
list_type: ListType::Unordered,
};
- let Some(base) = nodes.get(i).and_then(|n| paragraph_list_info(n, ctx.numbering)) else {
+ let Some(base) = nodes
+ .get(i)
+ .and_then(|n| paragraph_list_info(n, ctx.numbering))
+ else {
return (list, i + 1);
};
list.list_type = base.list_type;
@@ -748,7 +755,11 @@ fn read_notes<R: Read + Seek>(zip: &mut ZipArchive<R>, kind: NoteKind, ctx: Ctx)
"word/_rels/footnotes.xml.rels",
"footnote",
),
- NoteKind::Endnote => ("word/endnotes.xml", "word/_rels/endnotes.xml.rels", "endnote"),
+ NoteKind::Endnote => (
+ "word/endnotes.xml",
+ "word/_rels/endnotes.xml.rels",
+ "endnote",
+ ),
};
let Some(text) = read_zip_text(zip, xml_path) else {
return Vec::new();
@@ -764,7 +775,10 @@ fn read_notes<R: Read + Seek>(zip: &mut ZipArchive<R>, kind: NoteKind, ctx: Ctx)
let Some(id) = attr(&n, "id") else {
continue;
};
- if matches!(attr(&n, "type"), Some("separator" | "continuationSeparator")) {
+ if matches!(
+ attr(&n, "type"),
+ Some("separator" | "continuationSeparator")
+ ) {
continue;
}
notes.push(Note {
@@ -800,4 +814,3 @@ fn read_comments<R: Read + Seek>(zip: &mut ZipArchive<R>, ctx: Ctx) -> Vec<Comme
}
comments
}
-
diff --git a/apps/api/native/src/document/providers/factory.rs b/apps/api/native/src/document/providers/factory.rs
index e9b98480f..b9bdb0b52 100644
--- a/apps/api/native/src/document/providers/factory.rs
+++ b/apps/api/native/src/document/providers/factory.rs
@@ -2,8 +2,8 @@ use super::doc::DocProvider;
use super::docx::DocxProvider;
use super::odt::OdtProvider;
use super::rtf::RtfProvider;
-use super::DocumentProvider;
use super::xlsx::XlsxProvider;
+use super::DocumentProvider;
use napi_derive::napi;
#[napi]
diff --git a/apps/api/native/src/document/providers/odt.rs b/apps/api/native/src/document/providers/odt.rs
index d894face8..92a35593a 100644
--- a/apps/api/native/src/document/providers/odt.rs
+++ b/apps/api/native/src/document/providers/odt.rs
@@ -57,8 +57,12 @@ impl DocumentProvider for OdtProvider {
fn read_meta<R: Read + Seek>(zip: &mut ZipArchive<R>) -> Option<DocumentMetadata> {
let text = read_zip_text(zip, "meta.xml")?;
let xml = XmlDoc::parse(strip_bom(&text)).ok()?;
- let element_text =
- |local: &str| xml.descendants().find(|n| is_tag(n, local)).and_then(|n| n.text());
+ let element_text = |local: &str| {
+ xml
+ .descendants()
+ .find(|n| is_tag(n, local))
+ .and_then(|n| n.text())
+ };
let mut meta = DocumentMetadata::default();
if let Some(title) = element_text("title") {
@@ -384,7 +388,6 @@ impl Parser<'_> {
} else if is_tag(&c, "line-break") {
out.push(Inline::LineBreak);
} else if is_tag(&c, "s") {
- // clamped so a malformed count cannot force a huge allocation
let count = attr(&c, "c")
.and_then(|v| v.parse::<usize>().ok())
.unwrap_or(1)
diff --git a/apps/api/native/src/document/providers/rtf.rs b/apps/api/native/src/document/providers/rtf.rs
index 77d791563..260216114 100644
--- a/apps/api/native/src/document/providers/rtf.rs
+++ b/apps/api/native/src/document/providers/rtf.rs
@@ -36,8 +36,7 @@ impl DocumentProvider for RtfProvider {
}
}
-/// Code page declared by the first \ansicpg control word. Token-based so
-/// escaped literals like "\\ansicpg" in body text cannot match.
+/// Code page declared by the first \ansicpg control word.
fn detect_encoding(src: &[u8]) -> &'static Encoding {
for token in Tokens::new(src) {
if let Token::Control("ansicpg", Some(codepage)) = token {
@@ -121,8 +120,7 @@ impl<'a> Tokens<'a> {
num_end += 1;
}
if num_end > num_start {
- // consume the digit run even when it overflows i32, so malformed
- // parameters do not leak digits into the text
+ // consumed even when the value overflows i32
self.pos = num_end;
if let Some(n) = std::str::from_utf8(&self.src[num_start..num_end])
.ok()
@@ -146,8 +144,7 @@ impl<'a> Tokens<'a> {
}
}
-/// Fallback byte count declared by \uc. Kept as declared: skipping consumes
-/// at most one byte per input token, so it is bounded by the input length.
+/// Fallback byte count declared by \uc.
fn uc_count(value: Option<i32>) -> usize {
value.unwrap_or(1).max(0) as usize
}
@@ -161,8 +158,7 @@ fn hex_val(b: u8) -> Option<u8> {
}
}
-/// Accumulates text, buffering raw bytes so multi-byte code pages decode
-/// across adjacent bytes.
+/// Text accumulator; raw bytes are buffered and decoded as a unit.
struct TextAccum {
encoding: &'static Encoding,
pending: Vec<u8>,
@@ -486,7 +482,10 @@ impl BodyParser {
self.state = CharState::default();
}
"trowd" => {
- self.table.get_or_insert_with(TableBuilder::default).start_row();
+ self
+ .table
+ .get_or_insert_with(TableBuilder::default)
+ .start_row();
self.in_table_cell = false;
}
"intbl" => self.in_table_cell = true,
@@ -756,7 +755,8 @@ mod tests {
#[test]
fn styles_and_tables_still_parse() {
- let rtf = b"{\\rtf1\\ansi {\\b Bold} and {\\i italic}\\par\\trowd\\intbl A\\cell B\\cell\\row\\par}";
+ let rtf =
+ b"{\\rtf1\\ansi {\\b Bold} and {\\i italic}\\par\\trowd\\intbl A\\cell B\\cell\\row\\par}";
let document = RtfProvider::new().parse_buffer(rtf).unwrap();
assert!(matches!(document.blocks[0], Block::Paragraph(_)));
assert!(document
@@ -779,8 +779,7 @@ mod tests {
#[test]
fn metadata_keeps_text_of_nested_formatting_groups() {
- let rtf =
- b"{\\rtf1{\\info{\\title Some {\\b Bold} Title{\\*\\junk secret}}}Body\\par}";
+ let rtf = b"{\\rtf1{\\info{\\title Some {\\b Bold} Title{\\*\\junk secret}}}Body\\par}";
let document = RtfProvider::new().parse_buffer(rtf).unwrap();
assert_eq!(document.metadata.title.as_deref(), Some("Some Bold Title"));
}
@@ -821,6 +820,3 @@ mod tests {
assert_eq!(paragraphs(rtf), vec!["У", "done"]);
}
}
-
-
-
diff --git a/apps/api/native/src/engpicker.rs b/apps/api/native/src/engpicker.rs
index b030eda7b..157a212ca 100644
--- a/apps/api/native/src/engpicker.rs
+++ b/apps/api/native/src/engpicker.rs
@@ -7,57 +7,57 @@ use tokio::task;
#[derive(Deserialize, Serialize)]
#[napi(object)]
pub struct EngpickerUrlResult {
- pub url: String,
- pub cdp_basic_markdown: Option<String>,
- pub cdp_basic_success: bool,
- pub cdp_stealth_markdown: Option<String>,
- pub cdp_stealth_success: bool,
- pub tls_basic_markdown: Option<String>,
- pub tls_basic_success: bool,
- pub tls_stealth_markdown: Option<String>,
- pub tls_stealth_success: bool,
+ pub url: String,
+ pub cdp_basic_markdown: Option<String>,
+ pub cdp_basic_success: bool,
+ pub cdp_stealth_markdown: Option<String>,
+ pub cdp_stealth_success: bool,
+ pub tls_basic_markdown: Option<String>,
+ pub tls_basic_success: bool,
+ pub tls_stealth_markdown: Option<String>,
+ pub tls_stealth_success: bool,
}
/// Verdict for a single URL
#[derive(Serialize)]
#[napi(object)]
pub struct EngpickerUrlVerdict {
- pub url: String,
- pub tls_client_sufficient: bool,
- pub cdp_failed: bool,
- pub similarity: Option<f64>,
- pub reason: String,
+ pub url: String,
+ pub tls_client_sufficient: bool,
+ pub cdp_failed: bool,
+ pub similarity: Option<f64>,
+ pub reason: String,
}
/// Final verdict enum
#[derive(Serialize)]
#[napi(string_enum)]
pub enum EngpickerFinalVerdict {
- /// tlsclient is sufficient for this site
- TlsClientOk,
- /// Chrome CDP is required for proper rendering
- ChromeCdpRequired,
- /// Too many CDP failures to determine verdict
- Uncertain,
+ /// tlsclient is sufficient for this site
+ TlsClientOk,
+ /// Chrome CDP is required for proper rendering
+ ChromeCdpRequired,
+ /// Too many CDP failures to determine verdict
+ Uncertain,
}
/// Final verdict result
#[derive(Serialize)]
#[napi(object)]
pub struct EngpickerVerdict {
- pub url_verdicts: Vec<EngpickerUrlVerdict>,
- pub tls_client_ok_count: u32,
- pub chrome_cdp_required_count: u32,
- pub cdp_failed_count: u32,
- pub total_urls: u32,
- pub verdict: EngpickerFinalVerdict,
+ pub url_verdicts: Vec<EngpickerUrlVerdict>,
+ pub tls_client_ok_count: u32,
+ pub chrome_cdp_required_count: u32,
+ pub cdp_failed_count: u32,
+ pub total_urls: u32,
+ pub verdict: EngpickerFinalVerdict,
}
/// Compute engpicker verdict using Levenshtein distance to compare tlsclient vs chrome-cdp results.
-///
+///
/// Chrome-CDP is the gold standard. We compare tlsclient markdown against it to determine
/// if tlsclient is sufficient for scraping this site (i.e., JS rendering not required).
-///
+///
/// Arguments:
/// - results: scrape results for each URL
/// - similarity_threshold: minimum similarity (0.0-1.0) for tlsclient to be considered sufficient
@@ -65,142 +65,158 @@ pub struct EngpickerVerdict {
/// - cdp_failure_threshold: maximum ratio of CDP failures before verdict becomes uncertain
#[napi]
pub async fn compute_engpicker_verdict(
- results: Vec<EngpickerUrlResult>,
- similarity_threshold: f64,
- success_rate_threshold: f64,
- cdp_failure_threshold: f64,
+ results: Vec<EngpickerUrlResult>,
+ similarity_threshold: f64,
+ success_rate_threshold: f64,
+ cdp_failure_threshold: f64,
) -> napi::Result<EngpickerVerdict> {
- task::spawn_blocking(move || {
- _compute_engpicker_verdict(results, similarity_threshold, success_rate_threshold, cdp_failure_threshold)
- })
- .await
- .map_err(|e| {
- napi::Error::new(
- napi::Status::GenericFailure,
- format!("compute_engpicker_verdict join error: {e}"),
- )
- })?
+ task::spawn_blocking(move || {
+ _compute_engpicker_verdict(
+ results,
+ similarity_threshold,
+ success_rate_threshold,
+ cdp_failure_threshold,
+ )
+ })
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("compute_engpicker_verdict join error: {e}"),
+ )
+ })?
}
fn _compute_engpicker_verdict(
- results: Vec<EngpickerUrlResult>,
- similarity_threshold: f64,
- success_rate_threshold: f64,
- cdp_failure_threshold: f64,
+ results: Vec<EngpickerUrlResult>,
+ similarity_threshold: f64,
+ success_rate_threshold: f64,
+ cdp_failure_threshold: f64,
) -> napi::Result<EngpickerVerdict> {
- let url_verdicts: Vec<EngpickerUrlVerdict> = results
- .iter()
- .map(|result| {
- // Get the best chrome-cdp result as gold standard (prefer stealth if both succeeded)
- let gold_standard = if result.cdp_stealth_success && result.cdp_stealth_markdown.is_some() {
- result.cdp_stealth_markdown.as_ref()
- } else if result.cdp_basic_success && result.cdp_basic_markdown.is_some() {
- result.cdp_basic_markdown.as_ref()
- } else {
- None
- };
-
- // Get the best tlsclient result (prefer stealth if both succeeded)
- let tls_result = if result.tls_stealth_success && result.tls_stealth_markdown.is_some() {
- result.tls_stealth_markdown.as_ref()
- } else if result.tls_basic_success && result.tls_basic_markdown.is_some() {
- result.tls_basic_markdown.as_ref()
- } else {
- None
- };
-
- // If chrome-cdp failed, we can't evaluate this URL
- let gold_standard = match gold_standard {
- Some(gs) if !gs.is_empty() => gs,
- _ => {
- return EngpickerUrlVerdict {
- url: result.url.clone(),
- tls_client_sufficient: false,
- cdp_failed: true,
- similarity: None,
- reason: "chrome-cdp failed".to_string(),
- };
- }
- };
-
- // If tlsclient failed entirely, it's definitely not enough
- let tls_result = match tls_result {
- Some(tls) if !tls.is_empty() => tls,
- _ => {
- return EngpickerUrlVerdict {
- url: result.url.clone(),
- tls_client_sufficient: false,
- cdp_failed: false,
- similarity: None,
- reason: "tlsclient failed".to_string(),
- };
- }
- };
-
- // Calculate Levenshtein distance and normalize to similarity score
- let distance = levenshtein(gold_standard, tls_result);
- let max_length = gold_standard.len().max(tls_result.len());
- let similarity = if max_length > 0 {
- 1.0 - (distance as f64 / max_length as f64)
- } else {
- 1.0
- };
-
- let tls_client_sufficient = similarity >= similarity_threshold;
-
- let reason = if tls_client_sufficient {
- format!("{:.1}% similar - tlsclient captures full content", similarity * 100.0)
- } else {
- format!("{:.1}% similar - JS rendering likely required", similarity * 100.0)
- };
-
- EngpickerUrlVerdict {
- url: result.url.clone(),
- tls_client_sufficient,
- cdp_failed: false,
- similarity: Some(similarity),
- reason,
- }
- })
- .collect();
-
- let total_urls = url_verdicts.len() as u32;
- let cdp_failed_count = url_verdicts.iter().filter(|v| v.cdp_failed).count() as u32;
- let tls_client_ok_count = url_verdicts.iter().filter(|v| v.tls_client_sufficient).count() as u32;
- let chrome_cdp_required_count = url_verdicts.iter().filter(|v| !v.tls_client_sufficient && !v.cdp_failed).count() as u32;
-
- // Determine final verdict
- let verdict = if total_urls == 0 {
- EngpickerFinalVerdict::Uncertain
+ let url_verdicts: Vec<EngpickerUrlVerdict> = results
+ .iter()
+ .map(|result| {
+ // Get the best chrome-cdp result as gold standard (prefer stealth if both succeeded)
+ let gold_standard = if result.cdp_stealth_success && result.cdp_stealth_markdown.is_some() {
+ result.cdp_stealth_markdown.as_ref()
+ } else if result.cdp_basic_success && result.cdp_basic_markdown.is_some() {
+ result.cdp_basic_markdown.as_ref()
+ } else {
+ None
+ };
+
+ // Get the best tlsclient result (prefer stealth if both succeeded)
+ let tls_result = if result.tls_stealth_success && result.tls_stealth_markdown.is_some() {
+ result.tls_stealth_markdown.as_ref()
+ } else if result.tls_basic_success && result.tls_basic_markdown.is_some() {
+ result.tls_basic_markdown.as_ref()
+ } else {
+ None
+ };
+
+ // If chrome-cdp failed, we can't evaluate this URL
+ let gold_standard = match gold_standard {
+ Some(gs) if !gs.is_empty() => gs,
+ _ => {
+ return EngpickerUrlVerdict {
+ url: result.url.clone(),
+ tls_client_sufficient: false,
+ cdp_failed: true,
+ similarity: None,
+ reason: "chrome-cdp failed".to_string(),
+ };
+ }
+ };
+
+ // If tlsclient failed entirely, it's definitely not enough
+ let tls_result = match tls_result {
+ Some(tls) if !tls.is_empty() => tls,
+ _ => {
+ return EngpickerUrlVerdict {
+ url: result.url.clone(),
+ tls_client_sufficient: false,
+ cdp_failed: false,
+ similarity: None,
+ reason: "tlsclient failed".to_string(),
+ };
+ }
+ };
+
+ // Calculate Levenshtein distance and normalize to similarity score
+ let distance = levenshtein(gold_standard, tls_result);
+ let max_length = gold_standard.len().max(tls_result.len());
+ let similarity = if max_length > 0 {
+ 1.0 - (distance as f64 / max_length as f64)
+ } else {
+ 1.0
+ };
+
+ let tls_client_sufficient = similarity >= similarity_threshold;
+
+ let reason = if tls_client_sufficient {
+ format!(
+ "{:.1}% similar - tlsclient captures full content",
+ similarity * 100.0
+ )
+ } else {
+ format!(
+ "{:.1}% similar - JS rendering likely required",
+ similarity * 100.0
+ )
+ };
+
+ EngpickerUrlVerdict {
+ url: result.url.clone(),
+ tls_client_sufficient,
+ cdp_failed: false,
+ similarity: Some(similarity),
+ reason,
+ }
+ })
+ .collect();
+
+ let total_urls = url_verdicts.len() as u32;
+ let cdp_failed_count = url_verdicts.iter().filter(|v| v.cdp_failed).count() as u32;
+ let tls_client_ok_count = url_verdicts
+ .iter()
+ .filter(|v| v.tls_client_sufficient)
+ .count() as u32;
+ let chrome_cdp_required_count = url_verdicts
+ .iter()
+ .filter(|v| !v.tls_client_sufficient && !v.cdp_failed)
+ .count() as u32;
+
+ // Determine final verdict
+ let verdict = if total_urls == 0 {
+ EngpickerFinalVerdict::Uncertain
+ } else {
+ let cdp_failure_rate = cdp_failed_count as f64 / total_urls as f64;
+
+ // If too many CDP failures, we can't make a confident verdict
+ if cdp_failure_rate > cdp_failure_threshold {
+ EngpickerFinalVerdict::Uncertain
} else {
- let cdp_failure_rate = cdp_failed_count as f64 / total_urls as f64;
-
- // If too many CDP failures, we can't make a confident verdict
- if cdp_failure_rate > cdp_failure_threshold {
- EngpickerFinalVerdict::Uncertain
+ // Calculate success rate among URLs where we could actually compare
+ let comparable_urls = total_urls - cdp_failed_count;
+ if comparable_urls == 0 {
+ EngpickerFinalVerdict::Uncertain
+ } else {
+ let tls_ok_rate = tls_client_ok_count as f64 / comparable_urls as f64;
+ if tls_ok_rate >= success_rate_threshold {
+ EngpickerFinalVerdict::TlsClientOk
} else {
- // Calculate success rate among URLs where we could actually compare
- let comparable_urls = total_urls - cdp_failed_count;
- if comparable_urls == 0 {
- EngpickerFinalVerdict::Uncertain
- } else {
- let tls_ok_rate = tls_client_ok_count as f64 / comparable_urls as f64;
- if tls_ok_rate >= success_rate_threshold {
- EngpickerFinalVerdict::TlsClientOk
- } else {
- EngpickerFinalVerdict::ChromeCdpRequired
- }
- }
+ EngpickerFinalVerdict::ChromeCdpRequired
}
- };
-
- Ok(EngpickerVerdict {
- url_verdicts,
- tls_client_ok_count,
- chrome_cdp_required_count,
- cdp_failed_count,
- total_urls,
- verdict,
- })
+ }
+ }
+ };
+
+ Ok(EngpickerVerdict {
+ url_verdicts,
+ tls_client_ok_count,
+ chrome_cdp_required_count,
+ cdp_failed_count,
+ total_urls,
+ verdict,
+ })
}
-
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rs
index 15c4faed4..71bae49f6 100644
--- a/apps/api/native/src/html.rs
+++ b/apps/api/native/src/html.rs
@@ -11,8 +11,9 @@ use serde_json::Value;
use tokio::task;
use url::Url;
-static URL_REGEX: LazyLock<Regex> =
- LazyLock::new(|| Regex::new(r#"url\(['"]?([^'")]+)['"]?\)"#).expect("URL_REGEX is a valid static regex pattern"));
+static URL_REGEX: LazyLock<Regex> = LazyLock::new(|| {
+ Regex::new(r#"url\(['"]?([^'")]+)['"]?\)"#).expect("URL_REGEX is a valid static regex pattern")
+});
use crate::utils::to_napi_err;
diff --git a/apps/api/native/src/logging.rs b/apps/api/native/src/logging.rs
index bf444e895..d29c778e9 100644
--- a/apps/api/native/src/logging.rs
+++ b/apps/api/native/src/logging.rs
@@ -40,9 +40,10 @@ impl Visit for FieldVisitor {
if field.name() == "message" {
self.message = Some(format!("{:?}", value));
} else {
- self
- .fields
- .insert(field.name().to_string(), Value::String(format!("{:?}", value)));
+ self.fields.insert(
+ field.name().to_string(),
+ Value::String(format!("{:?}", value)),
+ );
}
}
@@ -254,10 +255,7 @@ mod tests {
fn test_error_preserves_logs() {
let traced: TracingResult<napi::Result<()>> = with_native_tracing(None, "test", || {
tracing::info!("before error");
- Err(napi::Error::new(
- napi::Status::GenericFailure,
- "test error",
- ))
+ Err(napi::Error::new(napi::Status::GenericFailure, "test error"))
});
assert!(traced.value.is_err());
diff --git a/apps/api/native/src/pdf.rs b/apps/api/native/src/pdf.rs
index f97fc2743..400f010f2 100644
--- a/apps/api/native/src/pdf.rs
+++ b/apps/api/native/src/pdf.rs
@@ -1,6 +1,6 @@
use napi::bindgen_prelude::*;
use napi_derive::napi;
-use pdf_inspector::{PdfOptions, PdfType, process_pdf_with_options as rust_process_pdf};
+use pdf_inspector::{process_pdf_with_options as rust_process_pdf, PdfOptions, PdfType};
use crate::logging::{embed_logs_in_error, with_native_tracing, NativeContext, NativeLogEntry};
@@ -59,7 +59,10 @@ pub fn process_pdf(
let result = rust_process_pdf(&path, opts).map_err(|e| {
tracing::error!(error = %e, "PDF processing failed");
- Error::new(Status::GenericFailure, format!("Failed to process PDF: {e}"))
+ Error::new(
+ Status::GenericFailure,
+ format!("Failed to process PDF: {e}"),
+ )
})?;
tracing::info!(
@@ -86,10 +89,7 @@ pub fn process_pdf(
/// Skips text extraction, markdown generation, and layout analysis.
/// Pass `ctx` (NativeContext) for structured tracing with scrape_id/url.
#[napi]
-pub fn detect_pdf(
- path: String,
- ctx: Option<NativeContext>,
-) -> Result<PdfProcessResult> {
+pub fn detect_pdf(path: String, ctx: Option<NativeContext>) -> Result<PdfProcessResult> {
let traced = with_native_tracing(ctx.as_ref(), "pdf", || {
tracing::info!("starting PDF detection");