Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): parse formatting in legacy .doc (#4175)
tomsideguide348267c30824
diff --git a/apps/api/native/src/document/encoding.rs b/apps/api/native/src/document/encoding.rsindex 858c0cb73..2b107feaa 100644--- a/apps/api/native/src/document/encoding.rs+++ b/apps/api/native/src/document/encoding.rs@@ -1,7 +1,7 @@use encoding_rs::{Encoding, WINDOWS_1252};-/// Encoding for a Windows code page. Rejects the Unicode pseudo code pages,-/// which never describe single-byte legacy text.+/// Encoding for a Windows code page, excluding the Unicode pseudo code+/// pages.pub fn encoding_for_codepage(codepage: u16) -> Option<&'static Encoding> {match codepage {0 | 1200 | 1201 | 65000 | 65001 => None,diff --git a/apps/api/native/src/document/providers/doc.rs b/apps/api/native/src/document/providers/doc.rsindex 04c7b2311..7fd10a85e 100644--- a/apps/api/native/src/document/providers/doc.rs+++ b/apps/api/native/src/document/providers/doc.rs@@ -1,5 +1,5 @@-//! Legacy Word binary (.doc) provider, following the MS-DOC piece table-//! format.+//! Legacy Word binary (.doc) provider, following the MS-DOC piece table and+//! formatting bin table (FKP) formats.use crate::document::encoding::{encoding_for_codepage, encoding_for_lid};use crate::document::model::*;@@ -9,14 +9,17 @@ use cfb::CompoundFile;use encoding_rs::Encoding;use std::error::Error;use std::io::{Cursor, Read, Seek};+use std::num::NonZeroU32;const MAGIC_WORD_97: u16 = 0xA5EC;const MAGIC_WORD_6_95: u16 = 0xA5DC;-/// Caps text extraction so a malformed piece table cannot force-/// multi-gigabyte output from a small file.+/// Upper bound on text extracted from one document.const MAX_TEXT_CHARS: usize = 10_000_000;+const FKP_PAGE_SIZE: usize = 512;+const MAX_FIELD_INSTRUCTION_CHARS: usize = 2048;+pub struct DocProvider;impl DocProvider {@@ -51,7 +54,15 @@ impl DocumentProvider for DocProvider {.and_then(encoding_for_codepage).unwrap_or_else(|| encoding_for_lid(fib.lid));- let text = extract_text(&word_doc, table.as_deref(), &fib, ansi);+ let segments = extract_segments(&word_doc, table.as_deref(), &fib, ansi);+ let formatting = if fib.legacy {+ Formatting::default()+ } else {+ table+ .as_deref()+ .map(|table| Formatting::parse(&word_doc, table, &fib))+ .unwrap_or_default()+ };let mut metadata = DocumentMetadata::default();if let Some(summary) = summary {@@ -61,7 +72,7 @@ impl DocumentProvider for DocProvider {}Ok(Document {- blocks: text_to_blocks(&text),+ blocks: build_blocks(&segments, &formatting),metadata,notes: Vec::new(),comments: Vec::new(),@@ -93,6 +104,10 @@ struct Fib {ccp_text: usize,fc_clx: usize,lcb_clx: usize,+ fc_plcf_bte_chpx: usize,+ lcb_plcf_bte_chpx: usize,+ fc_plcf_bte_papx: usize,+ lcb_plcf_bte_papx: usize,}impl Fib {@@ -112,6 +127,14 @@ impl Fib {return Err("encrypted .doc files are not supported".into());}+ let u32_at = |offset: usize| {+ if word_doc.len() >= offset + 4 {+ read_u32(word_doc, offset) as usize+ } else {+ 0+ }+ };+Ok(Fib {legacy,lid: read_u16(word_doc, 0x06),@@ -119,31 +142,30 @@ impl Fib {ext_char: flags & 0x1000 != 0,fc_min: read_u32(word_doc, 0x18) as usize,fc_mac: read_u32(word_doc, 0x1C) as usize,- ccp_text: if word_doc.len() >= 0x50 {- read_u32(word_doc, 0x4C) as usize- } else {- 0- },- fc_clx: if word_doc.len() >= 0x1AA {- read_u32(word_doc, 0x1A2) as usize- } else {- 0- },- lcb_clx: if word_doc.len() >= 0x1AA {- read_u32(word_doc, 0x1A6) as usize- } else {- 0- },+ ccp_text: u32_at(0x4C),+ fc_clx: u32_at(0x1A2),+ lcb_clx: u32_at(0x1A6),+ fc_plcf_bte_chpx: u32_at(0xFA),+ lcb_plcf_bte_chpx: u32_at(0xFE),+ fc_plcf_bte_papx: u32_at(0x102),+ lcb_plcf_bte_papx: u32_at(0x106),})}}-fn extract_text(+/// Decoded text run and the stream offset its bytes came from.+struct TextSegment {+ text: String,+ fc_start: u32,+ bytes_per_char: u8,+}++fn extract_segments(word_doc: &[u8],table: Option<&[u8]>,fib: &Fib,ansi: &'static Encoding,-) -> String {+) -> Vec<TextSegment> {if !fib.legacy {if let Some(table) = table {if let Some(pieces) = parse_piece_table(table, fib) {@@ -156,11 +178,16 @@ fn extract_text(let start = fib.fc_min.min(word_doc.len());let end = fib.fc_mac.clamp(start, word_doc.len());let bytes = &word_doc[start..end];- if fib.ext_char {- decode_utf16le(bytes)+ let (text, bytes_per_char) = if fib.ext_char {+ (decode_utf16le(bytes), 2)} else {- ansi.decode_without_bom_handling(bytes).0.into_owned()- }+ (ansi.decode_without_bom_handling(bytes).0.into_owned(), 1)+ };+ vec![TextSegment {+ text,+ fc_start: start as u32,+ bytes_per_char,+ }]}struct Piece {@@ -231,27 +258,32 @@ fn decode_pieces(pieces: &[Piece],ccp_text: usize,ansi: &'static Encoding,-) -> String {- // piece CP ranges are contiguous and ascending, so clamping ccpText bounds- // the total output no matter how many pieces the table declares+) -> Vec<TextSegment> {+ // piece cp ranges are contiguous, so this clamp bounds total outputlet ccp_text = ccp_text.min(MAX_TEXT_CHARS);- let mut out = String::new();+ let mut segments = Vec::new();for piece in pieces {- // main document body only; headers, footnotes etc. live past ccpText+ // body text only; cps past ccpText are headers, footnotes etc.if piece.start_cp as usize >= ccp_text {continue;}let count = (piece.end_cp as usize).min(ccp_text) - piece.start_cp as usize;let available = word_doc.len().saturating_sub(piece.offset);- if piece.compressed {- let bytes = &word_doc[piece.offset.min(word_doc.len())..][..count.min(available)];- out.push_str(&ansi.decode_without_bom_handling(bytes).0);+ let start = piece.offset.min(word_doc.len());+ let (text, bytes_per_char) = if piece.compressed {+ let bytes = &word_doc[start..][..count.min(available)];+ (ansi.decode_without_bom_handling(bytes).0.into_owned(), 1)} else {- let bytes = &word_doc[piece.offset.min(word_doc.len())..][..(count * 2).min(available)];- out.push_str(&decode_utf16le(bytes));- }+ let bytes = &word_doc[start..][..(count * 2).min(available)];+ (decode_utf16le(bytes), 2)+ };+ segments.push(TextSegment {+ text,+ fc_start: start as u32,+ bytes_per_char,+ });}- out+ segments}fn decode_utf16le(bytes: &[u8]) -> String {@@ -262,81 +294,574 @@ fn decode_utf16le(bytes: &[u8]) -> String {String::from_utf16_lossy(&units)}-fn text_to_blocks(text: &str) -> Vec<Block> {- let mut blocks = Vec::new();- let mut inlines: Vec<Inline> = Vec::new();- let mut buf = String::new();- // per-field flag: true while inside the instruction part (before 0x14)- let mut field_stack: Vec<bool> = Vec::new();+#[derive(Clone, Copy, Default, PartialEq, Eq)]+struct CharProps {+ bold: bool,+ italic: bool,+ strike: bool,+}++#[derive(Clone, Copy, Default)]+struct ParaProps {+ in_table: bool,+ row_end: bool,+ heading: Option<u8>,+}++struct FcRun<T> {+ fc_start: u32,+ fc_end: u32,+ props: T,+}++#[derive(Default)]+struct Formatting {+ char_runs: Vec<FcRun<CharProps>>,+ para_runs: Vec<FcRun<ParaProps>>,+}++impl Formatting {+ fn parse(word_doc: &[u8], table: &[u8], fib: &Fib) -> Self {+ Self {+ char_runs: parse_bin_table(+ word_doc,+ table,+ fib.fc_plcf_bte_chpx,+ fib.lcb_plcf_bte_chpx,+ parse_chpx_fkp,+ ),+ para_runs: parse_bin_table(+ word_doc,+ table,+ fib.fc_plcf_bte_papx,+ fib.lcb_plcf_bte_papx,+ parse_papx_fkp,+ ),+ }+ }+}++/// Walks a bin table: a PLC of page numbers of FKPs in the WordDocument+/// stream.+fn parse_bin_table<T>(+ word_doc: &[u8],+ table: &[u8],+ fc: usize,+ lcb: usize,+ parse_fkp: impl Fn(&[u8], &mut Vec<FcRun<T>>),+) -> Vec<FcRun<T>> {+ let mut runs = Vec::new();+ let Some(end) = fc.checked_add(lcb) else {+ return runs;+ };+ let Some(plc) = table.get(fc..end) else {+ return runs;+ };+ if lcb < 12 || (lcb - 4) % 8 != 0 {+ return runs;+ }+ let n = (lcb - 4) / 8;+ for k in 0..n {+ let pn = read_u32(plc, 4 * (n + 1) + 4 * k) as usize;+ let Some(page_start) = pn.checked_mul(FKP_PAGE_SIZE) else {+ continue;+ };+ if let Some(page) = word_doc.get(page_start..page_start + FKP_PAGE_SIZE) {+ parse_fkp(page, &mut runs);+ }+ }+ runs.sort_by_key(|run| run.fc_start);+ runs+}++fn parse_chpx_fkp(page: &[u8], runs: &mut Vec<FcRun<CharProps>>) {+ let crun = page[FKP_PAGE_SIZE - 1] as usize;+ if crun == 0 || 4 * (crun + 1) + crun >= FKP_PAGE_SIZE {+ return;+ }+ for i in 0..crun {+ let fc_start = read_u32(page, 4 * i);+ let fc_end = read_u32(page, 4 * (i + 1));+ if fc_end <= fc_start {+ continue;+ }+ // rgb entry: word offset of the Chpx within the page; 0 means default+ let rgb = page[4 * (crun + 1) + i] as usize;+ let mut props = CharProps::default();+ if rgb != 0 {+ let offset = rgb * 2;+ if let Some(&cb) = page.get(offset) {+ if let Some(grpprl) = page.get(offset + 1..offset + 1 + cb as usize) {+ props = char_props_from_grpprl(grpprl);+ }+ }+ }+ runs.push(FcRun {+ fc_start,+ fc_end,+ props,+ });+ }+}- fn flush_text(buf: &mut String, inlines: &mut Vec<Inline>) {- if !buf.is_empty() {- inlines.push(Inline::Text(std::mem::take(buf)));+fn parse_papx_fkp(page: &[u8], runs: &mut Vec<FcRun<ParaProps>>) {+ let cpara = page[FKP_PAGE_SIZE - 1] as usize;+ if cpara == 0 || 4 * (cpara + 1) + 13 * cpara >= FKP_PAGE_SIZE {+ return;+ }+ for i in 0..cpara {+ let fc_start = read_u32(page, 4 * i);+ let fc_end = read_u32(page, 4 * (i + 1));+ if fc_end <= fc_start {+ continue;+ }+ // BxPap: first byte is the word offset of the PapxInFkp; 0 means default+ let bx = page[4 * (cpara + 1) + 13 * i] as usize;+ let mut props = ParaProps::default();+ if bx != 0 {+ let offset = bx * 2;+ // PapxInFkp: grpprl length is cb*2 - 1, or cb2*2 when cb is 0+ if let Some(&cb) = page.get(offset) {+ let (start, len) = if cb == 0 {+ match page.get(offset + 1) {+ Some(&cb2) => (offset + 2, cb2 as usize * 2),+ None => (offset, 0),+ }+ } else {+ (offset + 1, (cb as usize) * 2 - 1)+ };+ if let Some(grpprl) = page.get(start..start + len) {+ props = para_props_from_grpprl(grpprl);+ }+ }}+ runs.push(FcRun {+ fc_start,+ fc_end,+ props,+ });}+}- fn flush_paragraph(buf: &mut String, inlines: &mut Vec<Inline>, blocks: &mut Vec<Block>) {- flush_text(buf, inlines);- while matches!(inlines.last(), Some(Inline::LineBreak)) {- inlines.pop();+fn char_props_from_grpprl(grpprl: &[u8]) -> CharProps {+ let mut props = CharProps::default();+ walk_sprms(grpprl, |sprm, operand| {+ // toggle operand: 1 = on, 0x81 = invert style (assume on)+ let on = matches!(operand.first(), Some(1) | Some(0x81));+ match sprm {+ 0x0835 => props.bold = on,+ 0x0836 => props.italic = on,+ 0x0837 => props.strike = on,+ _ => {}}- if has_visible_content(inlines) {- if let Some(Inline::Text(first)) = inlines.first_mut() {- *first = first.trim_start().to_string();+ });+ props+}++fn para_props_from_grpprl(grpprl: &[u8]) -> ParaProps {+ let mut props = ParaProps::default();+ if grpprl.len() < 2 {+ return props;+ }+ props.heading = heading_from_istd(read_u16(grpprl, 0));+ walk_sprms(&grpprl[2..], |sprm, operand| match sprm {+ // sprmPFInTable, sprmPFTtp, sprmPOutLvl, sprmPIstd+ 0x2416 => props.in_table = operand.first().is_some_and(|&v| v != 0),+ 0x2417 => props.row_end = operand.first().is_some_and(|&v| v != 0),+ 0x2640 => {+ if let Some(&level) = operand.first() {+ if level < 9 {+ props.heading = Some((level + 1).min(6));+ }}- if let Some(Inline::Text(last)) = inlines.last_mut() {- *last = last.trim_end().to_string();+ }+ 0x4600 => {+ if operand.len() >= 2 {+ if let Some(level) = heading_from_istd(read_u16(operand, 0)) {+ props.heading = Some(level);+ }}- blocks.push(Block::Paragraph(Paragraph {- kind: ParagraphKind::Normal,- inlines: std::mem::take(inlines),- }));- } else {- inlines.clear();}+ _ => {}+ });+ props+}++/// Built-in style indexes 1..=9 are "heading 1".."heading 9".+fn heading_from_istd(istd: u16) -> Option<u8> {+ if (1..=9).contains(&istd) {+ Some((istd as u8).min(6))+ } else {+ None+ }+}++/// Visits each sprm in a grpprl; operand size comes from the top 3 bits.+fn walk_sprms(grpprl: &[u8], mut visit: impl FnMut(u16, &[u8])) {+ let mut i = 0;+ while i + 2 <= grpprl.len() {+ let sprm = read_u16(grpprl, i);+ i += 2;+ let size = match sprm >> 13 {+ 0 | 1 => 1,+ 2 | 4 | 5 => 2,+ 7 => 3,+ 3 => 4,+ _ => {+ // variable size: length byte first (u16 + 1 for sprmTDefTable)+ if sprm == 0xD608 {+ if i + 2 > grpprl.len() {+ return;+ }+ read_u16(grpprl, i) as usize + 1+ } else {+ match grpprl.get(i) {+ Some(&cb) => cb as usize + 1,+ None => return,+ }+ }+ }+ };+ let Some(operand) = grpprl.get(i..i + size) else {+ return;+ };+ visit(sprm, operand);+ i += size;+ }+}++/// Props for the run containing `fc`, with a cursor for sequential lookups.+fn run_lookup<T: Copy + Default>(runs: &[FcRun<T>], fc: u32, cursor: &mut usize) -> T {+ if runs.is_empty() {+ return T::default();+ }+ if *cursor >= runs.len() || fc < runs[*cursor].fc_start {+ *cursor = runs.partition_point(|run| run.fc_end <= fc);+ } else {+ while *cursor < runs.len() && runs[*cursor].fc_end <= fc {+ *cursor += 1;+ }+ }+ match runs.get(*cursor) {+ Some(run) if fc >= run.fc_start && fc < run.fc_end => run.props,+ _ => T::default(),}+}++fn build_blocks(segments: &[TextSegment], formatting: &Formatting) -> Vec<Block> {+ let mut builder = BlockBuilder {+ formatting,+ blocks: Vec::new(),+ inlines: Vec::new(),+ run_text: String::new(),+ run_props: CharProps::default(),+ char_cursor: 0,+ para_cursor: 0,+ fields: Vec::new(),+ table_rows: Vec::new(),+ row_cells: Vec::new(),+ cell_blocks: Vec::new(),+ };++ for segment in segments {+ let mut fc = segment.fc_start;+ for ch in segment.text.chars() {+ builder.handle_char(ch, fc);+ fc = fc.saturating_add(match segment.bytes_per_char {+ 1 => 1,+ _ => 2 * ch.len_utf16() as u32,+ });+ }+ }+ builder.finish()+}++struct FieldFrame {+ in_instruction: bool,+ instruction: String,+ result_inline_start: usize,+}++struct BlockBuilder<'a> {+ formatting: &'a Formatting,+ blocks: Vec<Block>,+ inlines: Vec<Inline>,+ run_text: String,+ run_props: CharProps,+ char_cursor: usize,+ para_cursor: usize,+ fields: Vec<FieldFrame>,+ table_rows: Vec<TableRow>,+ row_cells: Vec<TableCell>,+ cell_blocks: Vec<Block>,+}- for ch in text.chars() {+impl BlockBuilder<'_> {+ fn handle_char(&mut self, ch: char, fc: u32) {match ch {'\u{13}' => {- field_stack.push(true);- continue;+ self.fields.push(FieldFrame {+ in_instruction: true,+ instruction: String::new(),+ result_inline_start: usize::MAX,+ });+ return;}'\u{14}' => {- if let Some(top) = field_stack.last_mut() {- *top = false;+ self.flush_run();+ let start = self.inlines.len();+ if let Some(frame) = self.fields.last_mut() {+ frame.in_instruction = false;+ frame.result_inline_start = start;}- continue;+ return;}'\u{15}' => {- field_stack.pop();- continue;+ self.flush_run();+ if let Some(frame) = self.fields.pop() {+ self.close_field(frame);+ }+ return;}_ => {}}- if field_stack.iter().any(|&in_instruction| in_instruction) {- continue;++ if let Some(frame) = self.fields.iter_mut().rev().find(|f| f.in_instruction) {+ if frame.instruction.len() < MAX_FIELD_INSTRUCTION_CHARS {+ frame.instruction.push(ch);+ }+ return;}+match ch {- // paragraph mark, cell/row mark, page break, column break- '\r' | '\n' | '\u{7}' | '\u{c}' | '\u{e}' => {- flush_paragraph(&mut buf, &mut inlines, &mut blocks)+ '\r' | '\n' | '\u{c}' | '\u{e}' => {+ let props = self.para_props(fc);+ self.end_paragraph(props);+ }+ '\u{7}' => {+ let props = self.para_props(fc);+ if props.row_end {+ self.end_row();+ } else if props.in_table {+ self.end_cell(props);+ } else {+ // no table context: behave like a paragraph mark+ self.end_paragraph(props);+ }}'\u{b}' => {- flush_text(&mut buf, &mut inlines);- if !inlines.is_empty() {- inlines.push(Inline::LineBreak);+ self.flush_run();+ if !self.inlines.is_empty() {+ self.inlines.push(Inline::LineBreak);}}- '\t' => buf.push('\t'),- '\u{1e}' => buf.push('-'),+ '\t' => self.push_char('\t', fc),+ '\u{1e}' => self.push_char('-', fc),c if c.is_control() || c == '\u{1f}' => {}- c => buf.push(c),+ c => self.push_char(c, fc),+ }+ }++ fn push_char(&mut self, ch: char, fc: u32) {+ let props = run_lookup(&self.formatting.char_runs, fc, &mut self.char_cursor);+ if props != self.run_props {+ self.flush_run();+ self.run_props = props;+ }+ self.run_text.push(ch);+ }++ fn para_props(&mut self, fc: u32) -> ParaProps {+ run_lookup(&self.formatting.para_runs, fc, &mut self.para_cursor)+ }++ fn flush_run(&mut self) {+ if self.run_text.is_empty() {+ return;+ }+ let mut node = Inline::Text(std::mem::take(&mut self.run_text));+ if self.run_props.strike {+ node = Inline::Del(vec![node]);+ }+ if self.run_props.italic {+ node = Inline::Em(vec![node]);+ }+ if self.run_props.bold {+ node = Inline::Strong(vec![node]);+ }+ self.inlines.push(node);+ }++ fn close_field(&mut self, frame: FieldFrame) {+ let Some(href) = hyperlink_target(&frame.instruction) else {+ return;+ };+ let start = frame.result_inline_start.min(self.inlines.len());+ let children = self.inlines.split_off(start);+ if has_visible_content(&children) {+ self.inlines.push(Inline::Link { href, children });+ } else {+ self.inlines.extend(children);+ }+ }++ /// Takes the pending inlines as a paragraph, or `None` when nothing+ /// visible accumulated.+ fn take_paragraph(&mut self, props: ParaProps) -> Option<Block> {+ self.flush_run();+ // links cannot span paragraphs+ for frame in &mut self.fields {+ frame.result_inline_start = 0;+ }+ while matches!(self.inlines.last(), Some(Inline::LineBreak)) {+ self.inlines.pop();+ }+ if !has_visible_content(&self.inlines) {+ self.inlines.clear();+ return None;+ }+ if let Some(Inline::Text(first)) = self.inlines.first_mut() {+ *first = first.trim_start().to_string();+ }+ if let Some(Inline::Text(last)) = self.inlines.last_mut() {+ *last = last.trim_end().to_string();+ }+ let kind = match props.heading {+ Some(level) => ParagraphKind::Heading(level),+ None => ParagraphKind::Normal,+ };+ Some(Block::Paragraph(Paragraph {+ kind,+ inlines: std::mem::take(&mut self.inlines),+ }))+ }++ fn end_paragraph(&mut self, props: ParaProps) {+ let paragraph = self.take_paragraph(props);+ if props.in_table {+ if let Some(paragraph) = paragraph {+ self.cell_blocks.push(paragraph);+ }+ } else {+ self.flush_table();+ if let Some(paragraph) = paragraph {+ self.blocks.push(paragraph);+ }+ }+ }++ fn end_cell(&mut self, props: ParaProps) {+ if let Some(paragraph) = self.take_paragraph(props) {+ self.cell_blocks.push(paragraph);+ }+ self.row_cells.push(TableCell {+ blocks: std::mem::take(&mut self.cell_blocks),+ colspan: NonZeroU32::MIN,+ rowspan: NonZeroU32::MIN,+ });+ }++ /// Row terminator: the TTP mark's own paragraph carries no cell content.+ fn end_row(&mut self) {+ self.flush_run();+ self.inlines.clear();+ if !self.cell_blocks.is_empty() {+ self.row_cells.push(TableCell {+ blocks: std::mem::take(&mut self.cell_blocks),+ colspan: NonZeroU32::MIN,+ rowspan: NonZeroU32::MIN,+ });+ }+ if !self.row_cells.is_empty() {+ self.table_rows.push(TableRow {+ cells: std::mem::take(&mut self.row_cells),+ kind: TableRowKind::Body,+ });+ }+ }++ fn flush_table(&mut self) {+ if !self.cell_blocks.is_empty() || !self.row_cells.is_empty() {+ self.end_row();+ }+ if !self.table_rows.is_empty() {+ self.blocks.push(Block::Table(Table {+ rows: std::mem::take(&mut self.table_rows),+ }));+ }+ }++ fn finish(mut self) -> Vec<Block> {+ self.end_paragraph(ParaProps::default());+ self.flush_table();+ self.blocks+ }+}++/// Target of a HYPERLINK field instruction.+fn hyperlink_target(instruction: &str) -> Option<String> {+ let rest = instruction.trim_start();+ if !rest+ .get(..9)+ .is_some_and(|p| p.eq_ignore_ascii_case("HYPERLINK"))+ {+ return None;+ }+ let mut tokens = field_tokens(&rest[9..]);+ let mut anchor = false;+ while let Some(token) = tokens.next() {+ match token.as_str() {+ "\\l" => anchor = true,+ // switches whose argument is not the target+ "\\o" | "\\t" => {+ tokens.next();+ }+ t if t.starts_with('\\') => {}+ t => {+ return Some(if anchor {+ format!("#{t}")+ } else {+ t.to_string()+ });+ }}}- flush_paragraph(&mut buf, &mut inlines, &mut blocks);+ None+}- blocks+/// Whitespace-separated tokens; quoted strings stay together, unquoted.+fn field_tokens(input: &str) -> impl Iterator<Item = String> + '_ {+ let mut chars = input.chars().peekable();+ std::iter::from_fn(move || {+ while chars.peek().is_some_and(|c| c.is_whitespace()) {+ chars.next();+ }+ match chars.peek() {+ None => None,+ Some('"') => {+ chars.next();+ let mut token = String::new();+ for c in chars.by_ref() {+ if c == '"' {+ break;+ }+ token.push(c);+ }+ Some(token)+ }+ Some(_) => {+ let mut token = String::new();+ while let Some(&c) = chars.peek() {+ if c.is_whitespace() {+ break;+ }+ token.push(c);+ chars.next();+ }+ Some(token)+ }+ }+ })}fn read_u16(data: &[u8], offset: usize) -> u16 {@@ -362,13 +887,103 @@ mod tests {Unicode(&'static str),}+ /// Contiguous formatting runs: (fc_start, fc_end, grpprl). An empty grpprl+ /// becomes a default (offset 0) FKP entry.+ #[derive(Default)]+ struct TestFmt {+ char_runs: Vec<(u32, u32, Vec<u8>)>,+ para_runs: Vec<(u32, u32, Vec<u8>)>,+ }+fn utf16_bytes(text: &str) -> Vec<u8> {text.encode_utf16().flat_map(u16::to_le_bytes).collect()}const TEXT_START: usize = 0x200;+ /// FC of the i-th UTF-16 unit of a single Unicode test piece.+ fn ufc(i: usize) -> u32 {+ (TEXT_START + 2 * i) as u32+ }++ fn sprm1(code: u16, value: u8) -> Vec<u8> {+ let mut out = code.to_le_bytes().to_vec();+ out.push(value);+ out+ }++ fn papx(istd: u16, sprms: &[Vec<u8>]) -> Vec<u8> {+ let mut out = istd.to_le_bytes().to_vec();+ for sprm in sprms {+ out.extend_from_slice(sprm);+ }+ out+ }++ fn build_chpx_fkp(runs: &[(u32, u32, Vec<u8>)]) -> Vec<u8> {+ let mut page = vec![0u8; FKP_PAGE_SIZE];+ let crun = runs.len();+ for (i, (fc_start, fc_end, _)) in runs.iter().enumerate() {+ page[4 * i..4 * i + 4].copy_from_slice(&fc_start.to_le_bytes());+ page[4 * (i + 1)..4 * (i + 1) + 4].copy_from_slice(&fc_end.to_le_bytes());+ }+ let mut blob_end = FKP_PAGE_SIZE - 1;+ for (i, (_, _, grpprl)) in runs.iter().enumerate() {+ if grpprl.is_empty() {+ continue;+ }+ let blob_len = 1 + grpprl.len();+ let mut pos = blob_end - blob_len;+ pos &= !1;+ page[pos] = grpprl.len() as u8;+ page[pos + 1..pos + 1 + grpprl.len()].copy_from_slice(grpprl);+ page[4 * (crun + 1) + i] = (pos / 2) as u8;+ blob_end = pos;+ }+ page[FKP_PAGE_SIZE - 1] = crun as u8;+ page+ }++ fn build_papx_fkp(runs: &[(u32, u32, Vec<u8>)]) -> Vec<u8> {+ let mut page = vec![0u8; FKP_PAGE_SIZE];+ let cpara = runs.len();+ for (i, (fc_start, fc_end, _)) in runs.iter().enumerate() {+ page[4 * i..4 * i + 4].copy_from_slice(&fc_start.to_le_bytes());+ page[4 * (i + 1)..4 * (i + 1) + 4].copy_from_slice(&fc_end.to_le_bytes());+ }+ let mut blob_end = FKP_PAGE_SIZE - 1;+ for (i, (_, _, grpprl)) in runs.iter().enumerate() {+ if grpprl.is_empty() {+ continue;+ }+ let blob = if grpprl.len() % 2 == 1 {+ let mut b = vec![((grpprl.len() + 1) / 2) as u8];+ b.extend_from_slice(grpprl);+ b+ } else {+ let mut b = vec![0, (grpprl.len() / 2) as u8];+ b.extend_from_slice(grpprl);+ b+ };+ let mut pos = blob_end - blob.len();+ pos &= !1;+ page[pos..pos + blob.len()].copy_from_slice(&blob);+ page[4 * (cpara + 1) + 13 * i] = (pos / 2) as u8;+ blob_end = pos;+ }+ page[FKP_PAGE_SIZE - 1] = cpara as u8;+ page+ }+fn build_word_doc_streams(pieces: &[TestPiece], lid: u16) -> (Vec<u8>, Vec<u8>) {+ build_word_doc_streams_fmt(pieces, lid, &TestFmt::default())+ }++ fn build_word_doc_streams_fmt(+ pieces: &[TestPiece],+ lid: u16,+ fmt: &TestFmt,+ ) -> (Vec<u8>, Vec<u8>) {let mut word_doc = vec![0u8; TEXT_START];let mut cps = vec![0u32];let mut pcds = Vec::new();@@ -403,6 +1018,30 @@ mod tests {let mut table = vec![0x02];table.extend_from_slice(&(plc.len() as u32).to_le_bytes());table.extend_from_slice(&plc);+ let lcb_clx = table.len() as u32;++ let append_fkp =+ |word_doc: &mut Vec<u8>, table: &mut Vec<u8>, page: Vec<u8>, runs: &[(u32, u32, Vec<u8>)]| {+ while word_doc.len() % FKP_PAGE_SIZE != 0 {+ word_doc.push(0);+ }+ let pn = (word_doc.len() / FKP_PAGE_SIZE) as u32;+ word_doc.extend_from_slice(&page);+ let bin_table_offset = table.len() as u32;+ table.extend_from_slice(&runs[0].0.to_le_bytes());+ table.extend_from_slice(&runs.last().unwrap().1.to_le_bytes());+ table.extend_from_slice(&pn.to_le_bytes());+ (bin_table_offset, 12u32)+ };++ let chpx_bt = (!fmt.char_runs.is_empty()).then(|| {+ let page = build_chpx_fkp(&fmt.char_runs);+ append_fkp(&mut word_doc, &mut table, page, &fmt.char_runs)+ });+ let papx_bt = (!fmt.para_runs.is_empty()).then(|| {+ let page = build_papx_fkp(&fmt.para_runs);+ append_fkp(&mut word_doc, &mut table, page, &fmt.para_runs)+ });word_doc[0x00..0x02].copy_from_slice(&MAGIC_WORD_97.to_le_bytes());word_doc[0x02..0x04].copy_from_slice(&0x00C1u16.to_le_bytes());@@ -410,7 +1049,15 @@ mod tests {word_doc[0x0A..0x0C].copy_from_slice(&0x0200u16.to_le_bytes());word_doc[0x4C..0x50].copy_from_slice(&ccp_text.to_le_bytes());word_doc[0x1A2..0x1A6].copy_from_slice(&0u32.to_le_bytes());- word_doc[0x1A6..0x1AA].copy_from_slice(&(table.len() as u32).to_le_bytes());+ word_doc[0x1A6..0x1AA].copy_from_slice(&lcb_clx.to_le_bytes());+ if let Some((fc, lcb)) = chpx_bt {+ word_doc[0xFA..0xFE].copy_from_slice(&fc.to_le_bytes());+ word_doc[0xFE..0x102].copy_from_slice(&lcb.to_le_bytes());+ }+ if let Some((fc, lcb)) = papx_bt {+ word_doc[0x102..0x106].copy_from_slice(&fc.to_le_bytes());+ word_doc[0x106..0x10A].copy_from_slice(&lcb.to_le_bytes());+ }(word_doc, table)}@@ -425,7 +1072,16 @@ mod tests {}fn build_doc(pieces: &[TestPiece], lid: u16, summary: Option<&[u8]>) -> Vec<u8> {- let (word_doc, table) = build_word_doc_streams(pieces, lid);+ build_doc_fmt(pieces, lid, summary, &TestFmt::default())+ }++ fn build_doc_fmt(+ pieces: &[TestPiece],+ lid: u16,+ summary: Option<&[u8]>,+ fmt: &TestFmt,+ ) -> Vec<u8> {+ let (word_doc, table) = build_word_doc_streams_fmt(pieces, lid, fmt);let mut streams: Vec<(&str, &[u8])> = vec![("WordDocument", &word_doc), ("1Table", &table)];if let Some(summary) = summary {streams.push(("\x05SummaryInformation", summary));@@ -470,20 +1126,45 @@ mod tests {data}+ /// Readable markup for inline assertions, e.g. `a <b>x</b> <a href=u>y</a>`.+ fn inline_markup(inlines: &[Inline]) -> String {+ inlines+ .iter()+ .map(|inline| match inline {+ Inline::Text(t) => t.clone(),+ Inline::LineBreak => "\n".to_string(),+ Inline::Strong(c) => format!("<b>{}</b>", inline_markup(c)),+ Inline::Em(c) => format!("<i>{}</i>", inline_markup(c)),+ Inline::Del(c) => format!("<del>{}</del>", inline_markup(c)),+ Inline::Link { href, children } => {+ format!("<a href={}>{}</a>", href, inline_markup(children))+ }+ _ => String::new(),+ })+ .collect()+ }++ fn inline_text(inlines: &[Inline]) -> String {+ inlines+ .iter()+ .map(|inline| match inline {+ Inline::Text(t) => t.clone(),+ Inline::LineBreak => "\n".to_string(),+ Inline::Link { children, .. } => inline_text(children),+ Inline::Strong(c) | Inline::Em(c) | Inline::Del(c) | Inline::Sup(c) | Inline::Sub(c) => {+ inline_text(c)+ }+ _ => String::new(),+ })+ .collect()+ }+fn paragraph_texts(document: &Document) -> Vec<String> {document.blocks.iter().map(|block| match block {- Block::Paragraph(p) => p- .inlines- .iter()- .map(|inline| match inline {- Inline::Text(t) => t.as_str(),- Inline::LineBreak => "\n",- _ => "",- })- .collect::<String>(),+ Block::Paragraph(p) => inline_text(&p.inlines),_ => String::new(),}).collect()@@ -541,7 +1222,10 @@ mod tests {0x0409,None,);- assert_eq!(paragraph_texts(&parse(&data)), vec!["Café \u{201C}quoted\u{201D}"]);+ assert_eq!(+ paragraph_texts(&parse(&data)),+ vec!["Café \u{201C}quoted\u{201D}"]+ );}#[test]@@ -572,10 +1256,61 @@ mod tests {assert_eq!(paragraph_texts(&parse(&data)), vec!["See the site now"]);}+ #[test]+ fn hyperlink_fields_become_links() {+ let data = build_doc(+ &[TestPiece::Unicode(+ "See \u{13} HYPERLINK \"https://example.com\" \u{14}the site\u{15} now\r",+ )],+ 0x0409,+ None,+ );+ let document = parse(&data);+ let Block::Paragraph(p) = &document.blocks[0] else {+ panic!("expected paragraph");+ };+ assert_eq!(+ inline_markup(&p.inlines),+ "See <a href=https://example.com>the site</a> now"+ );+ }++ #[test]+ fn anchor_hyperlink_targets_bookmark() {+ let data = build_doc(+ &[TestPiece::Unicode(+ "Go \u{13} HYPERLINK \\l \"section1\" \u{14}there\u{15}\r",+ )],+ 0x0409,+ None,+ );+ let document = parse(&data);+ let Block::Paragraph(p) = &document.blocks[0] else {+ panic!("expected paragraph");+ };+ assert_eq!(inline_markup(&p.inlines), "Go <a href=#section1>there</a>");+ }++ #[test]+ fn non_hyperlink_fields_keep_result_text_only() {+ let data = build_doc(+ &[TestPiece::Unicode("Page \u{13} PAGE \u{14}7\u{15} of 9\r")],+ 0x0409,+ None,+ );+ let document = parse(&data);+ let Block::Paragraph(p) = &document.blocks[0] else {+ panic!("expected paragraph");+ };+ assert_eq!(inline_markup(&p.inlines), "Page 7 of 9");+ }+#[test]fn cell_marks_and_line_breaks() {let data = build_doc(- &[TestPiece::Unicode("cell one\u{7}cell two\u{7}line\u{b}break\r")],+ &[TestPiece::Unicode(+ "cell one\u{7}cell two\u{7}line\u{b}break\r",+ )],0x0409,None,);@@ -585,6 +1320,112 @@ mod tests {);}+ #[test]+ fn bold_italic_strike_runs() {+ let text = "plain bold and style\r";+ let fmt = TestFmt {+ char_runs: vec![+ (ufc(0), ufc(6), vec![]),+ (ufc(6), ufc(10), sprm1(0x0835, 1)),+ (ufc(10), ufc(15), vec![]),+ (+ ufc(15),+ ufc(20),+ [sprm1(0x0836, 1), sprm1(0x0837, 1)].concat(),+ ),+ (ufc(20), ufc(21), vec![]),+ ],+ ..TestFmt::default()+ };+ let data = build_doc_fmt(&[TestPiece::Unicode(text)], 0x0409, None, &fmt);+ let document = parse(&data);+ let Block::Paragraph(p) = &document.blocks[0] else {+ panic!("expected paragraph");+ };+ assert_eq!(+ inline_markup(&p.inlines),+ "plain <b>bold</b> and <i><del>style</del></i>"+ );+ }++ #[test]+ fn headings_from_istd_and_outline_level() {+ let text = "Title\rSection\rBody\r";+ let fmt = TestFmt {+ para_runs: vec![+ (ufc(0), ufc(6), papx(1, &[])),+ (ufc(6), ufc(14), papx(0, &[sprm1(0x2640, 1)])),+ (ufc(14), ufc(19), papx(0, &[])),+ ],+ ..TestFmt::default()+ };+ let data = build_doc_fmt(&[TestPiece::Unicode(text)], 0x0409, None, &fmt);+ let document = parse(&data);+ let kinds: Vec<ParagraphKind> = document+ .blocks+ .iter()+ .filter_map(|b| match b {+ Block::Paragraph(p) => Some(p.kind),+ _ => None,+ })+ .collect();+ assert_eq!(+ kinds,+ vec![+ ParagraphKind::Heading(1),+ ParagraphKind::Heading(2),+ ParagraphKind::Normal+ ]+ );+ }++ #[test]+ fn table_from_cell_and_row_marks() {+ let text = "A1\u{7}B1\u{7}\u{7}A2\u{7}B2\u{7}\u{7}after\r";+ let in_table = papx(0, &[sprm1(0x2416, 1)]);+ let row_end = papx(0, &[sprm1(0x2416, 1), sprm1(0x2417, 1)]);+ let fmt = TestFmt {+ para_runs: vec![+ (ufc(0), ufc(3), in_table.clone()),+ (ufc(3), ufc(6), in_table.clone()),+ (ufc(6), ufc(7), row_end.clone()),+ (ufc(7), ufc(10), in_table.clone()),+ (ufc(10), ufc(13), in_table),+ (ufc(13), ufc(14), row_end),+ (ufc(14), ufc(20), papx(0, &[])),+ ],+ ..TestFmt::default()+ };+ let data = build_doc_fmt(&[TestPiece::Unicode(text)], 0x0409, None, &fmt);+ let document = parse(&data);++ let Block::Table(table) = &document.blocks[0] else {+ panic!("expected table, got {:?}", document.blocks[0]);+ };+ let cells: Vec<Vec<String>> = table+ .rows+ .iter()+ .map(|row| {+ row+ .cells+ .iter()+ .map(|cell| {+ cell+ .blocks+ .iter()+ .map(|b| match b {+ Block::Paragraph(p) => inline_text(&p.inlines),+ _ => String::new(),+ })+ .collect::<String>()+ })+ .collect()+ })+ .collect();+ assert_eq!(cells, vec![vec!["A1", "B1"], vec!["A2", "B2"]]);+ assert_eq!(paragraph_texts(&parse(&data)).last().unwrap(), "after");+ }+#[test]fn legacy_word_95_contiguous_text() {let text_bytes = cp1251("Стаття перша.\rСтаття друга.\r");@@ -647,7 +1488,8 @@ mod tests {#[test]fn corrupt_piece_table_falls_back_to_fc_range() {- let (mut word_doc, _) = build_word_doc_streams(&[TestPiece::Unicode("good text here\r")], 0x0409);+ let (mut word_doc, _) =+ build_word_doc_streams(&[TestPiece::Unicode("good text here\r")], 0x0409);word_doc[0x0A..0x0C].copy_from_slice(&0x1200u16.to_le_bytes());let fc_mac = word_doc.len() as u32;word_doc[0x18..0x1C].copy_from_slice(&(TEXT_START as u32).to_le_bytes());diff --git a/apps/api/native/src/document/providers/docx.rs b/apps/api/native/src/document/providers/docx.rsindex 866f0bf6d..2e2b60c8b 100644--- a/apps/api/native/src/document/providers/docx.rs+++ b/apps/api/native/src/document/providers/docx.rs@@ -102,8 +102,12 @@ fn read_relationships<R: Read + Seek>(zip: &mut ZipArchive<R>, path: &str) -> Refn read_core_properties<R: Read + Seek>(zip: &mut ZipArchive<R>) -> Option<DocumentMetadata> {let text = read_zip_text(zip, "docProps/core.xml")?;let xml = XmlDoc::parse(strip_bom(&text)).ok()?;- let element_text =- |local: &str| xml.descendants().find(|n| is_tag(n, local)).and_then(|n| n.text());+ let element_text = |local: &str| {+ xml+ .descendants()+ .find(|n| is_tag(n, local))+ .and_then(|n| n.text())+ };let mut meta = DocumentMetadata::default();if let Some(title) = element_text("title") {@@ -643,7 +647,10 @@ fn parse_list(nodes: &[Node], mut i: usize, ctx: Ctx) -> (List, usize) {items: Vec::new(),list_type: ListType::Unordered,};- let Some(base) = nodes.get(i).and_then(|n| paragraph_list_info(n, ctx.numbering)) else {+ let Some(base) = nodes+ .get(i)+ .and_then(|n| paragraph_list_info(n, ctx.numbering))+ else {return (list, i + 1);};list.list_type = base.list_type;@@ -748,7 +755,11 @@ fn read_notes<R: Read + Seek>(zip: &mut ZipArchive<R>, kind: NoteKind, ctx: Ctx)"word/_rels/footnotes.xml.rels","footnote",),- NoteKind::Endnote => ("word/endnotes.xml", "word/_rels/endnotes.xml.rels", "endnote"),+ NoteKind::Endnote => (+ "word/endnotes.xml",+ "word/_rels/endnotes.xml.rels",+ "endnote",+ ),};let Some(text) = read_zip_text(zip, xml_path) else {return Vec::new();@@ -764,7 +775,10 @@ fn read_notes<R: Read + Seek>(zip: &mut ZipArchive<R>, kind: NoteKind, ctx: Ctx)let Some(id) = attr(&n, "id") else {continue;};- if matches!(attr(&n, "type"), Some("separator" | "continuationSeparator")) {+ if matches!(+ attr(&n, "type"),+ Some("separator" | "continuationSeparator")+ ) {continue;}notes.push(Note {@@ -800,4 +814,3 @@ fn read_comments<R: Read + Seek>(zip: &mut ZipArchive<R>, ctx: Ctx) -> Vec<Comme}comments}-diff --git a/apps/api/native/src/document/providers/factory.rs b/apps/api/native/src/document/providers/factory.rsindex e9b98480f..b9bdb0b52 100644--- a/apps/api/native/src/document/providers/factory.rs+++ b/apps/api/native/src/document/providers/factory.rs@@ -2,8 +2,8 @@ use super::doc::DocProvider;use super::docx::DocxProvider;use super::odt::OdtProvider;use super::rtf::RtfProvider;-use super::DocumentProvider;use super::xlsx::XlsxProvider;+use super::DocumentProvider;use napi_derive::napi;#[napi]diff --git a/apps/api/native/src/document/providers/odt.rs b/apps/api/native/src/document/providers/odt.rsindex d894face8..92a35593a 100644--- a/apps/api/native/src/document/providers/odt.rs+++ b/apps/api/native/src/document/providers/odt.rs@@ -57,8 +57,12 @@ impl DocumentProvider for OdtProvider {fn read_meta<R: Read + Seek>(zip: &mut ZipArchive<R>) -> Option<DocumentMetadata> {let text = read_zip_text(zip, "meta.xml")?;let xml = XmlDoc::parse(strip_bom(&text)).ok()?;- let element_text =- |local: &str| xml.descendants().find(|n| is_tag(n, local)).and_then(|n| n.text());+ let element_text = |local: &str| {+ xml+ .descendants()+ .find(|n| is_tag(n, local))+ .and_then(|n| n.text())+ };let mut meta = DocumentMetadata::default();if let Some(title) = element_text("title") {@@ -384,7 +388,6 @@ impl Parser<'_> {} else if is_tag(&c, "line-break") {out.push(Inline::LineBreak);} else if is_tag(&c, "s") {- // clamped so a malformed count cannot force a huge allocationlet count = attr(&c, "c").and_then(|v| v.parse::<usize>().ok()).unwrap_or(1)diff --git a/apps/api/native/src/document/providers/rtf.rs b/apps/api/native/src/document/providers/rtf.rsindex 77d791563..260216114 100644--- a/apps/api/native/src/document/providers/rtf.rs+++ b/apps/api/native/src/document/providers/rtf.rs@@ -36,8 +36,7 @@ impl DocumentProvider for RtfProvider {}}-/// Code page declared by the first \ansicpg control word. Token-based so-/// escaped literals like "\\ansicpg" in body text cannot match.+/// Code page declared by the first \ansicpg control word.fn detect_encoding(src: &[u8]) -> &'static Encoding {for token in Tokens::new(src) {if let Token::Control("ansicpg", Some(codepage)) = token {@@ -121,8 +120,7 @@ impl<'a> Tokens<'a> {num_end += 1;}if num_end > num_start {- // consume the digit run even when it overflows i32, so malformed- // parameters do not leak digits into the text+ // consumed even when the value overflows i32self.pos = num_end;if let Some(n) = std::str::from_utf8(&self.src[num_start..num_end]).ok()@@ -146,8 +144,7 @@ impl<'a> Tokens<'a> {}}-/// Fallback byte count declared by \uc. Kept as declared: skipping consumes-/// at most one byte per input token, so it is bounded by the input length.+/// Fallback byte count declared by \uc.fn uc_count(value: Option<i32>) -> usize {value.unwrap_or(1).max(0) as usize}@@ -161,8 +158,7 @@ fn hex_val(b: u8) -> Option<u8> {}}-/// Accumulates text, buffering raw bytes so multi-byte code pages decode-/// across adjacent bytes.+/// Text accumulator; raw bytes are buffered and decoded as a unit.struct TextAccum {encoding: &'static Encoding,pending: Vec<u8>,@@ -486,7 +482,10 @@ impl BodyParser {self.state = CharState::default();}"trowd" => {- self.table.get_or_insert_with(TableBuilder::default).start_row();+ self+ .table+ .get_or_insert_with(TableBuilder::default)+ .start_row();self.in_table_cell = false;}"intbl" => self.in_table_cell = true,@@ -756,7 +755,8 @@ mod tests {#[test]fn styles_and_tables_still_parse() {- let rtf = b"{\\rtf1\\ansi {\\b Bold} and {\\i italic}\\par\\trowd\\intbl A\\cell B\\cell\\row\\par}";+ let rtf =+ b"{\\rtf1\\ansi {\\b Bold} and {\\i italic}\\par\\trowd\\intbl A\\cell B\\cell\\row\\par}";let document = RtfProvider::new().parse_buffer(rtf).unwrap();assert!(matches!(document.blocks[0], Block::Paragraph(_)));assert!(document@@ -779,8 +779,7 @@ mod tests {#[test]fn metadata_keeps_text_of_nested_formatting_groups() {- let rtf =- b"{\\rtf1{\\info{\\title Some {\\b Bold} Title{\\*\\junk secret}}}Body\\par}";+ let rtf = b"{\\rtf1{\\info{\\title Some {\\b Bold} Title{\\*\\junk secret}}}Body\\par}";let document = RtfProvider::new().parse_buffer(rtf).unwrap();assert_eq!(document.metadata.title.as_deref(), Some("Some Bold Title"));}@@ -821,6 +820,3 @@ mod tests {assert_eq!(paragraphs(rtf), vec!["У", "done"]);}}---diff --git a/apps/api/native/src/engpicker.rs b/apps/api/native/src/engpicker.rsindex b030eda7b..157a212ca 100644--- a/apps/api/native/src/engpicker.rs+++ b/apps/api/native/src/engpicker.rs@@ -7,57 +7,57 @@ use tokio::task;#[derive(Deserialize, Serialize)]#[napi(object)]pub struct EngpickerUrlResult {- pub url: String,- pub cdp_basic_markdown: Option<String>,- pub cdp_basic_success: bool,- pub cdp_stealth_markdown: Option<String>,- pub cdp_stealth_success: bool,- pub tls_basic_markdown: Option<String>,- pub tls_basic_success: bool,- pub tls_stealth_markdown: Option<String>,- pub tls_stealth_success: bool,+ pub url: String,+ pub cdp_basic_markdown: Option<String>,+ pub cdp_basic_success: bool,+ pub cdp_stealth_markdown: Option<String>,+ pub cdp_stealth_success: bool,+ pub tls_basic_markdown: Option<String>,+ pub tls_basic_success: bool,+ pub tls_stealth_markdown: Option<String>,+ pub tls_stealth_success: bool,}/// Verdict for a single URL#[derive(Serialize)]#[napi(object)]pub struct EngpickerUrlVerdict {- pub url: String,- pub tls_client_sufficient: bool,- pub cdp_failed: bool,- pub similarity: Option<f64>,- pub reason: String,+ pub url: String,+ pub tls_client_sufficient: bool,+ pub cdp_failed: bool,+ pub similarity: Option<f64>,+ pub reason: String,}/// Final verdict enum#[derive(Serialize)]#[napi(string_enum)]pub enum EngpickerFinalVerdict {- /// tlsclient is sufficient for this site- TlsClientOk,- /// Chrome CDP is required for proper rendering- ChromeCdpRequired,- /// Too many CDP failures to determine verdict- Uncertain,+ /// tlsclient is sufficient for this site+ TlsClientOk,+ /// Chrome CDP is required for proper rendering+ ChromeCdpRequired,+ /// Too many CDP failures to determine verdict+ Uncertain,}/// Final verdict result#[derive(Serialize)]#[napi(object)]pub struct EngpickerVerdict {- pub url_verdicts: Vec<EngpickerUrlVerdict>,- pub tls_client_ok_count: u32,- pub chrome_cdp_required_count: u32,- pub cdp_failed_count: u32,- pub total_urls: u32,- pub verdict: EngpickerFinalVerdict,+ pub url_verdicts: Vec<EngpickerUrlVerdict>,+ pub tls_client_ok_count: u32,+ pub chrome_cdp_required_count: u32,+ pub cdp_failed_count: u32,+ pub total_urls: u32,+ pub verdict: EngpickerFinalVerdict,}/// Compute engpicker verdict using Levenshtein distance to compare tlsclient vs chrome-cdp results.-///+////// Chrome-CDP is the gold standard. We compare tlsclient markdown against it to determine/// if tlsclient is sufficient for scraping this site (i.e., JS rendering not required).-///+////// Arguments:/// - results: scrape results for each URL/// - similarity_threshold: minimum similarity (0.0-1.0) for tlsclient to be considered sufficient@@ -65,142 +65,158 @@ pub struct EngpickerVerdict {/// - cdp_failure_threshold: maximum ratio of CDP failures before verdict becomes uncertain#[napi]pub async fn compute_engpicker_verdict(- results: Vec<EngpickerUrlResult>,- similarity_threshold: f64,- success_rate_threshold: f64,- cdp_failure_threshold: f64,+ results: Vec<EngpickerUrlResult>,+ similarity_threshold: f64,+ success_rate_threshold: f64,+ cdp_failure_threshold: f64,) -> napi::Result<EngpickerVerdict> {- task::spawn_blocking(move || {- _compute_engpicker_verdict(results, similarity_threshold, success_rate_threshold, cdp_failure_threshold)- })- .await- .map_err(|e| {- napi::Error::new(- napi::Status::GenericFailure,- format!("compute_engpicker_verdict join error: {e}"),- )- })?+ task::spawn_blocking(move || {+ _compute_engpicker_verdict(+ results,+ similarity_threshold,+ success_rate_threshold,+ cdp_failure_threshold,+ )+ })+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("compute_engpicker_verdict join error: {e}"),+ )+ })?}fn _compute_engpicker_verdict(- results: Vec<EngpickerUrlResult>,- similarity_threshold: f64,- success_rate_threshold: f64,- cdp_failure_threshold: f64,+ results: Vec<EngpickerUrlResult>,+ similarity_threshold: f64,+ success_rate_threshold: f64,+ cdp_failure_threshold: f64,) -> napi::Result<EngpickerVerdict> {- let url_verdicts: Vec<EngpickerUrlVerdict> = results- .iter()- .map(|result| {- // Get the best chrome-cdp result as gold standard (prefer stealth if both succeeded)- let gold_standard = if result.cdp_stealth_success && result.cdp_stealth_markdown.is_some() {- result.cdp_stealth_markdown.as_ref()- } else if result.cdp_basic_success && result.cdp_basic_markdown.is_some() {- result.cdp_basic_markdown.as_ref()- } else {- None- };-- // Get the best tlsclient result (prefer stealth if both succeeded)- let tls_result = if result.tls_stealth_success && result.tls_stealth_markdown.is_some() {- result.tls_stealth_markdown.as_ref()- } else if result.tls_basic_success && result.tls_basic_markdown.is_some() {- result.tls_basic_markdown.as_ref()- } else {- None- };-- // If chrome-cdp failed, we can't evaluate this URL- let gold_standard = match gold_standard {- Some(gs) if !gs.is_empty() => gs,- _ => {- return EngpickerUrlVerdict {- url: result.url.clone(),- tls_client_sufficient: false,- cdp_failed: true,- similarity: None,- reason: "chrome-cdp failed".to_string(),- };- }- };-- // If tlsclient failed entirely, it's definitely not enough- let tls_result = match tls_result {- Some(tls) if !tls.is_empty() => tls,- _ => {- return EngpickerUrlVerdict {- url: result.url.clone(),- tls_client_sufficient: false,- cdp_failed: false,- similarity: None,- reason: "tlsclient failed".to_string(),- };- }- };-- // Calculate Levenshtein distance and normalize to similarity score- let distance = levenshtein(gold_standard, tls_result);- let max_length = gold_standard.len().max(tls_result.len());- let similarity = if max_length > 0 {- 1.0 - (distance as f64 / max_length as f64)- } else {- 1.0- };-- let tls_client_sufficient = similarity >= similarity_threshold;-- let reason = if tls_client_sufficient {- format!("{:.1}% similar - tlsclient captures full content", similarity * 100.0)- } else {- format!("{:.1}% similar - JS rendering likely required", similarity * 100.0)- };-- EngpickerUrlVerdict {- url: result.url.clone(),- tls_client_sufficient,- cdp_failed: false,- similarity: Some(similarity),- reason,- }- })- .collect();-- let total_urls = url_verdicts.len() as u32;- let cdp_failed_count = url_verdicts.iter().filter(|v| v.cdp_failed).count() as u32;- let tls_client_ok_count = url_verdicts.iter().filter(|v| v.tls_client_sufficient).count() as u32;- let chrome_cdp_required_count = url_verdicts.iter().filter(|v| !v.tls_client_sufficient && !v.cdp_failed).count() as u32;-- // Determine final verdict- let verdict = if total_urls == 0 {- EngpickerFinalVerdict::Uncertain+ let url_verdicts: Vec<EngpickerUrlVerdict> = results+ .iter()+ .map(|result| {+ // Get the best chrome-cdp result as gold standard (prefer stealth if both succeeded)+ let gold_standard = if result.cdp_stealth_success && result.cdp_stealth_markdown.is_some() {+ result.cdp_stealth_markdown.as_ref()+ } else if result.cdp_basic_success && result.cdp_basic_markdown.is_some() {+ result.cdp_basic_markdown.as_ref()+ } else {+ None+ };++ // Get the best tlsclient result (prefer stealth if both succeeded)+ let tls_result = if result.tls_stealth_success && result.tls_stealth_markdown.is_some() {+ result.tls_stealth_markdown.as_ref()+ } else if result.tls_basic_success && result.tls_basic_markdown.is_some() {+ result.tls_basic_markdown.as_ref()+ } else {+ None+ };++ // If chrome-cdp failed, we can't evaluate this URL+ let gold_standard = match gold_standard {+ Some(gs) if !gs.is_empty() => gs,+ _ => {+ return EngpickerUrlVerdict {+ url: result.url.clone(),+ tls_client_sufficient: false,+ cdp_failed: true,+ similarity: None,+ reason: "chrome-cdp failed".to_string(),+ };+ }+ };++ // If tlsclient failed entirely, it's definitely not enough+ let tls_result = match tls_result {+ Some(tls) if !tls.is_empty() => tls,+ _ => {+ return EngpickerUrlVerdict {+ url: result.url.clone(),+ tls_client_sufficient: false,+ cdp_failed: false,+ similarity: None,+ reason: "tlsclient failed".to_string(),+ };+ }+ };++ // Calculate Levenshtein distance and normalize to similarity score+ let distance = levenshtein(gold_standard, tls_result);+ let max_length = gold_standard.len().max(tls_result.len());+ let similarity = if max_length > 0 {+ 1.0 - (distance as f64 / max_length as f64)+ } else {+ 1.0+ };++ let tls_client_sufficient = similarity >= similarity_threshold;++ let reason = if tls_client_sufficient {+ format!(+ "{:.1}% similar - tlsclient captures full content",+ similarity * 100.0+ )+ } else {+ format!(+ "{:.1}% similar - JS rendering likely required",+ similarity * 100.0+ )+ };++ EngpickerUrlVerdict {+ url: result.url.clone(),+ tls_client_sufficient,+ cdp_failed: false,+ similarity: Some(similarity),+ reason,+ }+ })+ .collect();++ let total_urls = url_verdicts.len() as u32;+ let cdp_failed_count = url_verdicts.iter().filter(|v| v.cdp_failed).count() as u32;+ let tls_client_ok_count = url_verdicts+ .iter()+ .filter(|v| v.tls_client_sufficient)+ .count() as u32;+ let chrome_cdp_required_count = url_verdicts+ .iter()+ .filter(|v| !v.tls_client_sufficient && !v.cdp_failed)+ .count() as u32;++ // Determine final verdict+ let verdict = if total_urls == 0 {+ EngpickerFinalVerdict::Uncertain+ } else {+ let cdp_failure_rate = cdp_failed_count as f64 / total_urls as f64;++ // If too many CDP failures, we can't make a confident verdict+ if cdp_failure_rate > cdp_failure_threshold {+ EngpickerFinalVerdict::Uncertain} else {- let cdp_failure_rate = cdp_failed_count as f64 / total_urls as f64;-- // If too many CDP failures, we can't make a confident verdict- if cdp_failure_rate > cdp_failure_threshold {- EngpickerFinalVerdict::Uncertain+ // Calculate success rate among URLs where we could actually compare+ let comparable_urls = total_urls - cdp_failed_count;+ if comparable_urls == 0 {+ EngpickerFinalVerdict::Uncertain+ } else {+ let tls_ok_rate = tls_client_ok_count as f64 / comparable_urls as f64;+ if tls_ok_rate >= success_rate_threshold {+ EngpickerFinalVerdict::TlsClientOk} else {- // Calculate success rate among URLs where we could actually compare- let comparable_urls = total_urls - cdp_failed_count;- if comparable_urls == 0 {- EngpickerFinalVerdict::Uncertain- } else {- let tls_ok_rate = tls_client_ok_count as f64 / comparable_urls as f64;- if tls_ok_rate >= success_rate_threshold {- EngpickerFinalVerdict::TlsClientOk- } else {- EngpickerFinalVerdict::ChromeCdpRequired- }- }+ EngpickerFinalVerdict::ChromeCdpRequired}- };-- Ok(EngpickerVerdict {- url_verdicts,- tls_client_ok_count,- chrome_cdp_required_count,- cdp_failed_count,- total_urls,- verdict,- })+ }+ }+ };++ Ok(EngpickerVerdict {+ url_verdicts,+ tls_client_ok_count,+ chrome_cdp_required_count,+ cdp_failed_count,+ total_urls,+ verdict,+ })}-diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rsindex 15c4faed4..71bae49f6 100644--- a/apps/api/native/src/html.rs+++ b/apps/api/native/src/html.rs@@ -11,8 +11,9 @@ use serde_json::Value;use tokio::task;use url::Url;-static URL_REGEX: LazyLock<Regex> =- LazyLock::new(|| Regex::new(r#"url\(['"]?([^'")]+)['"]?\)"#).expect("URL_REGEX is a valid static regex pattern"));+static URL_REGEX: LazyLock<Regex> = LazyLock::new(|| {+ Regex::new(r#"url\(['"]?([^'")]+)['"]?\)"#).expect("URL_REGEX is a valid static regex pattern")+});use crate::utils::to_napi_err;diff --git a/apps/api/native/src/logging.rs b/apps/api/native/src/logging.rsindex bf444e895..d29c778e9 100644--- a/apps/api/native/src/logging.rs+++ b/apps/api/native/src/logging.rs@@ -40,9 +40,10 @@ impl Visit for FieldVisitor {if field.name() == "message" {self.message = Some(format!("{:?}", value));} else {- self- .fields- .insert(field.name().to_string(), Value::String(format!("{:?}", value)));+ self.fields.insert(+ field.name().to_string(),+ Value::String(format!("{:?}", value)),+ );}}@@ -254,10 +255,7 @@ mod tests {fn test_error_preserves_logs() {let traced: TracingResult<napi::Result<()>> = with_native_tracing(None, "test", || {tracing::info!("before error");- Err(napi::Error::new(- napi::Status::GenericFailure,- "test error",- ))+ Err(napi::Error::new(napi::Status::GenericFailure, "test error"))});assert!(traced.value.is_err());diff --git a/apps/api/native/src/pdf.rs b/apps/api/native/src/pdf.rsindex f97fc2743..400f010f2 100644--- a/apps/api/native/src/pdf.rs+++ b/apps/api/native/src/pdf.rs@@ -1,6 +1,6 @@use napi::bindgen_prelude::*;use napi_derive::napi;-use pdf_inspector::{PdfOptions, PdfType, process_pdf_with_options as rust_process_pdf};+use pdf_inspector::{process_pdf_with_options as rust_process_pdf, PdfOptions, PdfType};use crate::logging::{embed_logs_in_error, with_native_tracing, NativeContext, NativeLogEntry};@@ -59,7 +59,10 @@ pub fn process_pdf(let result = rust_process_pdf(&path, opts).map_err(|e| {tracing::error!(error = %e, "PDF processing failed");- Error::new(Status::GenericFailure, format!("Failed to process PDF: {e}"))+ Error::new(+ Status::GenericFailure,+ format!("Failed to process PDF: {e}"),+ )})?;tracing::info!(@@ -86,10 +89,7 @@ pub fn process_pdf(/// Skips text extraction, markdown generation, and layout analysis./// Pass `ctx` (NativeContext) for structured tracing with scrape_id/url.#[napi]-pub fn detect_pdf(- path: String,- ctx: Option<NativeContext>,-) -> Result<PdfProcessResult> {+pub fn detect_pdf(path: String, ctx: Option<NativeContext>) -> Result<PdfProcessResult> {let traced = with_native_tracing(ctx.as_ref(), "pdf", || {tracing::info!("starting PDF detection");