Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(document): add support for parsing legacy .doc files (#2808)
firecrawl-spring[bot]3d2aae674109
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.tomlindex dbce050d4..18833603f 100644--- a/apps/api/native/Cargo.toml+++ b/apps/api/native/Cargo.toml@@ -27,6 +27,7 @@ texting_robots = "0.2.2"url = "2.5.7"zip = "5.0.0"calamine = "0.26"+cfb = "0.10"tokio = "1.48.0"[build-dependencies]diff --git a/apps/api/native/src/document/providers/doc.rs b/apps/api/native/src/document/providers/doc.rsnew file mode 100644index 000000000..1c94b656b--- /dev/null+++ b/apps/api/native/src/document/providers/doc.rs@@ -0,0 +1,420 @@+use crate::document::model::*;+use crate::document::providers::DocumentProvider;+use cfb::CompoundFile;+use std::error::Error;+use std::io::Cursor;+use std::io::Read;++pub struct DocProvider;++impl DocProvider {+ pub fn new() -> Self {+ Self+ }+}++impl DocumentProvider for DocProvider {+ fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {+ let cursor = Cursor::new(data);+ let mut cfb = CompoundFile::open(cursor)?;++ let mut metadata = DocumentMetadata::default();++ // Try to extract metadata from SummaryInformation stream+ if let Ok(summary_info) = extract_summary_info(&mut cfb) {+ metadata.title = summary_info.title;+ metadata.author = summary_info.author;+ }++ // Extract text content from the document+ let text_content = extract_text_content(&mut cfb)?;++ // Convert the extracted text to document blocks+ let blocks = text_to_blocks(&text_content);++ Ok(Document {+ blocks,+ metadata,+ notes: Vec::new(),+ comments: Vec::new(),+ })+ }++ fn name(&self) -> &'static str {+ "doc"+ }+}++#[derive(Default)]+struct SummaryInfo {+ title: Option<String>,+ author: Option<String>,+}++fn extract_summary_info<R: Read + std::io::Seek>(+ cfb: &mut CompoundFile<R>,+) -> Result<SummaryInfo, Box<dyn Error + Send + Sync>> {+ let mut info = SummaryInfo::default();++ // Try to read the SummaryInformation stream+ if let Ok(mut stream) = cfb.open_stream("\x05SummaryInformation") {+ let mut buf = Vec::new();+ stream.read_to_end(&mut buf)?;++ // Parse the OLE property set stream to extract title and author+ if let Some((title, author)) = parse_summary_info_stream(&buf) {+ info.title = title;+ info.author = author;+ }+ }++ Ok(info)+}++fn parse_summary_info_stream(data: &[u8]) -> Option<(Option<String>, Option<String>)> {+ // MS-OLEPS: Property Set Stream format+ // This is a simplified parser that extracts strings from the property stream++ if data.len() < 48 {+ return None;+ }++ // Byte order mark at offset 0 should be 0xFFFE (little-endian)+ if data.len() >= 2 && (data[0] != 0xFE || data[1] != 0xFF) {+ return None;+ }++ let mut title: Option<String> = None;+ let mut author: Option<String> = None;++ // Extract readable strings from the property stream+ let strings = extract_ascii_strings(data, 3);++ // Filter out common non-title/author strings+ let filtered: Vec<&str> = strings+ .iter()+ .map(|s| s.as_str())+ .filter(|s| {+ !s.contains("Microsoft")+ && !s.contains("Normal")+ && !s.contains("template")+ && !s.starts_with("http")+ && s.len() >= 2+ && s.len() <= 200+ })+ .collect();++ // Title and author are typically the first meaningful strings+ if let Some(t) = filtered.first() {+ title = Some(t.to_string());+ }+ if let Some(a) = filtered.get(1) {+ author = Some(a.to_string());+ }++ Some((title, author))+}++fn extract_text_content<R: Read + std::io::Seek>(+ cfb: &mut CompoundFile<R>,+) -> Result<String, Box<dyn Error + Send + Sync>> {+ // Try to read the WordDocument stream+ if let Ok(mut stream) = cfb.open_stream("WordDocument") {+ let mut doc_data = Vec::new();+ stream.read_to_end(&mut doc_data)?;++ // Extract text from the WordDocument stream+ if let Some(text) = extract_text_from_word_document(&doc_data) {+ if !text.trim().is_empty() {+ return Ok(text);+ }+ }+ }++ // Fallback: scan all streams for text+ extract_text_fallback(cfb)+}++fn extract_text_from_word_document(doc_data: &[u8]) -> Option<String> {+ if doc_data.len() < 32 {+ return None;+ }++ // Check for Word magic number (0xA5EC for Word 97-2003, 0xA5DC for older)+ let magic = u16::from_le_bytes([doc_data[0], doc_data[1]]);+ if magic != 0xA5EC && magic != 0xA5DC {+ return None;+ }++ // Read the FIB (File Information Block) to get text encoding info+ // Bit 9 of flags (offset 0x0A) indicates which table stream to use+ // But for text extraction, we'll use a more robust approach++ // The FIB contains ccpText at offset 0x4C (character count of main text)+ let ccp_text = if doc_data.len() > 0x50 {+ u32::from_le_bytes([+ doc_data[0x4C],+ doc_data[0x4D],+ doc_data[0x4E],+ doc_data[0x4F],+ ]) as usize+ } else {+ 0+ };++ // For complex documents, text may be in pieces. For simple ones, it's contiguous.+ // Either way, we'll scan for text runs since the piece table parsing is complex.++ // .doc files typically store text as CP1252 (single-byte) or UTF-16LE+ // We'll try to detect which one by looking for patterns++ // First, try to find substantial ASCII/CP1252 text runs+ let ascii_text = extract_document_text_cp1252(doc_data, ccp_text);+ if !ascii_text.trim().is_empty() && has_enough_words(&ascii_text, 10) {+ return Some(ascii_text);+ }++ // If ASCII extraction didn't work well, try UTF-16LE+ let utf16_text = extract_document_text_utf16(doc_data, ccp_text);+ if !utf16_text.trim().is_empty() && has_enough_words(&utf16_text, 10) {+ return Some(utf16_text);+ }++ // Return whichever has more content+ if ascii_text.len() > utf16_text.len() {+ Some(ascii_text)+ } else if !utf16_text.is_empty() {+ Some(utf16_text)+ } else {+ None+ }+}++fn extract_document_text_cp1252(data: &[u8], expected_chars: usize) -> String {+ // Find long runs of printable ASCII/CP1252 characters+ // This works well for most .doc files where text is stored as single-byte+ let mut text_runs: Vec<String> = Vec::new();+ let mut current_run = String::new();+ let mut total_chars = 0;+ let max_chars = if expected_chars > 0 && expected_chars < 10_000_000 {+ expected_chars * 2 // Allow some extra for headers/footers+ } else {+ 10_000_000+ };++ for &byte in data.iter() {+ if total_chars >= max_chars {+ break;+ }++ let ch = decode_cp1252(byte);++ if is_text_char(ch) {+ current_run.push(ch);+ } else if byte == 0x0D || byte == 0x0A {+ // Carriage return or line feed - end of paragraph+ if current_run.len() >= 20 && has_word_chars(¤t_run) {+ text_runs.push(current_run.clone());+ total_chars += current_run.len();+ }+ current_run.clear();+ } else if byte == 0x09 {+ // Tab+ current_run.push('\t');+ } else {+ // Non-text byte - might be end of a text run+ if current_run.len() >= 20 && has_word_chars(¤t_run) {+ text_runs.push(current_run.clone());+ total_chars += current_run.len();+ }+ current_run.clear();+ }+ }++ // Don't forget the last run+ if current_run.len() >= 20 && has_word_chars(¤t_run) {+ text_runs.push(current_run);+ }++ // Join text runs with newlines+ text_runs.join("\n")+}++fn extract_document_text_utf16(data: &[u8], expected_chars: usize) -> String {+ let mut text = String::new();+ let max_chars = if expected_chars > 0 && expected_chars < 10_000_000 {+ expected_chars * 2+ } else {+ 10_000_000+ };++ let mut i = 0;+ let mut char_count = 0;+ while i + 1 < data.len() && char_count < max_chars {+ let code = u16::from_le_bytes([data[i], data[i + 1]]);++ if let Some(ch) = char::from_u32(code as u32) {+ if is_text_char(ch) || ch == '\r' || ch == '\n' || ch == '\t' {+ if ch == '\r' {+ text.push('\n');+ } else {+ text.push(ch);+ }+ char_count += 1;+ }+ }+ i += 2;+ }++ // Filter to only keep substantial text portions+ let lines: Vec<&str> = text+ .lines()+ .filter(|line| line.len() >= 10 && has_word_chars(line))+ .collect();++ lines.join("\n")+}++fn has_word_chars(s: &str) -> bool {+ // Check if the string contains actual word characters (letters)+ let letter_count = s.chars().filter(|c| c.is_alphabetic()).count();+ let total_count = s.chars().count();+ // At least 30% should be letters+ letter_count > 0 && (letter_count * 100 / total_count.max(1)) >= 30+}++fn has_enough_words(s: &str, min_words: usize) -> bool {+ s.split_whitespace().count() >= min_words+}++fn is_text_char(ch: char) -> bool {+ // Printable character (not control chars, but allow some special ones)+ (ch >= ' ' && ch != '\x7F') || ch == '\t'+}++fn extract_ascii_strings(data: &[u8], min_length: usize) -> Vec<String> {+ let mut strings = Vec::new();+ let mut current = String::new();++ for &byte in data {+ let ch = decode_cp1252(byte);+ if ch.is_ascii_graphic() || ch == ' ' {+ current.push(ch);+ } else {+ if current.len() >= min_length {+ strings.push(current.clone());+ }+ current.clear();+ }+ }++ if current.len() >= min_length {+ strings.push(current);+ }++ strings+}++fn extract_text_fallback<R: Read + std::io::Seek>(+ cfb: &mut CompoundFile<R>,+) -> Result<String, Box<dyn Error + Send + Sync>> {+ let mut all_text = String::new();++ // List all streams and try to extract text from each+ let entries: Vec<String> = cfb+ .walk()+ .filter(|e| e.is_stream())+ .map(|e| e.path().to_string_lossy().to_string())+ .collect();++ for entry in entries {+ // Skip known non-text streams+ if entry.contains("CompObj")+ || entry.contains("Data")+ || entry.contains("ObjectPool")+ || entry.contains("Pictures")+ {+ continue;+ }++ if let Ok(mut stream) = cfb.open_stream(&entry) {+ let mut buf = Vec::new();+ if stream.read_to_end(&mut buf).is_ok() {+ let stream_text = extract_document_text_cp1252(&buf, 0);+ if !stream_text.trim().is_empty() && has_enough_words(&stream_text, 5) {+ if !all_text.is_empty() {+ all_text.push('\n');+ }+ all_text.push_str(&stream_text);+ }+ }+ }+ }++ Ok(all_text)+}++fn decode_cp1252(b: u8) -> char {+ if b < 0x80 {+ return b as char;+ }+ match b {+ 0x80 => '\u{20AC}', // Euro sign+ 0x82 => '\u{201A}', // Single low-9 quotation mark+ 0x83 => '\u{0192}', // Latin small letter f with hook+ 0x84 => '\u{201E}', // Double low-9 quotation mark+ 0x85 => '\u{2026}', // Horizontal ellipsis+ 0x86 => '\u{2020}', // Dagger+ 0x87 => '\u{2021}', // Double dagger+ 0x88 => '\u{02C6}', // Modifier letter circumflex accent+ 0x89 => '\u{2030}', // Per mille sign+ 0x8A => '\u{0160}', // Latin capital letter S with caron+ 0x8B => '\u{2039}', // Single left-pointing angle quotation mark+ 0x8C => '\u{0152}', // Latin capital ligature OE+ 0x8E => '\u{017D}', // Latin capital letter Z with caron+ 0x91 => '\u{2018}', // Left single quotation mark+ 0x92 => '\u{2019}', // Right single quotation mark+ 0x93 => '\u{201C}', // Left double quotation mark+ 0x94 => '\u{201D}', // Right double quotation mark+ 0x95 => '\u{2022}', // Bullet+ 0x96 => '\u{2013}', // En dash+ 0x97 => '\u{2014}', // Em dash+ 0x98 => '\u{02DC}', // Small tilde+ 0x99 => '\u{2122}', // Trade mark sign+ 0x9A => '\u{0161}', // Latin small letter s with caron+ 0x9B => '\u{203A}', // Single right-pointing angle quotation mark+ 0x9C => '\u{0153}', // Latin small ligature oe+ 0x9E => '\u{017E}', // Latin small letter z with caron+ 0x9F => '\u{0178}', // Latin capital letter Y with diaeresis+ _ => char::from_u32(b as u32).unwrap_or('?'),+ }+}++fn text_to_blocks(text: &str) -> Vec<Block> {+ let mut blocks = Vec::new();++ // Split text into paragraphs and create blocks+ for paragraph in text.split('\n') {+ let trimmed = paragraph.trim();+ if trimmed.is_empty() {+ continue;+ }++ // Clean up the text - remove control characters except tabs+ let cleaned: String = trimmed+ .chars()+ .filter(|c| !c.is_control() || *c == '\t')+ .collect();++ if cleaned.is_empty() {+ continue;+ }++ blocks.push(Block::Paragraph(Paragraph {+ kind: ParagraphKind::Normal,+ inlines: vec![Inline::Text(cleaned)],+ }));+ }++ blocks+}diff --git a/apps/api/native/src/document/providers/factory.rs b/apps/api/native/src/document/providers/factory.rsindex 9fc935bc4..e9b98480f 100644--- a/apps/api/native/src/document/providers/factory.rs+++ b/apps/api/native/src/document/providers/factory.rs@@ -1,3 +1,4 @@+use super::doc::DocProvider;use super::docx::DocxProvider;use super::odt::OdtProvider;use super::rtf::RtfProvider;@@ -8,6 +9,7 @@ use napi_derive::napi;#[napi]#[derive(Debug, Clone, PartialEq, Eq)]pub enum DocumentType {+ Doc,Docx,Rtf,Odt,@@ -15,6 +17,7 @@ pub enum DocumentType {}pub struct ProviderFactory {+ doc_provider: DocProvider,docx_provider: DocxProvider,rtf_provider: RtfProvider,odt_provider: OdtProvider,@@ -24,6 +27,7 @@ pub struct ProviderFactory {impl ProviderFactory {pub fn new() -> Self {Self {+ doc_provider: DocProvider::new(),docx_provider: DocxProvider::new(),rtf_provider: RtfProvider::new(),odt_provider: OdtProvider::new(),@@ -33,6 +37,7 @@ impl ProviderFactory {pub fn get_provider(&self, doc_type: DocumentType) -> &dyn DocumentProvider {match doc_type {+ DocumentType::Doc => &self.doc_provider,DocumentType::Docx => &self.docx_provider,DocumentType::Rtf => &self.rtf_provider,DocumentType::Odt => &self.odt_provider,diff --git a/apps/api/native/src/document/providers/mod.rs b/apps/api/native/src/document/providers/mod.rsindex 67899c741..645d0fdba 100644--- a/apps/api/native/src/document/providers/mod.rs+++ b/apps/api/native/src/document/providers/mod.rs@@ -1,6 +1,7 @@use crate::document::model::Document;use std::error::Error;+pub mod doc;pub mod docx;pub mod factory;pub mod odt;diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.tsindex c1352e027..993635601 100644--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts@@ -12,8 +12,11 @@ function getDocumentTypeFromUrl(url: string): DocumentType {const urlLower = url.toLowerCase();// Check for extensions at the end or in the middle (e.g., file.xlsx/hash)+ // Check .docx before .doc to avoid false matchesif (urlLower.endsWith(".docx") || urlLower.includes(".docx/"))return DocumentType.Docx;+ if (urlLower.endsWith(".doc") || urlLower.includes(".doc/"))+ return DocumentType.Doc;if (urlLower.endsWith(".odt") || urlLower.includes(".odt/"))return DocumentType.Odt;if (urlLower.endsWith(".rtf") || urlLower.includes(".rtf/"))@@ -36,15 +39,20 @@ function getDocumentTypeFromContentType(const ct = contentType.toLowerCase();+ // Check for modern .docx format first (Office Open XML)if (ct.includes("application/vnd.openxmlformats-officedocument.wordprocessingml.document",- ) ||- ct.includes("application/msword")+ )) {return DocumentType.Docx;}+ // Legacy .doc format (OLE2/CFB binary format)+ if (ct.includes("application/msword")) {+ return DocumentType.Doc;+ }+if (ct.includes("application/vnd.oasis.opendocument.text")) {return DocumentType.Odt;}diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.tsindex cb9f6c4dc..9233008f5 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts@@ -47,11 +47,12 @@ async function feResToDocumentPrefetch(): Promise<Meta["documentPrefetch"]> {// Determine file extension from content typelet extension = "tmp";- if (- contentType.includes("wordprocessingml") ||- contentType.includes("msword")- ) {+ if (contentType.includes("wordprocessingml")) {+ // Modern .docx format (Office Open XML)extension = "docx";+ } else if (contentType.includes("msword")) {+ // Legacy .doc format (OLE2/CFB binary)+ extension = "doc";} else if (contentType.includes("spreadsheetml") ||contentType.includes("ms-excel")@@ -107,16 +108,38 @@ export async function specialtyScrapeCheck();}- // Check for octet-stream with document signature (Office files are ZIP archives starting with "PK")- if (- isOctetStream &&- (feRes?.file?.content.startsWith("UEsD") || feRes?.content.startsWith("PK"))- ) {- throw new AddFeatureError(- ["document"],- undefined,- await feResToDocumentPrefetch(logger, feRes, contentType),- );+ // Check for octet-stream with document signature+ // Modern Office files (.docx, .xlsx) are ZIP archives starting with "PK" (base64: "UEsD")+ // Legacy Office files (.doc, .xls) are OLE2/CFB files starting with D0 CF 11 E0 (base64: "0M8R4K")+ if (isOctetStream) {+ const isZipSignature =+ feRes?.file?.content.startsWith("UEsD") ||+ feRes?.content.startsWith("PK");+ const isOleSignature =+ feRes?.file?.content.startsWith("0M8R4K") ||+ feRes?.content.startsWith("\xD0\xCF\x11\xE0");++ if (isZipSignature) {+ throw new AddFeatureError(+ ["document"],+ undefined,+ await feResToDocumentPrefetch(logger, feRes, contentType),+ );+ }+ if (isOleSignature) {+ // OLE2 signature is shared by .doc/.xls/.ppt files+ // Only override to application/msword if URL suggests it's a .doc file+ const url = feRes?.url?.toLowerCase() ?? "";+ const isDocUrl = url.endsWith(".doc") || url.includes(".doc?");+ const effectiveContentType = isDocUrl+ ? "application/msword"+ : contentType;+ throw new AddFeatureError(+ ["document"],+ undefined,+ await feResToDocumentPrefetch(logger, feRes, effectiveContentType),+ );+ }}// Check for PDF