Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): include DOCX headers and footers in parse output (#4713)
Gaurav Chadhaf79189a0d110
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.tomlindex c884a31a0..3c8ffadc3 100644--- a/apps/api/native/Cargo.toml+++ b/apps/api/native/Cargo.toml@@ -25,6 +25,7 @@ serde_json = "1.0.143"strsim = "0.11"texting_robots = "0.2.2"url = "2.5.7"+zip = { version = "8.6.0", default-features = false, features = ["deflate"] }tokio = "1.48.0"# Transitive dep (pdf-inspector -> unicode-normalization -> tinyvec). Pinned because# tinyvec 1.13.0 fails to compile (https://github.com/Lokathor/tinyvec/issues/225)diff --git a/apps/api/native/src/document.rs b/apps/api/native/src/document.rsindex 9bde03f5f..ee75ba34f 100644--- a/apps/api/native/src/document.rs+++ b/apps/api/native/src/document.rs@@ -1,5 +1,42 @@use napi::bindgen_prelude::*;use napi_derive::napi;+use roxmltree::Document;+use std::collections::{HashMap, HashSet};+use std::io::{Cursor, Read};++const MAX_OOXML_PART_BYTES: u64 = 128 * 1024 * 1024;+const MAX_OOXML_TOTAL_BYTES: u64 = 128 * 1024 * 1024;+const MAX_OOXML_PARTS: usize = 256;+const MAX_HEADER_FOOTER_REFERENCES: usize = 256;+const WORDPROCESSINGML_NAMESPACES: [&str; 2] = [+ "http://schemas.openxmlformats.org/wordprocessingml/2006/main",+ "http://purl.oclc.org/ooxml/wordprocessingml/main",+];++#[derive(Clone, Copy, PartialEq, Eq, Hash)]+enum RepeatingPartKind {+ Header,+ Footer,+}++struct Relationship {+ target: String,+ kind: RepeatingPartKind,+}++struct ZipReadBudget {+ bytes_remaining: u64,+ parts_remaining: usize,+}++impl Default for ZipReadBudget {+ fn default() -> Self {+ Self {+ bytes_remaining: MAX_OOXML_TOTAL_BYTES,+ parts_remaining: MAX_OOXML_PARTS,+ }+ }+}/// Convert a document (doc, docx, odt/ods/odp, rtf, xls/xlsx, ppt/pptx, epub,/// csv, ...) to GitHub-Flavored Markdown using anydoc. The format is detected@@ -14,6 +51,241 @@ pub fn convert_document_to_markdown(data: &[u8], extension_hint: Option<String>).and_then(anydoc::Format::from_extension)});- anydoc::to_markdown_bytes(data, format)- .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))+ let markdown = anydoc::to_markdown_bytes(data, format)+ .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?;++ if format != Some(anydoc::Format::Docx) {+ return Ok(markdown);+ }++ let (headers, footers) = extract_docx_headers_and_footers(data);+ if headers.is_empty() && footers.is_empty() {+ return Ok(markdown);+ }+ let sections = [headers, markdown.trim().to_string(), footers]+ .into_iter()+ .filter(|section| !section.is_empty())+ .collect::<Vec<_>>();++ Ok(format!("{}\n", sections.join("\n\n")))+}++fn extract_docx_headers_and_footers(data: &[u8]) -> (String, String) {+ let Ok(mut archive) = zip::ZipArchive::new(Cursor::new(data)) else {+ return (String::new(), String::new());+ };+ let mut budget = ZipReadBudget::default();++ let main_part = read_zip_text(&mut archive, "_rels/.rels", &mut budget)+ .and_then(|xml| office_document_target(&xml))+ .unwrap_or_else(|| "word/document.xml".to_string());+ let Some(document_xml) = read_zip_text(&mut archive, &main_part, &mut budget) else {+ return (String::new(), String::new());+ };+ let relationship_part = relationship_part_for(&main_part);+ let Some(relationships_xml) = read_zip_text(&mut archive, &relationship_part, &mut budget) else {+ return (String::new(), String::new());+ };+ let relationships = parse_repeating_relationships(&relationships_xml);+ let Ok(document) = Document::parse(&document_xml) else {+ return (String::new(), String::new());+ };++ let mut seen = HashSet::new();+ let mut headers = Vec::new();+ let mut footers = Vec::new();+ let mut reference_count = 0;++ for reference in document+ .descendants()+ .filter(|node| is_wordprocessing_node(*node))+ {+ let reference_kind = match reference.tag_name().name() {+ "headerReference" => RepeatingPartKind::Header,+ "footerReference" => RepeatingPartKind::Footer,+ _ => continue,+ };+ reference_count += 1;+ if reference_count > MAX_HEADER_FOOTER_REFERENCES {+ break;+ }+ let Some(id) = reference+ .attributes()+ .find(|attribute| attribute.name() == "id")+ .map(|attribute| attribute.value())+ else {+ continue;+ };+ let Some(relationship) = relationships.get(id) else {+ continue;+ };+ if relationship.kind != reference_kind {+ continue;+ }+ let Some(part) = resolve_part(&main_part, &relationship.target) else {+ continue;+ };+ if !seen.insert((reference_kind, part.clone())) {+ continue;+ }+ let Some(xml) = read_zip_text(&mut archive, &part, &mut budget) else {+ continue;+ };+ let Some(text) = extract_wordprocessing_text(&xml) else {+ continue;+ };++ match reference_kind {+ RepeatingPartKind::Header => headers.push(text),+ RepeatingPartKind::Footer => footers.push(text),+ }+ }++ (headers.join("\n\n"), footers.join("\n\n"))+}++fn read_zip_text<R: Read + std::io::Seek>(+ archive: &mut zip::ZipArchive<R>,+ name: &str,+ budget: &mut ZipReadBudget,+) -> Option<String> {+ if budget.parts_remaining == 0 || budget.bytes_remaining == 0 {+ return None;+ }+ let mut file = archive.by_name(name.trim_start_matches('/')).ok()?;+ if file.size() > MAX_OOXML_PART_BYTES || file.size() > budget.bytes_remaining {+ return None;+ }+ budget.parts_remaining -= 1;+ let mut bytes = Vec::with_capacity(file.size() as usize);+ let read_limit = MAX_OOXML_PART_BYTES.min(budget.bytes_remaining);+ let read_result = file.by_ref().take(read_limit + 1).read_to_end(&mut bytes);+ budget.bytes_remaining = budget.bytes_remaining.saturating_sub(bytes.len() as u64);+ if read_result.is_err() {+ return None;+ }+ if bytes.len() as u64 > read_limit {+ budget.bytes_remaining = 0;+ return None;+ }+ String::from_utf8(bytes).ok()+}++fn office_document_target(xml: &str) -> Option<String> {+ let document = Document::parse(xml).ok()?;+ let relationship = document.descendants().find(|node| {+ node.is_element()+ && node.tag_name().name() == "Relationship"+ && node+ .attribute("Type")+ .is_some_and(|value| value.ends_with("/officeDocument"))+ })?;+ normalize_part(relationship.attribute("Target")?)+}++fn parse_repeating_relationships(xml: &str) -> HashMap<String, Relationship> {+ let Ok(document) = Document::parse(xml) else {+ return HashMap::new();+ };++ document+ .descendants()+ .filter_map(|node| {+ if !node.is_element()+ || node.tag_name().name() != "Relationship"+ || node+ .attribute("TargetMode")+ .is_some_and(|value| value.eq_ignore_ascii_case("external"))+ {+ return None;+ }+ let kind = match node.attribute("Type")? {+ value if value.ends_with("/header") => RepeatingPartKind::Header,+ value if value.ends_with("/footer") => RepeatingPartKind::Footer,+ _ => return None,+ };+ Some((+ node.attribute("Id")?.to_string(),+ Relationship {+ target: node.attribute("Target")?.to_string(),+ kind,+ },+ ))+ })+ .collect()+}++fn relationship_part_for(part: &str) -> String {+ match part.rsplit_once('/') {+ Some((parent, file_name)) => format!("{parent}/_rels/{file_name}.rels"),+ None => format!("_rels/{part}.rels"),+ }+}++fn resolve_part(base: &str, target: &str) -> Option<String> {+ if target.starts_with('/') {+ return normalize_part(target);+ }+ let parent = base+ .rsplit_once('/')+ .map(|(parent, _)| parent)+ .unwrap_or("");+ normalize_part(&format!("{parent}/{target}"))+}++fn normalize_part(part: &str) -> Option<String> {+ let mut normalized = Vec::new();+ for component in part.trim_start_matches('/').split('/') {+ match component {+ "" | "." => {}+ ".." => normalized.pop().map(|_| ())?,+ value => normalized.push(value),+ }+ }+ (!normalized.is_empty()).then(|| normalized.join("/"))+}++fn extract_wordprocessing_text(xml: &str) -> Option<String> {+ let document = Document::parse(xml).ok()?;+ let paragraphs = document+ .descendants()+ .filter(|node| {+ is_wordprocessing_node(*node)+ && node.tag_name().name() == "p"+ && !node+ .ancestors()+ .skip(1)+ .any(|ancestor| is_wordprocessing_node(ancestor) && ancestor.tag_name().name() == "p")+ })+ .filter_map(|paragraph| {+ let mut text = String::new();+ for node in paragraph+ .descendants()+ .filter(|node| is_wordprocessing_node(*node))+ {+ match node.tag_name().name() {+ "t" => {+ if let Some(value) = node.text() {+ text.push_str(value);+ }+ }+ "tab" => text.push('\t'),+ "br" | "cr" => text.push('\n'),+ _ => {}+ }+ }+ let text = text.trim();+ (!text.is_empty()).then(|| text.to_string())+ })+ .collect::<Vec<_>>();++ (!paragraphs.is_empty()).then(|| paragraphs.join("\n\n"))+}++fn is_wordprocessing_node(node: roxmltree::Node<'_, '_>) -> bool {+ node.is_element()+ && node+ .tag_name()+ .namespace()+ .is_some_and(|namespace| WORDPROCESSINGML_NAMESPACES.contains(&namespace))}diff --git a/apps/api/src/__tests__/snips/v2/parse.test.ts b/apps/api/src/__tests__/snips/v2/parse.test.tsindex e840d7c9a..83d84922b 100644--- a/apps/api/src/__tests__/snips/v2/parse.test.ts+++ b/apps/api/src/__tests__/snips/v2/parse.test.ts@@ -21,7 +21,7 @@ import { config } from "../../../config";import { getRedisConnection } from "../../../services/queue-service";const DOCX_FIXTURE_BASE64 =- "UEsDBBQAAAAIAKtlbVzXeYTq8QAAALgBAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbH2QzU7DMBCE730Ky9cqccoBIZSkB36OwKE8wMreJFb9J69b2rdn00KREOVozXwz62nXB+/EHjPZGDq5qhspMOhobBg7+b55ru6koALBgIsBO3lEkut+0W6OCUkwHKiTUynpXinSE3qgOiYMrAwxeyj8zKNKoLcworppmlulYygYSlXmDNkvhGgfcYCdK+LpwMr5loyOpHg4e+e6TkJKzmoorKt9ML+Kqq+SmsmThyabaMkGqa6VzOL1jh/0lSfK1qB4g1xewLNRfcRslIl65xmu/0/649o4DFbjhZ/TUo4aiXh77+qL4sGG71+06jR8/wlQSwMEFAAAAAgAq2VtXCAbhuqyAAAALgEAAAsAAABfcmVscy8ucmVsc43Puw6CMBQG4J2naM4uBQdjDIXFmLAafICmPZRGeklbL7y9HRzEODie23fyN93TzOSOIWpnGdRlBQStcFJbxeAynDZ7IDFxK/nsLDJYMELXFs0ZZ57yTZy0jyQjNjKYUvIHSqOY0PBYOo82T0YXDE+5DIp6Lq5cId1W1Y6GTwPagpAVS3rJIPSyBjIsHv/h3ThqgUcnbgZt+vHlayPLPChMDB4uSCrf7TKzQHNKuorZvgBQSwMEFAAAAAgAq2VtXCCNfXOwAAAA7AAAABEAAAB3b3JkL2RvY3VtZW50LnhtbDWOMQvCMBCFd3/FkV1THURKGwfFVQcF19icWmjuQi5a/fcmBZeP93jw3TXbjx/gjVF6plYtF5UCpI5dT49WXc6H+UaBJEvODkzYqi+K2ppZM9aOu5dHSpANJPXYqmdKodZauid6KwsOSHm7c/Q25RofeuToQuQORfIBP+hVVa21tz0pMwPI1hu7b4lTCSYjFiRzslEQ9sfdFS5hYOvgjJIaXbbCODFMGv33lPT/0/wAUEsBAhQDFAAAAAgAq2VtXNd5hOrxAAAAuAEAABMAAAAAAAAAAAAAAIABAAAAAFtDb250ZW50X1R5cGVzXS54bWxQSwECFAMUAAAACACrZW1cIBuG6rIAAAAuAQAACwAAAAAAAAAAAAAAgAEiAQAAX3JlbHMvLnJlbHNQSwECFAMUAAAACACrZW1cII19c7AAAADsAAAAEQAAAAAAAAAAAAAAgAH9AQAAd29yZC9kb2N1bWVudC54bWxQSwUGAAAAAAMAAwC5AAAA3AIAAAAA";+ "UEsDBBQAAAAIAGJCNV0B3PB/6gAAAIsCAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbLWSu07DMBSGXyXyWtUnZWBASToAKzDwAkf2cWLhm+zTUt4epykdUKELjPZ/+X5L7rYH75o95WJj6MVGtmI7dK8fiUpTlVB6MTGnO4CiJvJYZEwUqmJi9sj1mEdIqN5wJLhp21tQMTAFXvPcIYbugQzuHDePh3q9UDK5Ipr7xTizeoEpOauQqw77oL9R1ieCrMmjp0w2lVU1CLhImJWfAafcc312tpqaF8z8hL664D1mDTqqna9J+XvNhZ3RGKvonJ/bUo6KSrFh9E6eFY82rK7tmAg15c3fz1iKr/JNjPwv/KX4iw/H7zZ8AlBLAwQUAAAACABiQjVdXzOVUpUAAAAHAQAACwAAAF9yZWxzLy5yZWxzjc87DsIwDAbgq0Q+QJ0yMKCmXVi6Ii4QJW5T0TzkhNftycBAEQOjf//6LHfDw6/iRpyXGBS0jYSh70606lKD7JaURW2ErMCVkg6I2TjyOjcxUaibKbLXpY48Y9LmomfCnZR75E8DtqYYrQIebQvi/Ez0jx2naTF0jObqKZQfJ74aVdY8U1Fwj2zRvuOmsoB9h5sX+xdQSwMEFAAAAAgAYkI1Xf2I2NHjAAAAkQEAABEAAAB3b3JkL2RvY3VtZW50LnhtbI2QQU8DIRCF/wrhB5TVg4fN7jbRptGTjdHEK8LQJQGGzFDX/nthrem1l0fIfO/Ng2H7E4P4BmKPaZR3m05up2HpLZpThFREHSful1HOpeReKTYzRM0bzJDqzCFFXeqVjmpBspnQALNPxxjUfdc9qKh9kpcYuiUGnfMGdpcCfyEEQZfakGefWbaCX2jP7cxNqEmZDpoYxO716VN85IDainfgIh4rOqgGNKVVVxuDKYfVO4O2QG/ggCAZEBU+ZxilBadPoUhBvbejpBf7vJJSNZdDLLe59ivZXOq6Vv2/Ql3/e/oFUEsDBBQAAAAIAGJCNV2/NrkppgAAAIMBAAAcAAAAd29yZC9fcmVscy9kb2N1bWVudC54bWwucmVsc62QywrCMBBFfyXkAzqtCxfSx0ak3Yo/ENJJU2weJKPo3xvUioUuXLi8M8O5hymbm5nYFUMcna14keW8qcsjToLSIOrRR5YubKy4JvI7gCg1GhEz59GmjXLBCEoxDOCFPIsBYZPnWwjfDL5ksq6veOj6FkWPgbPT3eMvBU6pUeLeyYtBSys9oGeiCANSYj5zkSUQh3WJg3P0Xwk1E98Sr/yRgMV76wdQSwMEFAAAAAgAYkI1XRIFb3mKAAAAqQAAABAAAAB3b3JkL2hlYWRlcjEueG1sHc5NCsMgEAXgq4gHiLaLLoIxi3aRXbtooVuJ0yj4x4w0PX41mw8ebxiemn8xsC8g+Zwmfhokn7XaR2eRtSbRuE/c1VpGIWh1EA0NuUBq3SdjNLVF3MSe0RbMKxD5tMUgzlJeRDQ+8f6tdLBT9cMgAbvdr2/2KiEby55AlS1gLKAS/aSLh+WwjdF/UEsDBBQAAAAIAGJCNV0TuxHRigAAAKkAAAAQAAAAd29yZC9mb290ZXIxLnhtbB3OPQrDMAwF4KsYHyB2O3QITjK0dG2HFrqaRPmB2DKSqHv82lk+eDwhnht+YVdfIN4wdvrUWD30LrezkCpN5DZ3ehVJrTE8rhA8N5gglm5GCl5KpMVkpCkRjsC8xSXs5mztxQS/RV2/pQpVpH96YlC3x/Wj3mlHP6kXsKg7ogA5U0+qdJgOy5j+D1BLAQIUAxQAAAAIAGJCNV0B3PB/6gAAAIsCAAATAAAAAAAAAAAAAACAAQAAAABbQ29udGVudF9UeXBlc10ueG1sUEsBAhQDFAAAAAgAYkI1XV8zlVKVAAAABwEAAAsAAAAAAAAAAAAAAIABGwEAAF9yZWxzLy5yZWxzUEsBAhQDFAAAAAgAYkI1Xf2I2NHjAAAAkQEAABEAAAAAAAAAAAAAAIAB2QEAAHdvcmQvZG9jdW1lbnQueG1sUEsBAhQDFAAAAAgAYkI1Xb82uSmmAAAAgwEAABwAAAAAAAAAAAAAAIAB6wIAAHdvcmQvX3JlbHMvZG9jdW1lbnQueG1sLnJlbHNQSwECFAMUAAAACABiQjVdEgVveYoAAACpAAAAEAAAAAAAAAAAAAAAgAHLAwAAd29yZC9oZWFkZXIxLnhtbFBLAQIUAxQAAAAIAGJCNV0TuxHRigAAAKkAAAAQAAAAAAAAAAAAAACAAYMEAAB3b3JkL2Zvb3RlcjEueG1sUEsFBgAAAAAGAAYAfwEAADsFAAAAAA==";const htmlFixture = `<!DOCTYPE html>@@ -572,7 +572,12 @@ describe("/v2/parse", () => {identity,);- expect(result.markdown).toMatch(/Parse DOCX Upload Test/i);+ const markdown = result.markdown!;+ expect(markdown).toContain("Parse DOCX Upload Test Header");+ expect(markdown).toContain("Parse DOCX Upload Test Body");+ expect(markdown).toContain("Parse DOCX Upload Test Footer");+ expect(markdown.indexOf("Header")).toBeLessThan(markdown.indexOf("Body"));+ expect(markdown.indexOf("Body")).toBeLessThan(markdown.indexOf("Footer"));expect(result.metadata.creditsUsed).toBe(1);},scrapeTimeout,