Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): include DOCX headers and footers in parse output (#4713)

Gaurav Chadhaf79189a0d110
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.toml
index c884a31a0..3c8ffadc3 100644
--- a/apps/api/native/Cargo.toml
+++ b/apps/api/native/Cargo.toml
@@ -25,6 +25,7 @@ serde_json = "1.0.143"
strsim = "0.11"
texting_robots = "0.2.2"
url = "2.5.7"
+zip = { version = "8.6.0", default-features = false, features = ["deflate"] }
tokio = "1.48.0"
# Transitive dep (pdf-inspector -> unicode-normalization -> tinyvec). Pinned because
# tinyvec 1.13.0 fails to compile (https://github.com/Lokathor/tinyvec/issues/225)
diff --git a/apps/api/native/src/document.rs b/apps/api/native/src/document.rs
index 9bde03f5f..ee75ba34f 100644
--- a/apps/api/native/src/document.rs
+++ b/apps/api/native/src/document.rs
@@ -1,5 +1,42 @@
use napi::bindgen_prelude::*;
use napi_derive::napi;
+use roxmltree::Document;
+use std::collections::{HashMap, HashSet};
+use std::io::{Cursor, Read};
+
+const MAX_OOXML_PART_BYTES: u64 = 128 * 1024 * 1024;
+const MAX_OOXML_TOTAL_BYTES: u64 = 128 * 1024 * 1024;
+const MAX_OOXML_PARTS: usize = 256;
+const MAX_HEADER_FOOTER_REFERENCES: usize = 256;
+const WORDPROCESSINGML_NAMESPACES: [&str; 2] = [
+ "http://schemas.openxmlformats.org/wordprocessingml/2006/main",
+ "http://purl.oclc.org/ooxml/wordprocessingml/main",
+];
+
+#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+enum RepeatingPartKind {
+ Header,
+ Footer,
+}
+
+struct Relationship {
+ target: String,
+ kind: RepeatingPartKind,
+}
+
+struct ZipReadBudget {
+ bytes_remaining: u64,
+ parts_remaining: usize,
+}
+
+impl Default for ZipReadBudget {
+ fn default() -> Self {
+ Self {
+ bytes_remaining: MAX_OOXML_TOTAL_BYTES,
+ parts_remaining: MAX_OOXML_PARTS,
+ }
+ }
+}
/// Convert a document (doc, docx, odt/ods/odp, rtf, xls/xlsx, ppt/pptx, epub,
/// csv, ...) to GitHub-Flavored Markdown using anydoc. The format is detected
@@ -14,6 +51,241 @@ pub fn convert_document_to_markdown(data: &[u8], extension_hint: Option<String>)
.and_then(anydoc::Format::from_extension)
});
- anydoc::to_markdown_bytes(data, format)
- .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))
+ let markdown = anydoc::to_markdown_bytes(data, format)
+ .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?;
+
+ if format != Some(anydoc::Format::Docx) {
+ return Ok(markdown);
+ }
+
+ let (headers, footers) = extract_docx_headers_and_footers(data);
+ if headers.is_empty() && footers.is_empty() {
+ return Ok(markdown);
+ }
+ let sections = [headers, markdown.trim().to_string(), footers]
+ .into_iter()
+ .filter(|section| !section.is_empty())
+ .collect::<Vec<_>>();
+
+ Ok(format!("{}\n", sections.join("\n\n")))
+}
+
+fn extract_docx_headers_and_footers(data: &[u8]) -> (String, String) {
+ let Ok(mut archive) = zip::ZipArchive::new(Cursor::new(data)) else {
+ return (String::new(), String::new());
+ };
+ let mut budget = ZipReadBudget::default();
+
+ let main_part = read_zip_text(&mut archive, "_rels/.rels", &mut budget)
+ .and_then(|xml| office_document_target(&xml))
+ .unwrap_or_else(|| "word/document.xml".to_string());
+ let Some(document_xml) = read_zip_text(&mut archive, &main_part, &mut budget) else {
+ return (String::new(), String::new());
+ };
+ let relationship_part = relationship_part_for(&main_part);
+ let Some(relationships_xml) = read_zip_text(&mut archive, &relationship_part, &mut budget) else {
+ return (String::new(), String::new());
+ };
+ let relationships = parse_repeating_relationships(&relationships_xml);
+ let Ok(document) = Document::parse(&document_xml) else {
+ return (String::new(), String::new());
+ };
+
+ let mut seen = HashSet::new();
+ let mut headers = Vec::new();
+ let mut footers = Vec::new();
+ let mut reference_count = 0;
+
+ for reference in document
+ .descendants()
+ .filter(|node| is_wordprocessing_node(*node))
+ {
+ let reference_kind = match reference.tag_name().name() {
+ "headerReference" => RepeatingPartKind::Header,
+ "footerReference" => RepeatingPartKind::Footer,
+ _ => continue,
+ };
+ reference_count += 1;
+ if reference_count > MAX_HEADER_FOOTER_REFERENCES {
+ break;
+ }
+ let Some(id) = reference
+ .attributes()
+ .find(|attribute| attribute.name() == "id")
+ .map(|attribute| attribute.value())
+ else {
+ continue;
+ };
+ let Some(relationship) = relationships.get(id) else {
+ continue;
+ };
+ if relationship.kind != reference_kind {
+ continue;
+ }
+ let Some(part) = resolve_part(&main_part, &relationship.target) else {
+ continue;
+ };
+ if !seen.insert((reference_kind, part.clone())) {
+ continue;
+ }
+ let Some(xml) = read_zip_text(&mut archive, &part, &mut budget) else {
+ continue;
+ };
+ let Some(text) = extract_wordprocessing_text(&xml) else {
+ continue;
+ };
+
+ match reference_kind {
+ RepeatingPartKind::Header => headers.push(text),
+ RepeatingPartKind::Footer => footers.push(text),
+ }
+ }
+
+ (headers.join("\n\n"), footers.join("\n\n"))
+}
+
+fn read_zip_text<R: Read + std::io::Seek>(
+ archive: &mut zip::ZipArchive<R>,
+ name: &str,
+ budget: &mut ZipReadBudget,
+) -> Option<String> {
+ if budget.parts_remaining == 0 || budget.bytes_remaining == 0 {
+ return None;
+ }
+ let mut file = archive.by_name(name.trim_start_matches('/')).ok()?;
+ if file.size() > MAX_OOXML_PART_BYTES || file.size() > budget.bytes_remaining {
+ return None;
+ }
+ budget.parts_remaining -= 1;
+ let mut bytes = Vec::with_capacity(file.size() as usize);
+ let read_limit = MAX_OOXML_PART_BYTES.min(budget.bytes_remaining);
+ let read_result = file.by_ref().take(read_limit + 1).read_to_end(&mut bytes);
+ budget.bytes_remaining = budget.bytes_remaining.saturating_sub(bytes.len() as u64);
+ if read_result.is_err() {
+ return None;
+ }
+ if bytes.len() as u64 > read_limit {
+ budget.bytes_remaining = 0;
+ return None;
+ }
+ String::from_utf8(bytes).ok()
+}
+
+fn office_document_target(xml: &str) -> Option<String> {
+ let document = Document::parse(xml).ok()?;
+ let relationship = document.descendants().find(|node| {
+ node.is_element()
+ && node.tag_name().name() == "Relationship"
+ && node
+ .attribute("Type")
+ .is_some_and(|value| value.ends_with("/officeDocument"))
+ })?;
+ normalize_part(relationship.attribute("Target")?)
+}
+
+fn parse_repeating_relationships(xml: &str) -> HashMap<String, Relationship> {
+ let Ok(document) = Document::parse(xml) else {
+ return HashMap::new();
+ };
+
+ document
+ .descendants()
+ .filter_map(|node| {
+ if !node.is_element()
+ || node.tag_name().name() != "Relationship"
+ || node
+ .attribute("TargetMode")
+ .is_some_and(|value| value.eq_ignore_ascii_case("external"))
+ {
+ return None;
+ }
+ let kind = match node.attribute("Type")? {
+ value if value.ends_with("/header") => RepeatingPartKind::Header,
+ value if value.ends_with("/footer") => RepeatingPartKind::Footer,
+ _ => return None,
+ };
+ Some((
+ node.attribute("Id")?.to_string(),
+ Relationship {
+ target: node.attribute("Target")?.to_string(),
+ kind,
+ },
+ ))
+ })
+ .collect()
+}
+
+fn relationship_part_for(part: &str) -> String {
+ match part.rsplit_once('/') {
+ Some((parent, file_name)) => format!("{parent}/_rels/{file_name}.rels"),
+ None => format!("_rels/{part}.rels"),
+ }
+}
+
+fn resolve_part(base: &str, target: &str) -> Option<String> {
+ if target.starts_with('/') {
+ return normalize_part(target);
+ }
+ let parent = base
+ .rsplit_once('/')
+ .map(|(parent, _)| parent)
+ .unwrap_or("");
+ normalize_part(&format!("{parent}/{target}"))
+}
+
+fn normalize_part(part: &str) -> Option<String> {
+ let mut normalized = Vec::new();
+ for component in part.trim_start_matches('/').split('/') {
+ match component {
+ "" | "." => {}
+ ".." => normalized.pop().map(|_| ())?,
+ value => normalized.push(value),
+ }
+ }
+ (!normalized.is_empty()).then(|| normalized.join("/"))
+}
+
+fn extract_wordprocessing_text(xml: &str) -> Option<String> {
+ let document = Document::parse(xml).ok()?;
+ let paragraphs = document
+ .descendants()
+ .filter(|node| {
+ is_wordprocessing_node(*node)
+ && node.tag_name().name() == "p"
+ && !node
+ .ancestors()
+ .skip(1)
+ .any(|ancestor| is_wordprocessing_node(ancestor) && ancestor.tag_name().name() == "p")
+ })
+ .filter_map(|paragraph| {
+ let mut text = String::new();
+ for node in paragraph
+ .descendants()
+ .filter(|node| is_wordprocessing_node(*node))
+ {
+ match node.tag_name().name() {
+ "t" => {
+ if let Some(value) = node.text() {
+ text.push_str(value);
+ }
+ }
+ "tab" => text.push('\t'),
+ "br" | "cr" => text.push('\n'),
+ _ => {}
+ }
+ }
+ let text = text.trim();
+ (!text.is_empty()).then(|| text.to_string())
+ })
+ .collect::<Vec<_>>();
+
+ (!paragraphs.is_empty()).then(|| paragraphs.join("\n\n"))
+}
+
+fn is_wordprocessing_node(node: roxmltree::Node<'_, '_>) -> bool {
+ node.is_element()
+ && node
+ .tag_name()
+ .namespace()
+ .is_some_and(|namespace| WORDPROCESSINGML_NAMESPACES.contains(&namespace))
}
diff --git a/apps/api/src/__tests__/snips/v2/parse.test.ts b/apps/api/src/__tests__/snips/v2/parse.test.ts
index e840d7c9a..83d84922b 100644
--- a/apps/api/src/__tests__/snips/v2/parse.test.ts
+++ b/apps/api/src/__tests__/snips/v2/parse.test.ts
@@ -21,7 +21,7 @@ import { config } from "../../../config";
import { getRedisConnection } from "../../../services/queue-service";
const DOCX_FIXTURE_BASE64 =
- "UEsDBBQAAAAIAKtlbVzXeYTq8QAAALgBAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbH2QzU7DMBCE730Ky9cqccoBIZSkB36OwKE8wMreJFb9J69b2rdn00KREOVozXwz62nXB+/EHjPZGDq5qhspMOhobBg7+b55ru6koALBgIsBO3lEkut+0W6OCUkwHKiTUynpXinSE3qgOiYMrAwxeyj8zKNKoLcworppmlulYygYSlXmDNkvhGgfcYCdK+LpwMr5loyOpHg4e+e6TkJKzmoorKt9ML+Kqq+SmsmThyabaMkGqa6VzOL1jh/0lSfK1qB4g1xewLNRfcRslIl65xmu/0/649o4DFbjhZ/TUo4aiXh77+qL4sGG71+06jR8/wlQSwMEFAAAAAgAq2VtXCAbhuqyAAAALgEAAAsAAABfcmVscy8ucmVsc43Puw6CMBQG4J2naM4uBQdjDIXFmLAafICmPZRGeklbL7y9HRzEODie23fyN93TzOSOIWpnGdRlBQStcFJbxeAynDZ7IDFxK/nsLDJYMELXFs0ZZ57yTZy0jyQjNjKYUvIHSqOY0PBYOo82T0YXDE+5DIp6Lq5cId1W1Y6GTwPagpAVS3rJIPSyBjIsHv/h3ThqgUcnbgZt+vHlayPLPChMDB4uSCrf7TKzQHNKuorZvgBQSwMEFAAAAAgAq2VtXCCNfXOwAAAA7AAAABEAAAB3b3JkL2RvY3VtZW50LnhtbDWOMQvCMBCFd3/FkV1THURKGwfFVQcF19icWmjuQi5a/fcmBZeP93jw3TXbjx/gjVF6plYtF5UCpI5dT49WXc6H+UaBJEvODkzYqi+K2ppZM9aOu5dHSpANJPXYqmdKodZauid6KwsOSHm7c/Q25RofeuToQuQORfIBP+hVVa21tz0pMwPI1hu7b4lTCSYjFiRzslEQ9sfdFS5hYOvgjJIaXbbCODFMGv33lPT/0/wAUEsBAhQDFAAAAAgAq2VtXNd5hOrxAAAAuAEAABMAAAAAAAAAAAAAAIABAAAAAFtDb250ZW50X1R5cGVzXS54bWxQSwECFAMUAAAACACrZW1cIBuG6rIAAAAuAQAACwAAAAAAAAAAAAAAgAEiAQAAX3JlbHMvLnJlbHNQSwECFAMUAAAACACrZW1cII19c7AAAADsAAAAEQAAAAAAAAAAAAAAgAH9AQAAd29yZC9kb2N1bWVudC54bWxQSwUGAAAAAAMAAwC5AAAA3AIAAAAA";
+ "UEsDBBQAAAAIAGJCNV0B3PB/6gAAAIsCAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbLWSu07DMBSGXyXyWtUnZWBASToAKzDwAkf2cWLhm+zTUt4epykdUKELjPZ/+X5L7rYH75o95WJj6MVGtmI7dK8fiUpTlVB6MTGnO4CiJvJYZEwUqmJi9sj1mEdIqN5wJLhp21tQMTAFXvPcIYbugQzuHDePh3q9UDK5Ipr7xTizeoEpOauQqw77oL9R1ieCrMmjp0w2lVU1CLhImJWfAafcc312tpqaF8z8hL664D1mDTqqna9J+XvNhZ3RGKvonJ/bUo6KSrFh9E6eFY82rK7tmAg15c3fz1iKr/JNjPwv/KX4iw/H7zZ8AlBLAwQUAAAACABiQjVdXzOVUpUAAAAHAQAACwAAAF9yZWxzLy5yZWxzjc87DsIwDAbgq0Q+QJ0yMKCmXVi6Ii4QJW5T0TzkhNftycBAEQOjf//6LHfDw6/iRpyXGBS0jYSh70606lKD7JaURW2ErMCVkg6I2TjyOjcxUaibKbLXpY48Y9LmomfCnZR75E8DtqYYrQIebQvi/Ez0jx2naTF0jObqKZQfJ74aVdY8U1Fwj2zRvuOmsoB9h5sX+xdQSwMEFAAAAAgAYkI1Xf2I2NHjAAAAkQEAABEAAAB3b3JkL2RvY3VtZW50LnhtbI2QQU8DIRCF/wrhB5TVg4fN7jbRptGTjdHEK8LQJQGGzFDX/nthrem1l0fIfO/Ng2H7E4P4BmKPaZR3m05up2HpLZpThFREHSful1HOpeReKTYzRM0bzJDqzCFFXeqVjmpBspnQALNPxxjUfdc9qKh9kpcYuiUGnfMGdpcCfyEEQZfakGefWbaCX2jP7cxNqEmZDpoYxO716VN85IDainfgIh4rOqgGNKVVVxuDKYfVO4O2QG/ggCAZEBU+ZxilBadPoUhBvbejpBf7vJJSNZdDLLe59ivZXOq6Vv2/Ql3/e/oFUEsDBBQAAAAIAGJCNV2/NrkppgAAAIMBAAAcAAAAd29yZC9fcmVscy9kb2N1bWVudC54bWwucmVsc62QywrCMBBFfyXkAzqtCxfSx0ak3Yo/ENJJU2weJKPo3xvUioUuXLi8M8O5hymbm5nYFUMcna14keW8qcsjToLSIOrRR5YubKy4JvI7gCg1GhEz59GmjXLBCEoxDOCFPIsBYZPnWwjfDL5ksq6veOj6FkWPgbPT3eMvBU6pUeLeyYtBSys9oGeiCANSYj5zkSUQh3WJg3P0Xwk1E98Sr/yRgMV76wdQSwMEFAAAAAgAYkI1XRIFb3mKAAAAqQAAABAAAAB3b3JkL2hlYWRlcjEueG1sHc5NCsMgEAXgq4gHiLaLLoIxi3aRXbtooVuJ0yj4x4w0PX41mw8ebxiemn8xsC8g+Zwmfhokn7XaR2eRtSbRuE/c1VpGIWh1EA0NuUBq3SdjNLVF3MSe0RbMKxD5tMUgzlJeRDQ+8f6tdLBT9cMgAbvdr2/2KiEby55AlS1gLKAS/aSLh+WwjdF/UEsDBBQAAAAIAGJCNV0TuxHRigAAAKkAAAAQAAAAd29yZC9mb290ZXIxLnhtbB3OPQrDMAwF4KsYHyB2O3QITjK0dG2HFrqaRPmB2DKSqHv82lk+eDwhnht+YVdfIN4wdvrUWD30LrezkCpN5DZ3ehVJrTE8rhA8N5gglm5GCl5KpMVkpCkRjsC8xSXs5mztxQS/RV2/pQpVpH96YlC3x/Wj3mlHP6kXsKg7ogA5U0+qdJgOy5j+D1BLAQIUAxQAAAAIAGJCNV0B3PB/6gAAAIsCAAATAAAAAAAAAAAAAACAAQAAAABbQ29udGVudF9UeXBlc10ueG1sUEsBAhQDFAAAAAgAYkI1XV8zlVKVAAAABwEAAAsAAAAAAAAAAAAAAIABGwEAAF9yZWxzLy5yZWxzUEsBAhQDFAAAAAgAYkI1Xf2I2NHjAAAAkQEAABEAAAAAAAAAAAAAAIAB2QEAAHdvcmQvZG9jdW1lbnQueG1sUEsBAhQDFAAAAAgAYkI1Xb82uSmmAAAAgwEAABwAAAAAAAAAAAAAAIAB6wIAAHdvcmQvX3JlbHMvZG9jdW1lbnQueG1sLnJlbHNQSwECFAMUAAAACABiQjVdEgVveYoAAACpAAAAEAAAAAAAAAAAAAAAgAHLAwAAd29yZC9oZWFkZXIxLnhtbFBLAQIUAxQAAAAIAGJCNV0TuxHRigAAAKkAAAAQAAAAAAAAAAAAAACAAYMEAAB3b3JkL2Zvb3RlcjEueG1sUEsFBgAAAAAGAAYAfwEAADsFAAAAAA==";
const htmlFixture = `
<!DOCTYPE html>
@@ -572,7 +572,12 @@ describe("/v2/parse", () => {
identity,
);
- expect(result.markdown).toMatch(/Parse DOCX Upload Test/i);
+ const markdown = result.markdown!;
+ expect(markdown).toContain("Parse DOCX Upload Test Header");
+ expect(markdown).toContain("Parse DOCX Upload Test Body");
+ expect(markdown).toContain("Parse DOCX Upload Test Footer");
+ expect(markdown.indexOf("Header")).toBeLessThan(markdown.indexOf("Body"));
+ expect(markdown.indexOf("Body")).toBeLessThan(markdown.indexOf("Footer"));
expect(result.metadata.creditsUsed).toBe(1);
},
scrapeTimeout,