AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): name Office/OpenDocument/EPUB zips instead of advising unzip
Tom Villanifb1b64fd29a2
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 9c14cf077..695a90ac1 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -2334,6 +2334,7 @@ fn scan_file(// download IS one `.zip`/`.tgz`, and "binary content (zip)" tells its// owner nothing about why their mail is not searchable.let reason = sniff::archive_advice(&kind, sn.gzip)+ .or_else(|| sniff::unsupported_document_advice(&kind)).unwrap_or_else(|| format!("binary content ({kind})"));out.junk = Some(("junk".into(), reason));out.sniffed = Some(sn);diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rsindex 42f404a2c..680b46941 100644--- a/engine/crates/xerj-autoindex/src/sniff.rs+++ b/engine/crates/xerj-autoindex/src/sniff.rs@@ -219,23 +219,21 @@ fn sniff_bytes(return Ok(mk(Family::Sqlite));}if prefix.starts_with(b"PK\x03\x04") {- // zip container: DOCX iff it holds word/document.xml+ // zip container: DOCX iff it holds word/document.xml; another known+ // document container is named for what it is, not as an archive+ let mut kind = "zip";if !gzip {if let Ok(f) = std::fs::File::open(content_path) {if let Ok(mut z) = zip::ZipArchive::new(f) {- let is_docx = (0..z.len()).any(|i| {- z.by_index_raw(i)- .map(|e| e.name() == "word/document.xml")- .unwrap_or(false)- });- if is_docx {+ kind = zip_container_kind(&mut z);+ if kind == "docx" {return Ok(mk(Family::Docx));}}}}let mut s = mk(Family::Binary);- s.binary_kind = Some("zip".into());+ s.binary_kind = Some(kind.into());return Ok(s);}// Compressed image/audio/model payloads routinely pass the NUL and@@ -1072,6 +1070,63 @@ fn looks_like_tar_header(prefix: &[u8]) -> bool {stored == computed}+/// Name a zip container by its members: `"docx"`, another document format+/// autoindex has no extractor for (`"xlsx"`, `"pptx"`, `"odt"`, …), or+/// `"zip"` for anything else.+///+/// Office Open XML, OpenDocument and EPUB are all zips, so the `PK` magic alone+/// cannot tell a spreadsheet from a Takeout download — and calling a workbook+/// an "unextracted zip archive" tells its owner to unzip it, which yields a+/// folder of XML parts that is no more searchable than the file was. OOXML is+/// identified by its main part; OpenDocument and EPUB by the `mimetype` member+/// their container specs require, read through a bounded reader.+fn zip_container_kind<R: Read + std::io::Seek>(z: &mut zip::ZipArchive<R>) -> &'static str {+ const PARTS: &[(&str, &str)] = &[+ ("word/document.xml", "docx"),+ ("xl/workbook.xml", "xlsx"),+ ("xl/workbook.bin", "xlsb"),+ ("ppt/presentation.xml", "pptx"),+ ];+ for (part, kind) in PARTS {+ if z.index_for_name(part).is_some() {+ return kind;+ }+ }+ let mut mime = Vec::with_capacity(64);+ if let Ok(e) = z.by_name("mimetype") {+ e.take(64).read_to_end(&mut mime).ok();+ }+ match mime.trim_ascii() {+ b"application/vnd.oasis.opendocument.text" => "odt",+ b"application/vnd.oasis.opendocument.spreadsheet" => "ods",+ b"application/vnd.oasis.opendocument.presentation" => "odp",+ b"application/epub+zip" => "epub",+ _ => "zip",+ }+}++/// What to tell the user about a document container autoindex recognizes but+/// cannot extract yet, or `None` for any other binary kind.+///+/// Like `archive_advice`, the reason names an action: the export that turns the+/// file into a format autoindex does index today.+pub fn unsupported_document_advice(binary_kind: &str) -> Option<String> {+ let (what, how) = match binary_kind {+ "xlsx" => ("Excel workbook", "export each sheet as CSV"),+ "xlsb" => ("Excel binary workbook", "export each sheet as CSV"),+ "pptx" => ("PowerPoint presentation", "export it as PDF"),+ "odt" => ("OpenDocument text", "export it as DOCX or PDF"),+ "ods" => ("OpenDocument spreadsheet", "export each sheet as CSV"),+ "odp" => ("OpenDocument presentation", "export it as PDF"),+ "epub" => ("EPUB e-book", "convert it to PDF"),+ _ => return None,+ };+ Some(format!(+ "unsupported document format ({what}): autoindex has no extractor for it yet — \+ {how} to index it now"+ ))+}+/// What to tell the user about a file that is an ARCHIVE, or `None` for any/// other binary kind.///@@ -3930,3 +3985,130 @@ mod mail_and_archive_sniff_tests {}}}++#[cfg(test)]+mod zip_container_sniff_tests {+ use super::*;+ use std::io::Write;++ /// Write a zip with the named members to a temp dir and sniff it through+ /// the real read path. ODF and EPUB store `mimetype` uncompressed and first;+ /// fixtures do the same so the bounded read sees what a real file holds.+ fn sniff_zip(members: &[(&str, &str)]) -> Sniffed {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("blob");+ let mut z = zip::ZipWriter::new(std::fs::File::create(&path).unwrap());+ for (name, content) in members {+ let method = if *name == "mimetype" {+ zip::CompressionMethod::Stored+ } else {+ zip::CompressionMethod::Deflated+ };+ let opts = zip::write::SimpleFileOptions::default().compression_method(method);+ z.start_file(*name, opts).unwrap();+ z.write_all(content.as_bytes()).unwrap();+ }+ z.finish().unwrap();+ sniff(&path).unwrap()+ }++ fn kind(members: &[(&str, &str)]) -> (Family, Option<String>) {+ let sn = sniff_zip(members);+ (sn.family, sn.binary_kind)+ }++ fn binary(k: &str) -> (Family, Option<String>) {+ (Family::Binary, Some(k.into()))+ }++ /// An Office or OpenDocument file is a zip, but it is not an archive to+ /// unzip: before this, every container below except DOCX sniffed as+ /// `zip` and was junked with "extract it first (unzip <file>)".+ #[test]+ fn office_containers_are_named_by_their_main_part() {+ let ct = ("[Content_Types].xml", "<Types/>");+ assert_eq!(+ kind(&[ct, ("word/document.xml", "<w:document/>")]),+ (Family::Docx, None)+ );+ assert_eq!(+ kind(&[ct, ("xl/workbook.xml", "<workbook/>")]),+ binary("xlsx")+ );+ assert_eq!(kind(&[ct, ("xl/workbook.bin", "\u{1}")]), binary("xlsb"));+ assert_eq!(+ kind(&[ct, ("ppt/presentation.xml", "<p:presentation/>")]),+ binary("pptx")+ );+ }++ #[test]+ fn opendocument_and_epub_are_named_by_their_mimetype_member() {+ let odf = |mime: &'static str| {+ kind(&[+ ("mimetype", mime),+ ("content.xml", "<office:document-content/>"),+ ("META-INF/manifest.xml", "<manifest:manifest/>"),+ ])+ };+ assert_eq!(+ odf("application/vnd.oasis.opendocument.text"),+ binary("odt")+ );+ assert_eq!(+ odf("application/vnd.oasis.opendocument.spreadsheet"),+ binary("ods")+ );+ assert_eq!(+ odf("application/vnd.oasis.opendocument.presentation"),+ binary("odp")+ );+ assert_eq!(+ kind(&[+ ("mimetype", "application/epub+zip"),+ ("META-INF/container.xml", "<container/>"),+ ]),+ binary("epub")+ );+ }++ /// Everything else stays an archive, so the Takeout advice is unchanged:+ /// a zip of ordinary files, an unknown `mimetype`, and a `mimetype` member+ /// longer than the bounded read (which must not be mistaken for a match).+ #[test]+ fn other_zips_are_still_archives() {+ assert_eq!(+ kind(&[+ ("Takeout/Mail/All mail.mbox", "From x\n"),+ ("notes.txt", "hi")+ ]),+ binary("zip")+ );+ assert_eq!(+ kind(&[("mimetype", "application/vnd.oasis.opendocument.graphics")]),+ binary("zip")+ );+ let long = format!("application/epub+zip{}", " x".repeat(64));+ assert_eq!(kind(&[("mimetype", long.as_str())]), binary("zip"));+ }++ #[test]+ fn unsupported_documents_get_an_export_hint_not_archive_advice() {+ for k in ["xlsx", "xlsb", "pptx", "odt", "ods", "odp", "epub"] {+ assert_eq!(archive_advice(k, false), None, "{k}");+ let advice = unsupported_document_advice(k).unwrap();+ assert!(+ advice.starts_with("unsupported document format (")+ && advice.contains("to index it now"),+ "{k}: {advice}"+ );+ assert!(!advice.contains("unzip"), "{k}: {advice}");+ }+ assert!(unsupported_document_advice("xlsx")+ .unwrap()+ .contains("Excel workbook"));+ for k in ["zip", "tar", "png", "docx", "unknown", ""] {+ assert_eq!(unsupported_document_advice(k), None, "{k}");+ }+ }+}