AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(autoindex): name Parquet, Arrow, NumPy, HDF5 and pickle files and say how to index them
Tom Villani92aa4c1f4ca7
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 83b688bb6..90566d8c1 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -2336,6 +2336,7 @@ fn scan_file(// owner nothing about why their mail is not searchable.let reason = sniff::archive_advice(&kind, sn.gzip).or_else(|| sniff::unsupported_document_advice(&kind))+ .or_else(|| sniff::unsupported_data_advice(&kind)).unwrap_or_else(|| format!("binary content ({kind})"));out.junk = Some(("junk".into(), reason));out.sniffed = Some(sn);diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rsindex 010fcddba..bbb522b58 100644--- a/engine/crates/xerj-autoindex/src/sniff.rs+++ b/engine/crates/xerj-autoindex/src/sniff.rs@@ -501,8 +501,38 @@ const MAGIC_TABLE: &[MagicRow] = &[(b"7z\xbc\xaf\x27\x1c", "7z", accept),(b"Rar!\x1a\x07", "rar", accept),(b"\xfd7zXZ\x00", "xz", accept),+ // Data-science containers. Never opened: reading Parquet, Arrow or HDF5+ // takes a large format library, and a pickle runs code when loaded. Naming+ // the kind lets the run say how to make the file indexable+ // (`unsupported_data_advice`) instead of "binary content (unknown)".+ // `ARROW1\0\0`, `\x93NUMPY`, `\x89HDF\r\n\x1a\n` carry a byte text cannot+ // contain; `PAR1` and the pickle opcode are qualified.+ (b"PAR1", "parquet", parquet_page_header),+ (b"ARROW1\x00\x00", "arrow", accept),+ (b"\x93NUMPY", "npy", accept),+ (b"\x89HDF\r\n\x1a\n", "hdf5", accept),+ (b"\x80", "pickle", pickle_protocol),];+/// Parquet: `PAR1` is four printable letters, so the file's first page header+/// has to follow it — a Thrift compact struct whose first two fields are i32s,+/// each introduced by the byte `0x15` (field delta 1, type i32), with a+/// one-byte varint between them. `0x15` is a control byte prose does not+/// contain. Measured on files written by pyarrow (snappy, zstd and+/// uncompressed) and by Chroma.+fn parquet_page_header(prefix: &[u8]) -> bool {+ prefix.len() >= 7 && prefix[4] == 0x15 && prefix[6] == 0x15+}++/// Python pickle, protocols 2-5: the PROTO opcode `0x80` and its version byte;+/// protocols 4 and 5 then open a FRAME (`0x95`). `0x80` cannot start UTF-8+/// text, and a version byte of 2-5 is a control character in any single-byte+/// encoding (a cp1252 note opening with `€` is followed by a printable byte).+/// Protocols 0 and 1 have no header at all and are not recognized.+fn pickle_protocol(prefix: &[u8]) -> bool {+ prefix.len() >= 3 && (2..=5).contains(&prefix[1]) && (prefix[1] < 4 || prefix[2] == 0x95)+}+/// Magic-byte signature taken as sufficient on its own.////// Every caller is a signature containing a byte text cannot contain@@ -1142,6 +1172,10 @@ fn zip_container_kind<R: Read + std::io::Seek>(z: &mut zip::ZipArchive<R>) -> &'return kind;}}+ // numpy.savez: a zip of `.npy` members.+ if !z.is_empty() && z.file_names().all(|n| n.ends_with(".npy")) {+ return "npz";+ }let mut mime = Vec::with_capacity(64);if let Ok(e) = z.by_name("mimetype") {e.take(64).read_to_end(&mut mime).ok();@@ -1174,6 +1208,35 @@ pub fn unsupported_document_advice(binary_kind: &str) -> Option<String> {))}+/// What to tell the user about a data-science container autoindex recognizes+/// but does not read, or `None` for any other binary kind.+///+/// Reading Parquet, Arrow or HDF5 would add a large format library for files+/// whose owners can export them in one line, so the reason names the export.+/// A pickle is never loaded at all: unpickling runs code chosen by whoever+/// wrote the file.+pub fn unsupported_data_advice(binary_kind: &str) -> Option<String> {+ let (what, how) = match binary_kind {+ "parquet" => ("Parquet", "export it to CSV or JSON Lines"),+ "arrow" => ("Arrow IPC / Feather", "export it to CSV or JSON Lines"),+ "npy" | "npz" => ("NumPy array", "save the arrays you need as CSV"),+ "hdf5" => ("HDF5", "export the datasets you need to CSV"),+ "pickle" => {+ return Some(+ "Python pickle: autoindex never loads pickles (loading one runs code \+ chosen by whoever wrote the file) — export the object to CSV or JSON \+ from code you trust to index it"+ .into(),+ )+ }+ _ => return None,+ };+ Some(format!(+ "unsupported data format ({what}): autoindex has no reader for it — \+ {how} to index it now"+ ))+}+/// What to tell the user about a file that is an ARCHIVE, or `None` for any/// other binary kind.///@@ -4332,3 +4395,74 @@ mod zip_container_sniff_tests {}}}++#[cfg(test)]+mod data_container_tests {+ use super::*;+ use std::io::Write;++ fn sniffed(bytes: &[u8], name: &str) -> Sniffed {+ let p = Path::new(name);+ sniff_bytes(bytes, p, p, false).unwrap()+ }++ /// Prefixes copied from real files: pyarrow 17 (Parquet snappy/zstd/none,+ /// Feather v2), a Chroma Parquet store, numpy.save, h5py, pickle.dumps.+ #[test]+ fn data_containers_are_named_from_real_file_prefixes() {+ for (bytes, kind) in [+ (&b"PAR1\x15\x04\x150\x15.rest"[..], "parquet"),+ (&b"PAR1\x15\x00\x15\x5crest"[..], "parquet"),+ (&b"ARROW1\x00\x00\xff\xffrest"[..], "arrow"),+ (&b"\x93NUMPY\x01\x00v\x00rest"[..], "npy"),+ (&b"\x89HDF\r\n\x1a\n\x00\x00rest"[..], "hdf5"),+ (&b"\x80\x02}q\x00X\x01\x00rest"[..], "pickle"),+ (&b"\x80\x05\x95\n\x00\x00\x00\x00rest"[..], "pickle"),+ ] {+ let s = sniffed(bytes, "data.bin");+ assert_eq!(s.family, Family::Binary, "{kind}");+ assert_eq!(s.binary_kind.as_deref(), Some(kind));+ }+ // A protocol-4 opcode without its FRAME, and protocol 0 (no header):+ // not claimed as pickles.+ for bytes in [&b"\x80\x04}rest of it"[..], &b"(dp0\nS'k'\nI1\ns."[..]] {+ assert_ne!(+ sniffed(bytes, "x.bin").binary_kind.as_deref(),+ Some("pickle")+ );+ }+ }++ #[test]+ fn a_zip_of_npy_members_is_an_npz() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("arrays.npz");+ {+ let mut w = zip::ZipWriter::new(std::fs::File::create(&path).unwrap());+ let opts = zip::write::SimpleFileOptions::default();+ for m in ["x.npy", "y.npy"] {+ w.start_file(m, opts).unwrap();+ w.write_all(b"\x93NUMPY\x01\x00").unwrap();+ }+ w.finish().unwrap();+ }+ assert_eq!(sniff(&path).unwrap().binary_kind.as_deref(), Some("npz"));+ }++ #[test]+ fn data_containers_get_an_export_hint() {+ for k in ["parquet", "arrow", "npy", "npz", "hdf5"] {+ let advice = unsupported_data_advice(k).unwrap();+ assert!(+ advice.starts_with("unsupported data format (")+ && advice.contains("to index it now"),+ "{k}: {advice}"+ );+ }+ let pickle = unsupported_data_advice("pickle").unwrap();+ assert!(pickle.contains("never loads pickles"), "{pickle}");+ for k in ["zip", "png", "xlsx", "unknown", ""] {+ assert_eq!(unsupported_data_advice(k), None, "{k}");+ }+ }+}