Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): name Parquet, Arrow, NumPy, HDF5 and pickle files and say how to index them

Tom Villani92aa4c1f4ca7
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 83b688bb6..90566d8c1 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -2336,6 +2336,7 @@ fn scan_file(
// owner nothing about why their mail is not searchable.
let reason = sniff::archive_advice(&kind, sn.gzip)
.or_else(|| sniff::unsupported_document_advice(&kind))
+ .or_else(|| sniff::unsupported_data_advice(&kind))
.unwrap_or_else(|| format!("binary content ({kind})"));
out.junk = Some(("junk".into(), reason));
out.sniffed = Some(sn);
diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rs
index 010fcddba..bbb522b58 100644
--- a/engine/crates/xerj-autoindex/src/sniff.rs
+++ b/engine/crates/xerj-autoindex/src/sniff.rs
@@ -501,8 +501,38 @@ const MAGIC_TABLE: &[MagicRow] = &[
(b"7z\xbc\xaf\x27\x1c", "7z", accept),
(b"Rar!\x1a\x07", "rar", accept),
(b"\xfd7zXZ\x00", "xz", accept),
+ // Data-science containers. Never opened: reading Parquet, Arrow or HDF5
+ // takes a large format library, and a pickle runs code when loaded. Naming
+ // the kind lets the run say how to make the file indexable
+ // (`unsupported_data_advice`) instead of "binary content (unknown)".
+ // `ARROW1\0\0`, `\x93NUMPY`, `\x89HDF\r\n\x1a\n` carry a byte text cannot
+ // contain; `PAR1` and the pickle opcode are qualified.
+ (b"PAR1", "parquet", parquet_page_header),
+ (b"ARROW1\x00\x00", "arrow", accept),
+ (b"\x93NUMPY", "npy", accept),
+ (b"\x89HDF\r\n\x1a\n", "hdf5", accept),
+ (b"\x80", "pickle", pickle_protocol),
];
+/// Parquet: `PAR1` is four printable letters, so the file's first page header
+/// has to follow it — a Thrift compact struct whose first two fields are i32s,
+/// each introduced by the byte `0x15` (field delta 1, type i32), with a
+/// one-byte varint between them. `0x15` is a control byte prose does not
+/// contain. Measured on files written by pyarrow (snappy, zstd and
+/// uncompressed) and by Chroma.
+fn parquet_page_header(prefix: &[u8]) -> bool {
+ prefix.len() >= 7 && prefix[4] == 0x15 && prefix[6] == 0x15
+}
+
+/// Python pickle, protocols 2-5: the PROTO opcode `0x80` and its version byte;
+/// protocols 4 and 5 then open a FRAME (`0x95`). `0x80` cannot start UTF-8
+/// text, and a version byte of 2-5 is a control character in any single-byte
+/// encoding (a cp1252 note opening with `€` is followed by a printable byte).
+/// Protocols 0 and 1 have no header at all and are not recognized.
+fn pickle_protocol(prefix: &[u8]) -> bool {
+ prefix.len() >= 3 && (2..=5).contains(&prefix[1]) && (prefix[1] < 4 || prefix[2] == 0x95)
+}
+
/// Magic-byte signature taken as sufficient on its own.
///
/// Every caller is a signature containing a byte text cannot contain
@@ -1142,6 +1172,10 @@ fn zip_container_kind<R: Read + std::io::Seek>(z: &mut zip::ZipArchive<R>) -> &'
return kind;
}
}
+ // numpy.savez: a zip of `.npy` members.
+ if !z.is_empty() && z.file_names().all(|n| n.ends_with(".npy")) {
+ return "npz";
+ }
let mut mime = Vec::with_capacity(64);
if let Ok(e) = z.by_name("mimetype") {
e.take(64).read_to_end(&mut mime).ok();
@@ -1174,6 +1208,35 @@ pub fn unsupported_document_advice(binary_kind: &str) -> Option<String> {
))
}
+/// What to tell the user about a data-science container autoindex recognizes
+/// but does not read, or `None` for any other binary kind.
+///
+/// Reading Parquet, Arrow or HDF5 would add a large format library for files
+/// whose owners can export them in one line, so the reason names the export.
+/// A pickle is never loaded at all: unpickling runs code chosen by whoever
+/// wrote the file.
+pub fn unsupported_data_advice(binary_kind: &str) -> Option<String> {
+ let (what, how) = match binary_kind {
+ "parquet" => ("Parquet", "export it to CSV or JSON Lines"),
+ "arrow" => ("Arrow IPC / Feather", "export it to CSV or JSON Lines"),
+ "npy" | "npz" => ("NumPy array", "save the arrays you need as CSV"),
+ "hdf5" => ("HDF5", "export the datasets you need to CSV"),
+ "pickle" => {
+ return Some(
+ "Python pickle: autoindex never loads pickles (loading one runs code \
+ chosen by whoever wrote the file) — export the object to CSV or JSON \
+ from code you trust to index it"
+ .into(),
+ )
+ }
+ _ => return None,
+ };
+ Some(format!(
+ "unsupported data format ({what}): autoindex has no reader for it — \
+ {how} to index it now"
+ ))
+}
+
/// What to tell the user about a file that is an ARCHIVE, or `None` for any
/// other binary kind.
///
@@ -4332,3 +4395,74 @@ mod zip_container_sniff_tests {
}
}
}
+
+#[cfg(test)]
+mod data_container_tests {
+ use super::*;
+ use std::io::Write;
+
+ fn sniffed(bytes: &[u8], name: &str) -> Sniffed {
+ let p = Path::new(name);
+ sniff_bytes(bytes, p, p, false).unwrap()
+ }
+
+ /// Prefixes copied from real files: pyarrow 17 (Parquet snappy/zstd/none,
+ /// Feather v2), a Chroma Parquet store, numpy.save, h5py, pickle.dumps.
+ #[test]
+ fn data_containers_are_named_from_real_file_prefixes() {
+ for (bytes, kind) in [
+ (&b"PAR1\x15\x04\x150\x15.rest"[..], "parquet"),
+ (&b"PAR1\x15\x00\x15\x5crest"[..], "parquet"),
+ (&b"ARROW1\x00\x00\xff\xffrest"[..], "arrow"),
+ (&b"\x93NUMPY\x01\x00v\x00rest"[..], "npy"),
+ (&b"\x89HDF\r\n\x1a\n\x00\x00rest"[..], "hdf5"),
+ (&b"\x80\x02}q\x00X\x01\x00rest"[..], "pickle"),
+ (&b"\x80\x05\x95\n\x00\x00\x00\x00rest"[..], "pickle"),
+ ] {
+ let s = sniffed(bytes, "data.bin");
+ assert_eq!(s.family, Family::Binary, "{kind}");
+ assert_eq!(s.binary_kind.as_deref(), Some(kind));
+ }
+ // A protocol-4 opcode without its FRAME, and protocol 0 (no header):
+ // not claimed as pickles.
+ for bytes in [&b"\x80\x04}rest of it"[..], &b"(dp0\nS'k'\nI1\ns."[..]] {
+ assert_ne!(
+ sniffed(bytes, "x.bin").binary_kind.as_deref(),
+ Some("pickle")
+ );
+ }
+ }
+
+ #[test]
+ fn a_zip_of_npy_members_is_an_npz() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("arrays.npz");
+ {
+ let mut w = zip::ZipWriter::new(std::fs::File::create(&path).unwrap());
+ let opts = zip::write::SimpleFileOptions::default();
+ for m in ["x.npy", "y.npy"] {
+ w.start_file(m, opts).unwrap();
+ w.write_all(b"\x93NUMPY\x01\x00").unwrap();
+ }
+ w.finish().unwrap();
+ }
+ assert_eq!(sniff(&path).unwrap().binary_kind.as_deref(), Some("npz"));
+ }
+
+ #[test]
+ fn data_containers_get_an_export_hint() {
+ for k in ["parquet", "arrow", "npy", "npz", "hdf5"] {
+ let advice = unsupported_data_advice(k).unwrap();
+ assert!(
+ advice.starts_with("unsupported data format (")
+ && advice.contains("to index it now"),
+ "{k}: {advice}"
+ );
+ }
+ let pickle = unsupported_data_advice("pickle").unwrap();
+ assert!(pickle.contains("never loads pickles"), "{pickle}");
+ for k in ["zip", "png", "xlsx", "unknown", ""] {
+ assert_eq!(unsupported_data_advice(k), None, "{k}");
+ }
+ }
+}