AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): skip hidden names that are not valid UTF-8
Sebastien Tardifa8b9d792c394
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 58bc5a17a..f51fd0887 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -140,6 +140,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- **`xerj autoindex` now skips hidden names that are not valid UTF-8.**+ The walker already refused `.env`, `.ssh`, and `.git`, but the check+ used `file_name().to_str().starts_with('.')`. On Unix a name whose+ first byte is `.` and that is not valid UTF-8 (`to_str()` is `None`)+ was walked and indexed. The skip now compares the first encoded byte+ to `.`. Typical UTF-8 secrets were already skipped.+- **Autoindex snapshot and replay one-shot failpoints can no longer be stolen by unrelated parallel tests.** ([#385](https://github.com/xerj-org/xerj/issues/385)).The two test-only failpoints are now owned by the thread that arms them, whileretaining their existing one-shot behavior and serialization locks.diff --git a/engine/crates/xerj-autoindex/src/walk.rs b/engine/crates/xerj-autoindex/src/walk.rsindex db0e4eddb..4213cc642 100644--- a/engine/crates/xerj-autoindex/src/walk.rs+++ b/engine/crates/xerj-autoindex/src/walk.rs@@ -9,8 +9,16 @@use crate::ignore_rules::{IgnoreOptions, IgnoreReport, IgnoreStack};use anyhow::{Context, Result};+use std::ffi::OsStr;use std::path::{Path, PathBuf};+/// Hidden names start with `.` even when they are not valid UTF-8.+/// `OsStr::to_str()` is `None` for those, so a UTF-8 `starts_with('.')`+/// would walk them.+fn is_hidden_name(name: &OsStr) -> bool {+ name.as_encoded_bytes().first() == Some(&b'.')+}+#[derive(Debug, Clone)]pub struct FileEntry {pub path: PathBuf,@@ -139,12 +147,7 @@ pub fn walk_reporting(// itself (depth 0) is exempt so a brain over a dot-named folder still// works. `--no-ignore` does NOT turn this off: it is not one of the// ignore rules, it is the reason secrets stay out.- if entry.depth() > 0- && entry- .file_name()- .to_str()- .is_some_and(|n| n.starts_with('.'))- {+ if entry.depth() > 0 && is_hidden_name(entry.file_name()) {stack.record_hidden(is_dir);if is_dir {it.skip_current_dir();@@ -270,6 +273,58 @@ mod hidden_skip_tests {);}+ /// A name whose first byte is '.' but is not valid UTF-8 must still be+ /// treated as hidden. `file_name().to_str()` is `None` for that name, so+ /// the UTF-8 `starts_with('.')` check misses it.+ #[cfg(unix)]+ #[test]+ fn hidden_non_utf8_dot_name_is_detected() {+ use super::is_hidden_name;+ use std::ffi::OsStr;+ use std::os::unix::ffi::OsStrExt;++ let hidden = OsStr::from_bytes(b".\x80");+ assert!(+ hidden.to_str().is_none(),+ "fixture must be non-UTF-8 so to_str() cannot save us"+ );+ assert!(+ is_hidden_name(hidden),+ "first byte is '.', even when to_str() is None"+ );+ assert!(is_hidden_name(OsStr::from_bytes(b".env")));+ assert!(!is_hidden_name(OsStr::from_bytes(b"README.md")));+ assert!(!is_hidden_name(OsStr::from_bytes(b"\x80env")));+ }++ /// Same name on disk. APFS/HFS reject non-UTF-8 names; Linux does not.+ #[cfg(target_os = "linux")]+ #[test]+ fn hidden_non_utf8_dotfile_is_never_indexed() {+ use std::ffi::OsStr;+ use std::os::unix::ffi::OsStrExt;++ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ fs::write(root.join("README.md"), "hello").unwrap();+ fs::write(root.join(OsStr::from_bytes(b".\x80")), "SECRET=pat-abc123").unwrap();++ let files = walk(root, false).unwrap();+ let rels: Vec<String> = files.iter().map(|e| e.rel.clone()).collect();+ assert!(+ rels.contains(&"README.md".to_string()),+ "visible file must still be indexed: {rels:?}"+ );+ assert!(+ !files.iter().any(|e| {+ e.path+ .file_name()+ .is_some_and(|n| n.as_encoded_bytes().first() == Some(&b'.'))+ }),+ "indexed a hidden non-UTF-8 name: {rels:?}"+ );+ }+/// #276: the walk itself must drop gitignored junk, not a later stage —/// an ignored directory is never descended, so it never reaches hashing.#[test]