Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

Prepare interrupted-clone contribution for combined recovery review

xerj-org02325b45f147
diff --cc engine/crates/xerj-autoindex/src/xc.rs
index 1490d9910,c5560d79b..8371472b6
--- a/engine/crates/xerj-autoindex/src/xc.rs
+++ b/engine/crates/xerj-autoindex/src/xc.rs
@@@ -6359,227 -6373,4 +6425,250 @@@ precedence = ["a", "b"
"an escaping paths entry is refused before any directory is created"
);
}
+
+ /// #2486: a `paths` entry may name a file. The pandas time-series pack
+ /// needs `pandas/core/resample.py` without the rest of `pandas/core`, and
+ /// naming the parent directory crowds the documentation out of the top
+ /// hits. A file entry checks out that file and nothing beside it: not a
+ /// sibling whose name starts the same way, and not a file of the same
+ /// name deeper in the tree (the pattern is anchored at the root).
+ #[test]
+ fn corpus_add_from_paths_harvests_a_named_file() {
+ let home = tempfile::tempdir().unwrap();
+ let _hold = code_home(home.path());
+ let origin = home.path().join("origin");
+ init_committed_repo(
+ &origin,
+ &[
+ ("LICENSE", "bsd\n"),
+ ("pkg/core/resample.py", "def resample(): pass\n"),
+ ("pkg/core/resample.pyx", "cdef int x\n"),
+ ("pkg/core/frame.py", "class DataFrame: pass\n"),
+ ("other/pkg/core/resample.py", "def other(): pass\n"),
+ ("doc/timeseries.rst", "Time series\n"),
+ ],
+ );
+ let sha = head_sha(&origin);
+ let url = origin.display().to_string();
+ let manifest = home.path().join("pin.json");
+ let checkout = home.path().join("corpora/sparse-demo/vulndb");
+
+ write_pin(
+ &manifest,
+ &url,
+ &sha,
+ Some(&["pkg/core/resample.py", "doc"]),
+ );
+ assert_eq!(
+ run_corpus_add(&["--from".into(), manifest.display().to_string()]),
+ 0,
+ "a paths entry naming a file must be harvested"
+ );
+ assert_eq!(
+ worktree_files(&checkout),
+ vec![
+ "doc/timeseries.rst".to_string(),
+ "pkg/core/resample.py".to_string(),
+ ]
+ );
+ let recorded =
+ manifest::read_corpus_manifest(&home.path().join("corpora/sparse-demo/corpus.json"))
+ .unwrap();
+ assert_eq!(
+ recorded.repos[0].paths,
+ vec!["pkg/core/resample.py".to_string(), "doc".to_string()]
+ );
+ assert_eq!(recorded.repos[0].files, Some(2));
+
+ // Re-adding over the existing checkout narrows and widens it too.
+ write_pin(&manifest, &url, &sha, Some(&["doc"]));
+ assert_eq!(
+ run_corpus_add(&["--from".into(), manifest.display().to_string()]),
+ 0
+ );
+ assert_eq!(
+ worktree_files(&checkout),
+ vec!["doc/timeseries.rst".to_string()]
+ );
+ write_pin(&manifest, &url, &sha, Some(&["pkg/core/frame.py"]));
+ assert_eq!(
+ run_corpus_add(&["--from".into(), manifest.display().to_string()]),
+ 0
+ );
+ assert_eq!(
+ worktree_files(&checkout),
+ vec!["pkg/core/frame.py".to_string()]
+ );
+ }
+
+ /// Each tree entry kind gets its own answer, read from the mode, so a
+ /// submodule is not reported as a missing path.
+ #[test]
+ fn confirm_paths_exist_classifies_each_entry_kind() {
+ let home = tempfile::tempdir().unwrap();
+ let repo = home.path().join("repo");
+ init_committed_repo(&repo, &[("doc/a.rst", "a\n"), ("pkg/m.py", "x = 1\n")]);
+ let first = head_sha(&repo);
+ let gitlink = format!("160000,{first},vendor/sub");
+ let (rc, out) = git(
+ Some(&repo),
+ &["update-index", "--add", "--cacheinfo", &gitlink],
+ )
+ .unwrap();
+ assert_eq!(rc, 0, "git update-index: {out}");
+ let (rc, out) = git(
+ Some(&repo),
+ &[
+ "-c",
+ "-c",
+ "user.name=xerj-test",
+ "commit",
+ "--quiet",
+ "-m",
+ "gitlink",
+ ],
+ )
+ .unwrap();
+ assert_eq!(rc, 0, "git commit: {out}");
+ let sha = head_sha(&repo);
+ let paths = |list: &[&str]| list.iter().map(|p| p.to_string()).collect::<Vec<_>>();
+
+ assert_eq!(
+ confirm_paths_exist(&repo, &sha, &paths(&["pkg/m.py", "doc"])).unwrap(),
+ vec![PathKind::File, PathKind::Dir]
+ );
+ let err = confirm_paths_exist(&repo, &sha, &paths(&["vendor/sub"]))
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("is a submodule"), "{err}");
+ let err = confirm_paths_exist(&repo, &sha, &paths(&["doc/nope.rst"]))
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("has no such path"), "{err}");
+ }
+
+ /// A symlink entry is refused end to end. Naming one would hand
+ /// `corpus index` a link the manifest author chose, possibly pointing out
+ /// of the checkout, so it is refused and no corpus is recorded.
+ #[cfg(unix)]
+ #[test]
+ fn corpus_add_from_paths_refuses_a_symlink_entry() {
+ let home = tempfile::tempdir().unwrap();
+ let _hold = code_home(home.path());
+ let origin = home.path().join("origin");
+ init_committed_repo(&origin, &[("LICENSE", "bsd\n")]);
+ std::os::unix::fs::symlink("/etc/hostname", origin.join("link")).unwrap();
+ let (rc, out) = git(Some(&origin), &["add", "--", "link"]).unwrap();
+ assert_eq!(rc, 0, "git add link: {out}");
+ let (rc, out) = git(
+ Some(&origin),
+ &[
+ "-c",
+ "-c",
+ "user.name=xerj-test",
+ "commit",
+ "--quiet",
+ "-m",
+ "link",
+ ],
+ )
+ .unwrap();
+ assert_eq!(rc, 0, "git commit: {out}");
+ let sha = head_sha(&origin);
+ let manifest = home.path().join("pin.json");
+ write_pin(
+ &manifest,
+ &origin.display().to_string(),
+ &sha,
+ Some(&["link"]),
+ );
+ assert_eq!(
+ run_corpus_add(&["--from".into(), manifest.display().to_string()]),
+ 1
+ );
+ assert!(
+ !home.path().join("corpora/sparse-demo/corpus.json").exists(),
+ "a refused symlink entry must not record a corpus"
+ );
+ assert!(!home
+ .path()
+ .join("corpora/sparse-demo/vulndb/LICENSE")
+ .exists());
+ }
+ #[test]
+ fn corpus_bad_sparse_path_preserves_existing_checkout_and_manifest() {
+ let home = tempfile::tempdir().unwrap();
+ let _hold = code_home(home.path());
+ let origin = home.path().join("origin");
+ init_committed_repo(
+ &origin,
+ &[
+ ("doc/keep.txt", "retained content\n"),
+ ("pkg/code.rs", "fn f() {}\n"),
+ ],
+ );
+ let sha = head_sha(&origin);
+ let pin = home.path().join("pin.json");
+ let url = origin.display().to_string();
+ write_pin(&pin, &url, &sha, Some(&["doc"]));
+ let args = vec!["--from".to_string(), pin.display().to_string()];
+ assert_eq!(run_corpus_add(&args), 0);
+ let checkout = home.path().join("corpora/sparse-demo/vulndb");
+ let manifest = home.path().join("corpora/sparse-demo/corpus.json");
+ let before = std::fs::read(&manifest).unwrap();
+ let index_before = std::fs::read(checkout.join(".git/index")).unwrap();
+ let sparse_before = std::fs::read(checkout.join(".git/info/sparse-checkout")).unwrap();
+ std::fs::write(checkout.join("local-note.txt"), "uncommitted local note").unwrap();
+ write_pin(&pin, &url, &sha, Some(&["pkg/code.rs", "missing.txt"]));
+ assert_eq!(run_corpus_add(&args), 1);
+ assert_eq!(std::fs::read(&manifest).unwrap(), before);
+ assert_eq!(
+ std::fs::read(checkout.join(".git/index")).unwrap(),
+ index_before
+ );
+ assert_eq!(
+ std::fs::read(checkout.join(".git/info/sparse-checkout")).unwrap(),
+ sparse_before
+ );
+ assert_eq!(
+ std::fs::read_to_string(checkout.join("doc/keep.txt")).unwrap(),
+ "retained content\n"
+ );
+ assert_eq!(
+ std::fs::read_to_string(checkout.join("local-note.txt")).unwrap(),
+ "uncommitted local note"
+ );
+ assert!(!checkout.join("pkg/code.rs").exists());
+ assert_eq!(head_sha(&checkout), sha);
+ // Correcting the path performs the requested narrower harvest.
+ write_pin(&pin, &url, &sha, Some(&["pkg/code.rs"]));
+ assert_eq!(run_corpus_add(&args), 0);
+ assert!(checkout.join("pkg/code.rs").is_file());
+ assert!(!checkout.join("doc/keep.txt").exists());
+ }
++ #[test]
++ fn corpus_add_does_not_delete_an_unborn_repo_with_user_files() {
++ let home = tempfile::tempdir().unwrap();
++ let _hold = code_home(home.path());
++ let origin = home.path().join("widget");
++ init_committed_repo(&origin, &[("keep.txt", "remote content\n")]);
++ let target = home.path().join("corpora/preserve/widget");
++ std::fs::create_dir_all(&target).unwrap();
++ assert_eq!(git(Some(&target), &["init", "--quiet"]).unwrap().0, 0);
++ std::fs::write(target.join("local-note.txt"), "irreplaceable local content").unwrap();
++ let code = run_corpus_add(&["preserve".into(), origin.display().to_string()]);
++ assert_eq!(std::fs::read_to_string(target.join("local-note.txt")).ok().as_deref(),
++ Some("irreplaceable local content"), "an unborn repo is not proof of disposable state");
++ assert_eq!(code, 1, "an ambiguous existing checkout must be refused");
++ assert!(!home.path().join("corpora/preserve/corpus.json").exists());
++ // Keeping the old directory separately makes a normal retry safe.
++ let backup = home.path().join("saved-incomplete-widget");
++ std::fs::rename(&target, &backup).unwrap();
++ assert_eq!(run_corpus_add(&["preserve".into(), origin.display().to_string()]), 0);
++ assert_eq!(std::fs::read_to_string(backup.join("local-note.txt")).unwrap(), "irreplaceable local content");
++ assert_eq!(std::fs::read_to_string(target.join("keep.txt")).unwrap(), "remote content\n");
++ }
++
}