Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(judge): score CJK evidence inside continuous text (#2635)

Jarvise7777dcdc10a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6e673a7bd..ab60ae2fb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Opt-in lexical `judge` now matches CJK characters and adjacent pairs inside
+ uninterrupted Chinese/Japanese/Korean text instead of treating the whole
+ run as one word (refs #2580). Single-character evidence is retained, pairs
+ add adjacency evidence, and non-CJK word tokenization stays unchanged.
+ This changes judged CJK scores/threshold outcomes; it does not change the
+ first-stage analyzer or recover candidates absent from its result page.
+
- `xerj corpus --help`, `xerj autoindex --help` (`--watch`) and the XERJ.code README now say that corpora and folders do not follow their git remotes, and give the refresh recipe: `git pull` in each checkout, re-run `xerj corpus add` so `corpus.json` records the new commit, then `xerj corpus index`. (#2416)
- `xerj_batch_search` (and `xerj batch-search`): a batch whose combined `_msearch` response passes the 32 MiB cap no longer fails every item. The request is retried in halves, and only an item whose own response is over the cap fails, with `response_too_large`. (#2694)
- `xerj autoindex` and `xerj autoindex map`: when the node is unreachable at a `localhost` URL, the error now also suggests the same URL with `127.0.0.1`, since the node binds the IPv4 loopback by default and `localhost` can resolve to `::1` first (seen on Windows). (#2688)
diff --git a/docs/RERANK.md b/docs/RERANK.md
index 184fb92c6..11e554a26 100644
--- a/docs/RERANK.md
+++ b/docs/RERANK.md
@@ -51,6 +51,26 @@ client, the request parser and the failure policy are the leaf crate
`engine/crates/xerj-rerank`. The end-to-end tests are
`engine/crates/xerj-api/tests/rerank_stage_http.rs`.
+## Local lexical `judge` and CJK text
+
+The separate opt-in `judge` block can run a deterministic lexical scorer on
+the returned candidate page. It keeps `_score` and adds `_p_relevant`; the
+response identifies this scorer as `judged.scorer: "lexical"`. The lexical
+arm needs no hosted provider or model. A node configured with a local judge
+model uses that model's own tokenizer instead.
+
+The lexical arm splits CJK runs into single characters and adjacent pairs.
+For example, a query containing `雷同` can match inside `内容存在雷同需要核查`;
+the pair contributes evidence beyond two isolated or reversed characters.
+Punctuation, non-CJK runs and the title/body boundary break pairs. Non-CJK
+alphanumeric runs retain their existing lowercase word tokens. This is
+character n-gram overlap, not dictionary segmentation or semantic relevance.
+
+CJK scores and `judge.min_p` filtering can change with this tokenizer; no
+cross-language calibration is claimed. The judge only reorders/filters the
+candidates it receives. It cannot recover a document missed by the first
+stage, and this change does not change index analyzers or require reindexing.
+
## Set it up
A node needs a provider key. There are two places to put it, and **a value in
diff --git a/engine/crates/xerj-api/tests/judge_stage_http.rs b/engine/crates/xerj-api/tests/judge_stage_http.rs
index 5d4a5b717..5a6ce68e0 100644
--- a/engine/crates/xerj-api/tests/judge_stage_http.rs
+++ b/engine/crates/xerj-api/tests/judge_stage_http.rs
@@ -293,6 +293,91 @@ async fn no_judge_block_leaves_the_response_untouched() {
// Applied: _p_relevant per hit, reorder, the judged block
// ─────────────────────────────────────────────────────────────────────────────
+/// #2580: use match_all to hold the candidate set fixed. This isolates the
+/// judge's tokenization from the separate first-stage CJK recall question.
+#[tokio::test]
+async fn cjk_judge_finds_embedded_terms_and_thresholds_without_changing_engine_scores() {
+ let node = node().await;
+ let (st, body) = node
+ .call(
+ "PUT",
+ "/cjk",
+ json!({"mappings":{"properties":{
+ "body":{"type":"text"}
+ }}}),
+ )
+ .await;
+ assert!(st.is_success(), "{st} {body}");
+ for (id, text) in [
+ ("exact", "这里发现雷同内容雷同内容"),
+ ("partial", "部分雷声同时出现"),
+ ("unrelated", "今天阳光明媚风景宜人"),
+ ] {
+ let (st, body) = node
+ .call(
+ "PUT",
+ &format!("/cjk/_doc/{id}?refresh=true"),
+ json!({"body":text}),
+ )
+ .await;
+ assert!(st.is_success(), "{st} {body}");
+ }
+ for flushed in [false, true] {
+ if flushed {
+ let (st, body) = node.call("POST", "/cjk/_flush", json!({})).await;
+ assert!(st.is_success(), "{st} {body}");
+ }
+ let (_, base) = node
+ .search("/cjk/_search", json!({"query":{"match_all":{}},"size":3}))
+ .await;
+ assert!(base.get("judged").is_none(), "{base}");
+ let (st, judged) = node
+ .search(
+ "/cjk/_search",
+ json!({"query":{"match_all":{}},"size":3,"judge":{"query":"雷同"}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{judged}");
+ assert_eq!(ids(&judged)[0], "exact", "{judged}");
+ let p = |id| {
+ judged["hits"]["hits"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .find(|h| h["_id"] == id)
+ .unwrap()["_p_relevant"]
+ .as_f64()
+ .unwrap()
+ };
+ assert!(p("exact") > p("partial"), "{judged}");
+ assert!(p("partial") > 0.0, "{judged}");
+ assert_eq!(p("unrelated"), 0.0, "{judged}");
+ for hit in judged["hits"]["hits"].as_array().unwrap() {
+ let original = base["hits"]["hits"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .find(|h| h["_id"] == hit["_id"])
+ .unwrap();
+ assert_eq!(hit["_score"], original["_score"], "{hit}");
+ }
+ let cut = (p("exact") + p("partial")) / 2.0;
+ let (st, kept) = node
+ .search(
+ "/cjk/_search",
+ json!({"query":{"match_all":{}},"size":3,"judge":{"query":"雷同","min_p":cut}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{kept}");
+ assert_eq!(ids(&kept), ["exact"], "{kept}");
+ assert_eq!(kept["hits"]["total"], base["hits"]["total"]);
+ assert_eq!(kept["judged"]["scorer"], "lexical");
+ assert_eq!(kept["judged"]["judged"], 3);
+ assert_eq!(kept["judged"]["kept"], 1);
+ assert_eq!(kept["judged"]["dropped"], 2);
+ }
+}
+
#[tokio::test]
async fn judged_hits_carry_a_probability_and_the_block_reports_the_pass() {
let node = node().await;
diff --git a/engine/crates/xerj-rerank/src/judge.rs b/engine/crates/xerj-rerank/src/judge.rs
index fd51419b8..a587cb7f3 100644
--- a/engine/crates/xerj-rerank/src/judge.rs
+++ b/engine/crates/xerj-rerank/src/judge.rs
@@ -184,14 +184,59 @@ impl JudgeConfig {
const K1: f64 = 1.2;
const B: f64 = 0.75;
-/// Lowercase word tokens: alphanumeric runs, in order, via `char` boundaries
-/// only (a byte split here would panic on multi-byte text — the defect that
-/// crash-looped `autoindex`, see [`crate::clip`]). The same split the
-/// `rerank` conformance tests use for word overlap.
+/// Scripts for which an uninterrupted alphanumeric run is not a word (#2580).
+/// Include Han's supplementary ideographic planes; all slicing below follows
+/// char_indices, including four-byte characters. This is character n-gramming,
+/// not dictionary segmentation, normalization or semantic understanding.
+fn is_cjk(c: char) -> bool {
+ matches!(c as u32,
+ 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF |
+ 0x20000..=0x3FFFF | 0x3040..=0x309F | 0x30A0..=0x30FF |
+ 0xAC00..=0xD7AF)
+}
+
+/// Preserve lowercase non-CJK alphanumeric runs. CJK runs emit each character
+/// and each adjacent pair, so a single-character query can match inside a run
+/// and the pair `雷同` has evidence beyond the separate characters `同雷`.
+/// Separators and non-CJK runs break pairs; title and body are tokenized apart.
+///
+/// Reference approach (MIT, no implementation copied): Tantivy's
+/// src/tokenizer/ngram_tokenizer.rs:115-119 admits all inner n-grams, :154-164
+/// emits codepoint-bounded slices, :253-280 defines the UTF-8 frontiers.
+/// https://github.com/quickwit-oss/tantivy/blob/main/src/tokenizer/ngram_tokenizer.rs
+/// Unlike applying that tokenizer to the whole string, restrict grams to CJK
+/// here so the established Latin/number token and scoring contract survives.
fn tokens(s: &str) -> impl Iterator<Item = String> + '_ {
s.split(|c: char| !c.is_alphanumeric())
.filter(|w| !w.is_empty())
- .map(str::to_lowercase)
+ .flat_map(|word| {
+ let mut chars = word.char_indices().peekable();
+ let mut pair = None;
+ std::iter::from_fn(move || {
+ if let Some(pair) = pair.take() {
+ return Some(pair);
+ }
+ let (start, c) = chars.next()?;
+ let mut end = start + c.len_utf8();
+ if is_cjk(c) {
+ if let Some(&(next_start, next)) = chars.peek() {
+ if is_cjk(next) {
+ pair = Some(word[start..next_start + next.len_utf8()].to_string());
+ }
+ }
+ Some(c.to_string())
+ } else {
+ while let Some(&(next_start, next)) = chars.peek() {
+ if is_cjk(next) {
+ break;
+ }
+ chars.next();
+ end = next_start + next.len_utf8();
+ }
+ Some(word[start..end].to_lowercase())
+ }
+ })
+ })
}
/// The deterministic lexical judge.
@@ -240,23 +285,33 @@ pub fn lexical_scores(query: &str, candidates: &[Candidate]) -> Vec<Scored> {
.collect();
}
- // Tokenize each candidate once: title and text are one bag of words —
- // a judge reads a document, not a schema.
- let docs: Vec<Vec<String>> = candidates
+ // Count tokens once, retaining frequencies only for query terms. CJK
+ // expansion must not turn IDF into a scan of every document token for
+ // every query gram. Length still counts all tokens, exactly as before.
+ let qset: std::collections::HashSet<&str> = q_terms.iter().map(String::as_str).collect();
+ let docs: Vec<(usize, HashMap<&str, u64>)> = candidates
.iter()
.map(|c| {
- let mut ts: Vec<String> = Vec::new();
- if let Some(t) = &c.title {
- ts.extend(tokens(t));
+ let mut len = 0;
+ let mut tf = HashMap::new();
+ for token in c
+ .title
+ .iter()
+ .flat_map(|t| tokens(t))
+ .chain(tokens(&c.text))
+ {
+ len += 1;
+ if let Some(&q) = qset.get(token.as_str()) {
+ *tf.entry(q).or_insert(0) += 1;
+ }
}
- ts.extend(tokens(&c.text));
- ts
+ (len, tf)
})
.collect();
let n = docs.len() as f64;
let avgdl = {
- let total: usize = docs.iter().map(Vec::len).sum();
+ let total: usize = docs.iter().map(|(len, _)| len).sum();
(total as f64 / n).max(1.0)
};
@@ -266,26 +321,22 @@ pub fn lexical_scores(query: &str, candidates: &[Candidate]) -> Vec<Scored> {
let idf: Vec<f64> = q_terms
.iter()
.map(|q| {
- let df = docs.iter().filter(|d| d.iter().any(|t| t == q)).count() as f64;
+ let df = docs
+ .iter()
+ .filter(|(_, tf)| tf.contains_key(q.as_str()))
+ .count() as f64;
(1.0 + (n - df + 0.5) / (df + 0.5)).ln()
})
.collect();
// The maximum attainable numerator: every query term at full saturation.
let max_score: f64 = idf.iter().sum::<f64>() * (K1 + 1.0);
- let qset: std::collections::HashSet<&str> = q_terms.iter().map(String::as_str).collect();
candidates
.iter()
.enumerate()
.map(|(i, c)| {
- let d = &docs[i];
- let norm = K1 * (1.0 - B + B * d.len() as f64 / avgdl);
- let mut tf: HashMap<&str, u64> = HashMap::new();
- for t in d {
- if qset.contains(t.as_str()) {
- *tf.entry(t.as_str()).or_insert(0) += 1;
- }
- }
+ let (len, tf) = &docs[i];
+ let norm = K1 * (1.0 - B + B * *len as f64 / avgdl);
let numerator: f64 = idf
.iter()
.zip(q_terms.iter())
@@ -557,6 +608,112 @@ mod tests {
// ── the lexical scorer ──────────────────────────────────────────────────
+ #[test]
+ fn cjk_query_matches_inside_unsegmented_text() {
+ let out = lexical_scores(
+ "雷同",
+ &[
+ cand(7, None, "该段内容存在雷同需要核查"),
+ cand(2, None, "今天的天气晴朗适合出行"),
+ ],
+ );
+ assert!(
+ out[0].score > 0.0,
+ "embedded Chinese term must match: {out:?}"
+ );
+ assert_eq!(out[1].score, 0.0);
+ assert_eq!(out.iter().map(|s| s.ordinal).collect::<Vec<_>>(), [7, 2]);
+ }
+
+ #[test]
+ fn cjk_single_character_and_adjacent_pair_have_evidence() {
+ let candidates = [
+ cand(0, Some("中文检索"), "雷同内容"),
+ cand(1, None, "同雷内容"),
+ ];
+ let single = lexical_scores("雷", &candidates);
+ assert!(single.iter().all(|s| s.score > 0.0), "{single:?}");
+ let pair = lexical_scores("雷同", &candidates);
+ assert!(
+ pair[0].score > pair[1].score,
+ "adjacency adds evidence: {pair:?}"
+ );
+ }
+
+ #[test]
+ fn legacy_non_cjk_score_receipt() {
+ let candidates = [
+ cand(9, Some("Trial results"), "vitamin d improved bone density"),
+ cand(3, None, "vitamin rich vegetables"),
+ cand(2, None, "other topic"),
+ ];
+ let out = lexical_scores("vitamin bone density", &candidates);
+ assert_eq!(
+ out.iter().map(|s| s.score.to_bits()).collect::<Vec<_>>(),
+ [4599937497660334437, 4591716393592272481, 0]
+ );
+ }
+
+ #[test]
+ fn cjk_tokens_stop_at_separators_and_keep_non_cjk_words() {
+ assert_eq!(
+ tokens("API中文v2 CAFÉ İSTANBUL 123").collect::<Vec<_>>(),
+ [
+ "api",
+ "中",
+ "中文",
+ "文",
+ "v2",
+ "café",
+ "i\u{307}stanbul",
+ "123"
+ ]
+ );
+ for text in ["雷,同", "雷 同", "雷😀同", "雷A同"] {
+ assert!(!tokens(text).any(|t| t == "雷同"), "{text}");
+ }
+ assert_eq!(
+ tokens("𠀀中文").collect::<Vec<_>>(),
+ ["𠀀", "𠀀中", "中", "中文", "文"]
+ );
+ assert_eq!(
+ tokens("東京都 한국어").collect::<Vec<_>>(),
+ ["東", "東京", "京", "京都", "都", "한", "한국", "국", "국어", "어"]
+ );
+ assert_eq!(
+ tokens("alpha_beta can't naïve").collect::<Vec<_>>(),
+ ["alpha", "beta", "can", "t", "naïve"]
+ );
+ }
+
+ #[test]
+ fn cjk_title_and_text_do_not_create_a_cross_field_pair() {
+ let scores = lexical_scores("雷同", &[cand(4, Some("雷"), "同"), cand(8, None, "雷同")]);
+ assert!(scores[1].score > scores[0].score, "{scores:?}");
+ assert_eq!(
+ scores,
+ lexical_scores("雷同", &[cand(4, Some("雷"), "同"), cand(8, None, "雷同"),])
+ );
+ assert!(scores
+ .iter()
+ .all(|s| s.score.is_finite() && (0.0..=1.0).contains(&s.score)));
+ }
+
+ #[test]
+ fn cjk_scoring_handles_the_query_and_document_character_limits() {
+ let query: String = (0..crate::MAX_QUERY_CHARS)
+ .map(|i| char::from_u32(0x4e00 + i as u32).unwrap())
+ .collect();
+ let text = format!("{query}{query}");
+ assert_eq!(text.chars().count(), crate::MAX_DOC_CHARS);
+ let candidates: Vec<_> = (0..crate::JEV_MAX_DOCS_PER_CALL)
+ .map(|i| cand(i, None, &text))
+ .collect();
+ let scores = lexical_scores(&query, &candidates);
+ assert_eq!(scores.len(), candidates.len());
+ assert!(scores.iter().all(|s| s.score > 0.0 && s.score <= 1.0));
+ }
+
#[test]
fn no_query_words_scores_zero_everywhere() {
let out = lexical_scores(
diff --git a/tools/cjk-judge/VERIFICATION.md b/tools/cjk-judge/VERIFICATION.md
new file mode 100644
index 000000000..f48b997a2
--- /dev/null
+++ b/tools/cjk-judge/VERIFICATION.md
@@ -0,0 +1,212 @@
+# CJK lexical judge (#2580)
+
+Written by an AI coding agent (OpenAI Codex), operated by @Jarvis636431.
+macOS arm64, Rust 1.97.1, source-built xerj v1.0.0-rc.100.
+Original verification: independent branch initially based on upstream `25b108084`, rebased onto
+`8f62343c7`. The intervening commit changes only `.contributors`; the tested
+Rust sources and runtime snapshot are unchanged. No stacked dependency.
+
+## Main refresh after #2697 (2026-10-30)
+
+Merge upstream `6a7b390ac` into the prior PR head `25c61738d`. The only
+conflict is the CHANGELOG insertion point introduced by #2697; preserve its
+32 MiB batch-response fix entry and the CJK entry. The previous maintainer
+merge remains in the ancestry. The judge implementation, HTTP regression,
+smoke harness and rerank documentation are byte-identical to the previous
+PR head; the effective main diff is still the same six files.
+
+Fresh reruns on this composition: default rerank 59 passed, `judge-local`
+61 passed and HTTP judge 15 passed, all with zero failures, ignores or
+filters. Scoped rerank release build, formatting, strict Clippy in both
+configurations, Python syntax, whitespace and the workflow-timeout guard
+passed. The isolated node/runner build passed with the existing autoindex
+`resolve_reporting` dead-code warning. Original failing-before-fix evidence
+and scope exclusions below remain applicable; no new behavior is added by
+this conflict resolution.
+
+Fresh private-node ES-YAML on this composition: **1,453 passed, 0 failed,
+10 skipped across 233 files**, retaining 74 existing catch-lenient cases.
+The same executable then passed HTTP/MCP order, CJK evidence, threshold
+counts, engine totals and original `_score` before flush, after flush and
+after graceful restart. Scores remained exact 0.5900621118012421, partial
+0.24340586742297435 and unrelated 0.0. The harness, runner and node exited 0.
+Snapshot SHA-256:
+`71735dfe2699509fd66c6febc2e5e84b9b7414e795854d0c8a9a7add56dc9907`.
+These are fresh local receipts; hosted CI for the pushed refresh remains
+separate. Linux/Windows, trained-model quality and broad relevance were not
+re-tested. No performance claim is made.
+
+## Earlier main integration (2026-10-30)
+
+Merge current upstream `befe92035` into the PR's remote head `e45809c31`,
+preserving the maintainer-authored merge and its egress-inventory fix (#2639).
+Resolve the sole conflict in `CHANGELOG.md` by retaining both the CJK entry
+and every upstream entry. The judge implementation, HTTP regression, smoke
+harness and `docs/RERANK.md` are byte-identical to the previous PR head.
+The effective diff remains the same six files; no stacked prerequisite,
+workflow change, upstream rollback or new product behavior is introduced.
+
+Fresh commands on the composed tree, from `engine/`:
+
+```sh
+cargo test --locked --offline -p xerj-rerank
+cargo test --locked --offline -p xerj-rerank --features judge-local
+cargo build --locked --offline --release -j 32 -p xerj-rerank
+cargo fmt --all --check
+cargo clippy --locked --offline -p xerj-rerank --all-targets -- -D warnings
+cargo clippy --locked --offline -p xerj-rerank --features judge-local --all-targets -- -D warnings
+cargo build --locked --offline --profile ci-test -j 8 -p xerj-server -p es-yaml-runner
+cargo test --locked --offline --profile ci-test -p xerj-api --test judge_stage_http
+```
+
+Default rerank: 59 passed; `judge-local`: 61 passed. Both have zero failures,
+ignored tests or filters, including all four egress-inventory tests. The
+old filter below is historical and is no longer required. Scoped release
+build, formatting and strict Clippy in both configurations passed.
+The workflow-timeout guard and Python syntax check also passed.
+
+HTTP judge: 15 passed, zero failures, ignored tests or filters. The isolated
+node/runner build passed with the existing autoindex `resolve_reporting`
+dead-code warning. From repository root, the unchanged smoke command above
+(`python3 tools/cjk-judge/smoke.py --binary engine/target/ci-test/xerj
+--runner engine/target/ci-test/es-yaml-runner`) exited 0:
+
+- ES-YAML: 1,453 passed, 0 failed, 10 skipped across 233 files; 74 existing
+ catch-lenient cases. No suite, skip or catch-lenient rule changed in this PR.
+- The same executable snapshot passed CJK HTTP/MCP order, threshold counts,
+ engine totals and original `_score` checks before flush, after flush and
+ after graceful restart. Scores were exact 0.5900621118012421, partial
+ 0.24340586742297435, unrelated 0.0 in all three checks.
+- Executable snapshot SHA-256:
+ `c0a919e7c69e8a8c9418386e5703081ea3f3da19fc3380c0e0839760d5d69e5a`.
+
+Audited the composed diff against `docs/CONTRIBUTION_REVIEW.md`: upstream
+API/storage/workflow changes are intact, existing CJK/non-CJK assertions are
+unchanged, and the original failing-before-fix receipt below remains the
+regression evidence. All original scope limits and non-applicable durable
+state, migration, accelerator and distributed-write sections still apply.
+Linux/Windows and broad relevance/model quality were not re-tested. Hosted
+CI on the pushed head is separate from these local results.
+
+## Scope and evidence before implementation
+
+The old lexical judge splits only at non-alphanumeric characters. The query
+`雷同` therefore has no token in common with `该段内容存在雷同需要核查`.
+Both new CJK score tests were run against that implementation and failed:
+the containing document and the unrelated document both scored 0.0.
+The tests now pass with the same assertions.
+
+Reference approach, retrieved before implementation: Tantivy's MIT-licensed
+[`src/tokenizer/ngram_tokenizer.rs`](https://github.com/quickwit-oss/tantivy/blob/main/src/tokenizer/ngram_tokenizer.rs),
+lines 115–119 (all inner grams), 154–164 (codepoint-bounded slices), 253–280
+(UTF-8 frontiers). Its [LICENSE](https://github.com/quickwit-oss/tantivy/blob/main/LICENSE)
+was checked. No implementation was copied. XERJ's existing
+`xerj-fts/src/analyzer.rs:799–862` also isolates CJK runs; its bigram-only
+policy is not changed by this patch.
+
+The new judge emits characters and adjacent pairs within CJK runs. Non-CJK
+alphanumeric runs keep their old lowercase tokens. Separators and the
+title/body boundary do not synthesize pairs. No dictionary, normalization,
+model, calibrated confidence or first-stage analyzer change is introduced.
+Single-character overlap can contribute even when the whole query phrase is
+absent; adjacency is additional evidence, not a phrase-match guarantee.
+
+## Local tests and builds
+
+Run from `engine/`:
+
+```sh
+cargo test --locked --offline -p xerj-rerank --lib judge::tests -- --nocapture
+cargo test --locked --offline -p xerj-rerank -- --skip every_outbound_client_in_the_engine_is_on_the_published_list
+cargo test --locked --offline -p xerj-rerank --features judge-local -- --skip every_outbound_client_in_the_engine_is_on_the_published_list
+cargo test --locked --offline --profile ci-test -p xerj-api --test judge_stage_http
+cargo fmt --all --check
+cargo build --locked --offline --release -j 32 -p xerj-rerank
+cargo clippy --locked --offline -p xerj-rerank --all-targets -- -D warnings
+cargo clippy --locked --offline -p xerj-rerank --features judge-local --all-targets -- -D warnings
+```
+
+Observed: default rerank 58 passed / 0 failed / 1 explicit filter;
+`judge-local` rerank 60 passed / 0 failed / the same filter; HTTP judge 15
+passed / 0 failed. Release build, formatting and both strict Clippy variants
+passed. The HTTP suite's first sandboxed run passed 13 tests and failed the
+two tests needing a loopback listener with PermissionDenied; the full suite
+then passed outside the sandbox.
+
+The unfiltered rerank suite fails its unchanged egress inventory check:
+`new outbound network client(s) in ["xerj-mcp/src/status.rs"]`. The same
+unchanged test target was compiled/run on a clean `25b108084` worktree and
+failed with the identical path (3 passed / 1 failed). No egress table,
+assertion, committed skip or unrelated client is altered here.
+
+Tests cover the embedded Chinese term, one-character evidence, adjacency,
+punctuation/emoji/Latin boundaries, supplementary Han, Japanese/Hangul
+syllables, title/body separation, determinism and finite bounded scores at
+the existing 4,000-query/8,000-document character ceilings for 30 candidates.
+One non-CJK golden fixture records the old score bits
+`[4599937497660334437, 4591716393592272481, 0]`; the candidate matches them
+exactly. The query-term frequency table counts every emitted token for length
+normalization and keeps fixed summation order, without rescanning document
+tokens for each new gram.
+No latency, RSS, throughput or general retrieval-quality claim is made.
+
+## Real private-node control and candidate
+
+From repository root, `smoke.py --binary <path>` snapshots the executable,
+starts a temporary lexical-mode node and seeds three synthetic documents.
+It uses `match_all` to hold the candidate set fixed, judges `雷同`, checks
+threshold counts/engine totals and original `_score`, compares HTTP/MCP,
+flushes, and gracefully restarts from the same private data directory.
+Only this temporary node uses a 99% disk flood watermark on the shared
+laptop. Hosted rerank is disabled; no model, personal corpus or existing node
+is used. No index format or reindex/migration is involved.
+
+The original control snapshot from prior source base `7d3ca96b1` is
+`bf8e611d814f935e5a313cb735f33fb3d7ff532b78c04a5fe3af87eb020f3853`.
+That base and current `25b108084` have the identical original judge blob
+`fd51419b804a9240fb2248256671851a7a480ee1`. The control is behavior evidence,
+not a matched-build performance comparison. With `--expect-zero`, all three
+documents score 0.0 before flush, after flush and after restart; a positive
+threshold drops them all.
+
+The candidate snapshot is
+`9bef44e59d43b3403f15364452c3e8c7e67f6bdc5dcce4d40498d293754c0c87`.
+Observed scores in this fixed page: exact-containing document
+0.5900621118012421, partial single-character evidence 0.24340586742297435,
+unrelated document 0.0. All HTTP/MCP, pruning/count, flush and restart checks
+passed. These are lexical fixture scores, not probabilities calibrated
+against human relevance judgments.
+
+The same candidate executable passed the full ES-YAML gate on a fresh private
+node: **1,451 passed, 0 failed, 10 skipped across 232 files**, with the
+existing 75 catch-lenient cases. Runner and node exited 0. The subsequent
+HTTP/MCP, threshold, flush and restart checks repeated the same scores.
+
+```sh
+# Run the scoped builds from engine/ before this repository-root command:
+# cargo build --locked --offline --profile ci-test -j 8 -p xerj-server -p es-yaml-runner
+python3 tools/cjk-judge/smoke.py \
+ --binary engine/target/ci-test/xerj \
+ --runner engine/target/ci-test/es-yaml-runner
+```
+
+## Contribution review and exclusions
+
+Audited against `docs/CONTRIBUTION_REVIEW.md`: preserve opt-in behavior,
+non-CJK word tokens, deterministic arithmetic, input ordinals, scorer name,
+original engine scores, count semantics, egress policy and model path. The
+HTTP test uses the real router/storage before and after flush. The private
+binary harness adds MCP transport and durable restart coverage.
+
+No dependency, workflow, stored schema, journal, migration, provider/model
+execution, feature-gate or installation/deployment change. Durable-state
+failure atomicity, optional accelerator fallback and distributed replication
+are not applicable. The ordinary first-stage search path is unchanged.
+
+Not tested: the original six-file Windows corpus, broad CJK relevance,
+cross-language threshold calibration, dictionary segmentation, half-width
+kana/jamo normalization, or Linux/Windows execution. In particular, first-stage
+candidate misses and rank order without `judge` in #2580 remain separate.
+The existing model scorer/tokenizer is unchanged; feature tests use its tiny
+synthetic fixture, not a real trained-model quality evaluation. Hosted CI
+remains separate from local receipts.
diff --git a/tools/cjk-judge/smoke.py b/tools/cjk-judge/smoke.py
new file mode 100644
index 000000000..6b384f19a
--- /dev/null
+++ b/tools/cjk-judge/smoke.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""Check #2580 on a private node, holding first-stage candidates fixed.
+
+Accept --expect-zero for an original lexical-judge binary, or --runner for a
+fresh-node compatibility gate before the fixture. No model or personal data.
+"""
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import signal
+import socket
+import subprocess
+import tempfile
+import time
+from urllib.request import build_opener, ProxyHandler, Request
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--binary', type=Path, required=True)
+ parser.add_argument('--expect-zero', action='store_true')
+ parser.add_argument('--runner', type=Path)
+ args = parser.parse_args()
+ original = args.binary.resolve(strict=True)
+ runner = args.runner.resolve(strict=True) if args.runner else None
+ repo = Path(__file__).resolve().parents[2]
+ env = {k: v for k, v in os.environ.items() if not k.startswith('XERJ_')
+ and k.lower() not in ('http_proxy', 'https_proxy', 'all_proxy', 'no_proxy')}
+ env.update(XERJ_DISABLE_FEEDBACK='true', XERJ_NUM_CPUS='4')
+ opener = build_opener(ProxyHandler({}))
+ with tempfile.TemporaryDirectory(prefix='xerj-cjk-judge-') as temp:
+ root = Path(temp)
+ binary = root / ('xerj.exe' if os.name == 'nt' else 'xerj')
+ shutil.copyfile(original, binary)
+ binary.chmod(0o700)
+ print('Executable snapshot SHA-256:', hashlib.sha256(binary.read_bytes()).hexdigest(), flush=True)
+ config = root / 'node.toml'
+ # Only this tiny temporary fixture changes its volume reserve.
+ config.write_text('[server]\nauto_port = false\n\n[limits]\ndisk_flood_stage_percent = 99\n\n[rerank]\nenabled = false\n')
+ env['XERJ_CONFIG'] = str(config)
+ with socket.socket() as sock:
+ sock.bind(('127.0.0.1', 0))
+ port = sock.getsockname()[1]
+ assert port <= 65533
+ url = f'http://127.0.0.1:{port}'
+
+ def run(command, stdin=None, timeout=120):
+ result = subprocess.run([str(x) for x in command], input=stdin, capture_output=True,
+ text=True, cwd=root, env=env, timeout=timeout)
+ assert result.returncode == 0, (command, result.returncode, result.stdout, result.stderr)
+ return result.stdout
+
+ def request(method, path, body=None):
+ data = json.dumps(body).encode() if body is not None else None
+ req = Request(url + path, data=data, method=method, headers={'Content-Type': 'application/json'})
+ with opener.open(req, timeout=30) as response:
+ return json.load(response)
+
+ def query(judge=None):
+ body = {'query': {'match_all': {}}, 'size': 3}
+ if judge is not None:
+ body['judge'] = judge
+ return request('POST', '/cjkjudge-docs/_search', body)
+
+ def check():
+ base = query()
+ assert 'judged' not in base
+ assert all('_p_relevant' not in h for h in base['hits']['hits'])
+ judged = query({'query': '雷同'})
+ hits = judged['hits']['hits']
+ assert len(hits) == 3 and judged['judged']['scorer'] == 'lexical', judged
+ scores = {h['_id']: h['_p_relevant'] for h in hits}
+ print('Observed lexical scores:', json.dumps(scores, ensure_ascii=False), flush=True)
+ assert {h['_id']: h['_score'] for h in hits} == {h['_id']: h['_score'] for h in base['hits']['hits']}
+ if args.expect_zero:
+ assert all(p == 0.0 for p in scores.values()), scores
+ assert not query({'query': '雷同', 'min_p': 0.01})['hits']['hits']
+ return
+ assert scores['exact'] > scores['partial'] > 0.0 == scores['unrelated'], judged
+ assert hits[0]['_id'] == 'exact', judged
+ cut = (scores['exact'] + scores['partial']) / 2
+ kept = query({'query': '雷同', 'min_p': cut})
+ assert [h['_id'] for h in kept['hits']['hits']] == ['exact'], kept
+ assert kept['hits']['total'] == base['hits']['total']
+ assert [kept['judged'][k] for k in ['judged', 'kept', 'dropped']] == [3, 1, 2], kept
+ messages = [
+ {'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': {}},
+ {'jsonrpc': '2.0', 'id': 2, 'method': 'tools/call', 'params': {
+ 'name': 'xerj_search', 'arguments': {'index': 'cjkjudge-docs',
+ 'query_dsl': {'match_all': {}}, 'size': 3, 'judge': {'query': '雷同'}}}},
+ ]
+ raw = run([binary, 'mcp', '--url', url], ''.join(json.dumps(m) + '\n' for m in messages))
+ reply = next(json.loads(line)['result'] for line in raw.splitlines() if json.loads(line).get('id') == 2)
+ assert not reply.get('isError'), reply
+ mcp = json.loads(reply['content'][0]['text'])
+ assert [(h['_id'], h['_p_relevant']) for h in mcp['hits']['hits']] == [(h['_id'], h['_p_relevant']) for h in hits], mcp
+ print('PASS HTTP/MCP order, CJK evidence, threshold counts, total and original _score', flush=True)
+
+ print(run([binary, '--version']), end='', flush=True)
+ for phase in ['first-run', 'restart']:
+ with (root / f'{phase}.log').open('w') as log:
+ node = subprocess.Popen([str(binary), '--insecure', '--port', str(port),
+ '--data-dir', str(root / 'data'), '--embed-mode', 'lexical'],
+ cwd=root, env=env, stdout=log, stderr=log)
+ try:
+ deadline = time.monotonic() + 60
+ while True:
+ assert node.poll() is None, (root / f'{phase}.log').read_text()
+ try:
+ request('GET', '/')
+ break
+ except OSError:
+ if time.monotonic() >= deadline:
+ raise TimeoutError('node readiness')
+ time.sleep(0.1)
+ if phase == 'first-run':
+ if runner:
+ print(run([runner, '--url', url, '--dir', repo / 'engine/tests/es-compat-yaml/yaml'], timeout=900), flush=True)
+ request('PUT', '/cjkjudge-docs', {'mappings': {'properties': {'body': {'type': 'text'}}}})
+ for id_, body in [('exact', '这里发现雷同内容雷同内容'), ('partial', '部分雷声同时出现'), ('unrelated', '今天阳光明媚风景宜人')]:
+ request('PUT', f'/cjkjudge-docs/_doc/{id_}?refresh=true', {'body': body})
+ check()
+ request('POST', '/cjkjudge-docs/_flush', {})
+ check()
+ finally:
+ if node.poll() is None:
+ node.send_signal(signal.SIGTERM)
+ try:
+ node.wait(timeout=30)
+ except subprocess.TimeoutExpired:
+ node.kill()
+ node.wait()
+ raise
+ assert node.returncode == 0, (root / f'{phase}.log').read_text()
+ print('PASS buffered, flushed and restarted fixture' + (' (original zero-score control)' if args.expect_zero else ''), flush=True)
+
+
+if __name__ == '__main__':
+ main()