AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(judge): score CJK evidence inside continuous text (#2635)
Jarvise7777dcdc10a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6e673a7bd..ab60ae2fb 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -47,6 +47,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Opt-in lexical `judge` now matches CJK characters and adjacent pairs inside+ uninterrupted Chinese/Japanese/Korean text instead of treating the whole+ run as one word (refs #2580). Single-character evidence is retained, pairs+ add adjacency evidence, and non-CJK word tokenization stays unchanged.+ This changes judged CJK scores/threshold outcomes; it does not change the+ first-stage analyzer or recover candidates absent from its result page.+- `xerj corpus --help`, `xerj autoindex --help` (`--watch`) and the XERJ.code README now say that corpora and folders do not follow their git remotes, and give the refresh recipe: `git pull` in each checkout, re-run `xerj corpus add` so `corpus.json` records the new commit, then `xerj corpus index`. (#2416)- `xerj_batch_search` (and `xerj batch-search`): a batch whose combined `_msearch` response passes the 32 MiB cap no longer fails every item. The request is retried in halves, and only an item whose own response is over the cap fails, with `response_too_large`. (#2694)- `xerj autoindex` and `xerj autoindex map`: when the node is unreachable at a `localhost` URL, the error now also suggests the same URL with `127.0.0.1`, since the node binds the IPv4 loopback by default and `localhost` can resolve to `::1` first (seen on Windows). (#2688)diff --git a/docs/RERANK.md b/docs/RERANK.mdindex 184fb92c6..11e554a26 100644--- a/docs/RERANK.md+++ b/docs/RERANK.md@@ -51,6 +51,26 @@ client, the request parser and the failure policy are the leaf crate`engine/crates/xerj-rerank`. The end-to-end tests are`engine/crates/xerj-api/tests/rerank_stage_http.rs`.+## Local lexical `judge` and CJK text++The separate opt-in `judge` block can run a deterministic lexical scorer on+the returned candidate page. It keeps `_score` and adds `_p_relevant`; the+response identifies this scorer as `judged.scorer: "lexical"`. The lexical+arm needs no hosted provider or model. A node configured with a local judge+model uses that model's own tokenizer instead.++The lexical arm splits CJK runs into single characters and adjacent pairs.+For example, a query containing `雷同` can match inside `内容存在雷同需要核查`;+the pair contributes evidence beyond two isolated or reversed characters.+Punctuation, non-CJK runs and the title/body boundary break pairs. Non-CJK+alphanumeric runs retain their existing lowercase word tokens. This is+character n-gram overlap, not dictionary segmentation or semantic relevance.++CJK scores and `judge.min_p` filtering can change with this tokenizer; no+cross-language calibration is claimed. The judge only reorders/filters the+candidates it receives. It cannot recover a document missed by the first+stage, and this change does not change index analyzers or require reindexing.+## Set it upA node needs a provider key. There are two places to put it, and **a value indiff --git a/engine/crates/xerj-api/tests/judge_stage_http.rs b/engine/crates/xerj-api/tests/judge_stage_http.rsindex 5d4a5b717..5a6ce68e0 100644--- a/engine/crates/xerj-api/tests/judge_stage_http.rs+++ b/engine/crates/xerj-api/tests/judge_stage_http.rs@@ -293,6 +293,91 @@ async fn no_judge_block_leaves_the_response_untouched() {// Applied: _p_relevant per hit, reorder, the judged block// ─────────────────────────────────────────────────────────────────────────────+/// #2580: use match_all to hold the candidate set fixed. This isolates the+/// judge's tokenization from the separate first-stage CJK recall question.+#[tokio::test]+async fn cjk_judge_finds_embedded_terms_and_thresholds_without_changing_engine_scores() {+ let node = node().await;+ let (st, body) = node+ .call(+ "PUT",+ "/cjk",+ json!({"mappings":{"properties":{+ "body":{"type":"text"}+ }}}),+ )+ .await;+ assert!(st.is_success(), "{st} {body}");+ for (id, text) in [+ ("exact", "这里发现雷同内容雷同内容"),+ ("partial", "部分雷声同时出现"),+ ("unrelated", "今天阳光明媚风景宜人"),+ ] {+ let (st, body) = node+ .call(+ "PUT",+ &format!("/cjk/_doc/{id}?refresh=true"),+ json!({"body":text}),+ )+ .await;+ assert!(st.is_success(), "{st} {body}");+ }+ for flushed in [false, true] {+ if flushed {+ let (st, body) = node.call("POST", "/cjk/_flush", json!({})).await;+ assert!(st.is_success(), "{st} {body}");+ }+ let (_, base) = node+ .search("/cjk/_search", json!({"query":{"match_all":{}},"size":3}))+ .await;+ assert!(base.get("judged").is_none(), "{base}");+ let (st, judged) = node+ .search(+ "/cjk/_search",+ json!({"query":{"match_all":{}},"size":3,"judge":{"query":"雷同"}}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "{judged}");+ assert_eq!(ids(&judged)[0], "exact", "{judged}");+ let p = |id| {+ judged["hits"]["hits"]+ .as_array()+ .unwrap()+ .iter()+ .find(|h| h["_id"] == id)+ .unwrap()["_p_relevant"]+ .as_f64()+ .unwrap()+ };+ assert!(p("exact") > p("partial"), "{judged}");+ assert!(p("partial") > 0.0, "{judged}");+ assert_eq!(p("unrelated"), 0.0, "{judged}");+ for hit in judged["hits"]["hits"].as_array().unwrap() {+ let original = base["hits"]["hits"]+ .as_array()+ .unwrap()+ .iter()+ .find(|h| h["_id"] == hit["_id"])+ .unwrap();+ assert_eq!(hit["_score"], original["_score"], "{hit}");+ }+ let cut = (p("exact") + p("partial")) / 2.0;+ let (st, kept) = node+ .search(+ "/cjk/_search",+ json!({"query":{"match_all":{}},"size":3,"judge":{"query":"雷同","min_p":cut}}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "{kept}");+ assert_eq!(ids(&kept), ["exact"], "{kept}");+ assert_eq!(kept["hits"]["total"], base["hits"]["total"]);+ assert_eq!(kept["judged"]["scorer"], "lexical");+ assert_eq!(kept["judged"]["judged"], 3);+ assert_eq!(kept["judged"]["kept"], 1);+ assert_eq!(kept["judged"]["dropped"], 2);+ }+}+#[tokio::test]async fn judged_hits_carry_a_probability_and_the_block_reports_the_pass() {let node = node().await;diff --git a/engine/crates/xerj-rerank/src/judge.rs b/engine/crates/xerj-rerank/src/judge.rsindex fd51419b8..a587cb7f3 100644--- a/engine/crates/xerj-rerank/src/judge.rs+++ b/engine/crates/xerj-rerank/src/judge.rs@@ -184,14 +184,59 @@ impl JudgeConfig {const K1: f64 = 1.2;const B: f64 = 0.75;-/// Lowercase word tokens: alphanumeric runs, in order, via `char` boundaries-/// only (a byte split here would panic on multi-byte text — the defect that-/// crash-looped `autoindex`, see [`crate::clip`]). The same split the-/// `rerank` conformance tests use for word overlap.+/// Scripts for which an uninterrupted alphanumeric run is not a word (#2580).+/// Include Han's supplementary ideographic planes; all slicing below follows+/// char_indices, including four-byte characters. This is character n-gramming,+/// not dictionary segmentation, normalization or semantic understanding.+fn is_cjk(c: char) -> bool {+ matches!(c as u32,+ 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF |+ 0x20000..=0x3FFFF | 0x3040..=0x309F | 0x30A0..=0x30FF |+ 0xAC00..=0xD7AF)+}++/// Preserve lowercase non-CJK alphanumeric runs. CJK runs emit each character+/// and each adjacent pair, so a single-character query can match inside a run+/// and the pair `雷同` has evidence beyond the separate characters `同雷`.+/// Separators and non-CJK runs break pairs; title and body are tokenized apart.+///+/// Reference approach (MIT, no implementation copied): Tantivy's+/// src/tokenizer/ngram_tokenizer.rs:115-119 admits all inner n-grams, :154-164+/// emits codepoint-bounded slices, :253-280 defines the UTF-8 frontiers.+/// https://github.com/quickwit-oss/tantivy/blob/main/src/tokenizer/ngram_tokenizer.rs+/// Unlike applying that tokenizer to the whole string, restrict grams to CJK+/// here so the established Latin/number token and scoring contract survives.fn tokens(s: &str) -> impl Iterator<Item = String> + '_ {s.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty())- .map(str::to_lowercase)+ .flat_map(|word| {+ let mut chars = word.char_indices().peekable();+ let mut pair = None;+ std::iter::from_fn(move || {+ if let Some(pair) = pair.take() {+ return Some(pair);+ }+ let (start, c) = chars.next()?;+ let mut end = start + c.len_utf8();+ if is_cjk(c) {+ if let Some(&(next_start, next)) = chars.peek() {+ if is_cjk(next) {+ pair = Some(word[start..next_start + next.len_utf8()].to_string());+ }+ }+ Some(c.to_string())+ } else {+ while let Some(&(next_start, next)) = chars.peek() {+ if is_cjk(next) {+ break;+ }+ chars.next();+ end = next_start + next.len_utf8();+ }+ Some(word[start..end].to_lowercase())+ }+ })+ })}/// The deterministic lexical judge.@@ -240,23 +285,33 @@ pub fn lexical_scores(query: &str, candidates: &[Candidate]) -> Vec<Scored> {.collect();}- // Tokenize each candidate once: title and text are one bag of words —- // a judge reads a document, not a schema.- let docs: Vec<Vec<String>> = candidates+ // Count tokens once, retaining frequencies only for query terms. CJK+ // expansion must not turn IDF into a scan of every document token for+ // every query gram. Length still counts all tokens, exactly as before.+ let qset: std::collections::HashSet<&str> = q_terms.iter().map(String::as_str).collect();+ let docs: Vec<(usize, HashMap<&str, u64>)> = candidates.iter().map(|c| {- let mut ts: Vec<String> = Vec::new();- if let Some(t) = &c.title {- ts.extend(tokens(t));+ let mut len = 0;+ let mut tf = HashMap::new();+ for token in c+ .title+ .iter()+ .flat_map(|t| tokens(t))+ .chain(tokens(&c.text))+ {+ len += 1;+ if let Some(&q) = qset.get(token.as_str()) {+ *tf.entry(q).or_insert(0) += 1;+ }}- ts.extend(tokens(&c.text));- ts+ (len, tf)}).collect();let n = docs.len() as f64;let avgdl = {- let total: usize = docs.iter().map(Vec::len).sum();+ let total: usize = docs.iter().map(|(len, _)| len).sum();(total as f64 / n).max(1.0)};@@ -266,26 +321,22 @@ pub fn lexical_scores(query: &str, candidates: &[Candidate]) -> Vec<Scored> {let idf: Vec<f64> = q_terms.iter().map(|q| {- let df = docs.iter().filter(|d| d.iter().any(|t| t == q)).count() as f64;+ let df = docs+ .iter()+ .filter(|(_, tf)| tf.contains_key(q.as_str()))+ .count() as f64;(1.0 + (n - df + 0.5) / (df + 0.5)).ln()}).collect();// The maximum attainable numerator: every query term at full saturation.let max_score: f64 = idf.iter().sum::<f64>() * (K1 + 1.0);- let qset: std::collections::HashSet<&str> = q_terms.iter().map(String::as_str).collect();candidates.iter().enumerate().map(|(i, c)| {- let d = &docs[i];- let norm = K1 * (1.0 - B + B * d.len() as f64 / avgdl);- let mut tf: HashMap<&str, u64> = HashMap::new();- for t in d {- if qset.contains(t.as_str()) {- *tf.entry(t.as_str()).or_insert(0) += 1;- }- }+ let (len, tf) = &docs[i];+ let norm = K1 * (1.0 - B + B * *len as f64 / avgdl);let numerator: f64 = idf.iter().zip(q_terms.iter())@@ -557,6 +608,112 @@ mod tests {// ── the lexical scorer ──────────────────────────────────────────────────+ #[test]+ fn cjk_query_matches_inside_unsegmented_text() {+ let out = lexical_scores(+ "雷同",+ &[+ cand(7, None, "该段内容存在雷同需要核查"),+ cand(2, None, "今天的天气晴朗适合出行"),+ ],+ );+ assert!(+ out[0].score > 0.0,+ "embedded Chinese term must match: {out:?}"+ );+ assert_eq!(out[1].score, 0.0);+ assert_eq!(out.iter().map(|s| s.ordinal).collect::<Vec<_>>(), [7, 2]);+ }++ #[test]+ fn cjk_single_character_and_adjacent_pair_have_evidence() {+ let candidates = [+ cand(0, Some("中文检索"), "雷同内容"),+ cand(1, None, "同雷内容"),+ ];+ let single = lexical_scores("雷", &candidates);+ assert!(single.iter().all(|s| s.score > 0.0), "{single:?}");+ let pair = lexical_scores("雷同", &candidates);+ assert!(+ pair[0].score > pair[1].score,+ "adjacency adds evidence: {pair:?}"+ );+ }++ #[test]+ fn legacy_non_cjk_score_receipt() {+ let candidates = [+ cand(9, Some("Trial results"), "vitamin d improved bone density"),+ cand(3, None, "vitamin rich vegetables"),+ cand(2, None, "other topic"),+ ];+ let out = lexical_scores("vitamin bone density", &candidates);+ assert_eq!(+ out.iter().map(|s| s.score.to_bits()).collect::<Vec<_>>(),+ [4599937497660334437, 4591716393592272481, 0]+ );+ }++ #[test]+ fn cjk_tokens_stop_at_separators_and_keep_non_cjk_words() {+ assert_eq!(+ tokens("API中文v2 CAFÉ İSTANBUL 123").collect::<Vec<_>>(),+ [+ "api",+ "中",+ "中文",+ "文",+ "v2",+ "café",+ "i\u{307}stanbul",+ "123"+ ]+ );+ for text in ["雷,同", "雷 同", "雷😀同", "雷A同"] {+ assert!(!tokens(text).any(|t| t == "雷同"), "{text}");+ }+ assert_eq!(+ tokens("𠀀中文").collect::<Vec<_>>(),+ ["𠀀", "𠀀中", "中", "中文", "文"]+ );+ assert_eq!(+ tokens("東京都 한국어").collect::<Vec<_>>(),+ ["東", "東京", "京", "京都", "都", "한", "한국", "국", "국어", "어"]+ );+ assert_eq!(+ tokens("alpha_beta can't naïve").collect::<Vec<_>>(),+ ["alpha", "beta", "can", "t", "naïve"]+ );+ }++ #[test]+ fn cjk_title_and_text_do_not_create_a_cross_field_pair() {+ let scores = lexical_scores("雷同", &[cand(4, Some("雷"), "同"), cand(8, None, "雷同")]);+ assert!(scores[1].score > scores[0].score, "{scores:?}");+ assert_eq!(+ scores,+ lexical_scores("雷同", &[cand(4, Some("雷"), "同"), cand(8, None, "雷同"),])+ );+ assert!(scores+ .iter()+ .all(|s| s.score.is_finite() && (0.0..=1.0).contains(&s.score)));+ }++ #[test]+ fn cjk_scoring_handles_the_query_and_document_character_limits() {+ let query: String = (0..crate::MAX_QUERY_CHARS)+ .map(|i| char::from_u32(0x4e00 + i as u32).unwrap())+ .collect();+ let text = format!("{query}{query}");+ assert_eq!(text.chars().count(), crate::MAX_DOC_CHARS);+ let candidates: Vec<_> = (0..crate::JEV_MAX_DOCS_PER_CALL)+ .map(|i| cand(i, None, &text))+ .collect();+ let scores = lexical_scores(&query, &candidates);+ assert_eq!(scores.len(), candidates.len());+ assert!(scores.iter().all(|s| s.score > 0.0 && s.score <= 1.0));+ }+#[test]fn no_query_words_scores_zero_everywhere() {let out = lexical_scores(diff --git a/tools/cjk-judge/VERIFICATION.md b/tools/cjk-judge/VERIFICATION.mdnew file mode 100644index 000000000..f48b997a2--- /dev/null+++ b/tools/cjk-judge/VERIFICATION.md@@ -0,0 +1,212 @@+# CJK lexical judge (#2580)++Written by an AI coding agent (OpenAI Codex), operated by @Jarvis636431.+macOS arm64, Rust 1.97.1, source-built xerj v1.0.0-rc.100.+Original verification: independent branch initially based on upstream `25b108084`, rebased onto+`8f62343c7`. The intervening commit changes only `.contributors`; the tested+Rust sources and runtime snapshot are unchanged. No stacked dependency.++## Main refresh after #2697 (2026-10-30)++Merge upstream `6a7b390ac` into the prior PR head `25c61738d`. The only+conflict is the CHANGELOG insertion point introduced by #2697; preserve its+32 MiB batch-response fix entry and the CJK entry. The previous maintainer+merge remains in the ancestry. The judge implementation, HTTP regression,+smoke harness and rerank documentation are byte-identical to the previous+PR head; the effective main diff is still the same six files.++Fresh reruns on this composition: default rerank 59 passed, `judge-local`+61 passed and HTTP judge 15 passed, all with zero failures, ignores or+filters. Scoped rerank release build, formatting, strict Clippy in both+configurations, Python syntax, whitespace and the workflow-timeout guard+passed. The isolated node/runner build passed with the existing autoindex+`resolve_reporting` dead-code warning. Original failing-before-fix evidence+and scope exclusions below remain applicable; no new behavior is added by+this conflict resolution.++Fresh private-node ES-YAML on this composition: **1,453 passed, 0 failed,+10 skipped across 233 files**, retaining 74 existing catch-lenient cases.+The same executable then passed HTTP/MCP order, CJK evidence, threshold+counts, engine totals and original `_score` before flush, after flush and+after graceful restart. Scores remained exact 0.5900621118012421, partial+0.24340586742297435 and unrelated 0.0. The harness, runner and node exited 0.+Snapshot SHA-256:+`71735dfe2699509fd66c6febc2e5e84b9b7414e795854d0c8a9a7add56dc9907`.+These are fresh local receipts; hosted CI for the pushed refresh remains+separate. Linux/Windows, trained-model quality and broad relevance were not+re-tested. No performance claim is made.++## Earlier main integration (2026-10-30)++Merge current upstream `befe92035` into the PR's remote head `e45809c31`,+preserving the maintainer-authored merge and its egress-inventory fix (#2639).+Resolve the sole conflict in `CHANGELOG.md` by retaining both the CJK entry+and every upstream entry. The judge implementation, HTTP regression, smoke+harness and `docs/RERANK.md` are byte-identical to the previous PR head.+The effective diff remains the same six files; no stacked prerequisite,+workflow change, upstream rollback or new product behavior is introduced.++Fresh commands on the composed tree, from `engine/`:++```sh+cargo test --locked --offline -p xerj-rerank+cargo test --locked --offline -p xerj-rerank --features judge-local+cargo build --locked --offline --release -j 32 -p xerj-rerank+cargo fmt --all --check+cargo clippy --locked --offline -p xerj-rerank --all-targets -- -D warnings+cargo clippy --locked --offline -p xerj-rerank --features judge-local --all-targets -- -D warnings+cargo build --locked --offline --profile ci-test -j 8 -p xerj-server -p es-yaml-runner+cargo test --locked --offline --profile ci-test -p xerj-api --test judge_stage_http+```++Default rerank: 59 passed; `judge-local`: 61 passed. Both have zero failures,+ignored tests or filters, including all four egress-inventory tests. The+old filter below is historical and is no longer required. Scoped release+build, formatting and strict Clippy in both configurations passed.+The workflow-timeout guard and Python syntax check also passed.++HTTP judge: 15 passed, zero failures, ignored tests or filters. The isolated+node/runner build passed with the existing autoindex `resolve_reporting`+dead-code warning. From repository root, the unchanged smoke command above+(`python3 tools/cjk-judge/smoke.py --binary engine/target/ci-test/xerj+--runner engine/target/ci-test/es-yaml-runner`) exited 0:++- ES-YAML: 1,453 passed, 0 failed, 10 skipped across 233 files; 74 existing+ catch-lenient cases. No suite, skip or catch-lenient rule changed in this PR.+- The same executable snapshot passed CJK HTTP/MCP order, threshold counts,+ engine totals and original `_score` checks before flush, after flush and+ after graceful restart. Scores were exact 0.5900621118012421, partial+ 0.24340586742297435, unrelated 0.0 in all three checks.+- Executable snapshot SHA-256:+ `c0a919e7c69e8a8c9418386e5703081ea3f3da19fc3380c0e0839760d5d69e5a`.++Audited the composed diff against `docs/CONTRIBUTION_REVIEW.md`: upstream+API/storage/workflow changes are intact, existing CJK/non-CJK assertions are+unchanged, and the original failing-before-fix receipt below remains the+regression evidence. All original scope limits and non-applicable durable+state, migration, accelerator and distributed-write sections still apply.+Linux/Windows and broad relevance/model quality were not re-tested. Hosted+CI on the pushed head is separate from these local results.++## Scope and evidence before implementation++The old lexical judge splits only at non-alphanumeric characters. The query+`雷同` therefore has no token in common with `该段内容存在雷同需要核查`.+Both new CJK score tests were run against that implementation and failed:+the containing document and the unrelated document both scored 0.0.+The tests now pass with the same assertions.++Reference approach, retrieved before implementation: Tantivy's MIT-licensed+[`src/tokenizer/ngram_tokenizer.rs`](https://github.com/quickwit-oss/tantivy/blob/main/src/tokenizer/ngram_tokenizer.rs),+lines 115–119 (all inner grams), 154–164 (codepoint-bounded slices), 253–280+(UTF-8 frontiers). Its [LICENSE](https://github.com/quickwit-oss/tantivy/blob/main/LICENSE)+was checked. No implementation was copied. XERJ's existing+`xerj-fts/src/analyzer.rs:799–862` also isolates CJK runs; its bigram-only+policy is not changed by this patch.++The new judge emits characters and adjacent pairs within CJK runs. Non-CJK+alphanumeric runs keep their old lowercase tokens. Separators and the+title/body boundary do not synthesize pairs. No dictionary, normalization,+model, calibrated confidence or first-stage analyzer change is introduced.+Single-character overlap can contribute even when the whole query phrase is+absent; adjacency is additional evidence, not a phrase-match guarantee.++## Local tests and builds++Run from `engine/`:++```sh+cargo test --locked --offline -p xerj-rerank --lib judge::tests -- --nocapture+cargo test --locked --offline -p xerj-rerank -- --skip every_outbound_client_in_the_engine_is_on_the_published_list+cargo test --locked --offline -p xerj-rerank --features judge-local -- --skip every_outbound_client_in_the_engine_is_on_the_published_list+cargo test --locked --offline --profile ci-test -p xerj-api --test judge_stage_http+cargo fmt --all --check+cargo build --locked --offline --release -j 32 -p xerj-rerank+cargo clippy --locked --offline -p xerj-rerank --all-targets -- -D warnings+cargo clippy --locked --offline -p xerj-rerank --features judge-local --all-targets -- -D warnings+```++Observed: default rerank 58 passed / 0 failed / 1 explicit filter;+`judge-local` rerank 60 passed / 0 failed / the same filter; HTTP judge 15+passed / 0 failed. Release build, formatting and both strict Clippy variants+passed. The HTTP suite's first sandboxed run passed 13 tests and failed the+two tests needing a loopback listener with PermissionDenied; the full suite+then passed outside the sandbox.++The unfiltered rerank suite fails its unchanged egress inventory check:+`new outbound network client(s) in ["xerj-mcp/src/status.rs"]`. The same+unchanged test target was compiled/run on a clean `25b108084` worktree and+failed with the identical path (3 passed / 1 failed). No egress table,+assertion, committed skip or unrelated client is altered here.++Tests cover the embedded Chinese term, one-character evidence, adjacency,+punctuation/emoji/Latin boundaries, supplementary Han, Japanese/Hangul+syllables, title/body separation, determinism and finite bounded scores at+the existing 4,000-query/8,000-document character ceilings for 30 candidates.+One non-CJK golden fixture records the old score bits+`[4599937497660334437, 4591716393592272481, 0]`; the candidate matches them+exactly. The query-term frequency table counts every emitted token for length+normalization and keeps fixed summation order, without rescanning document+tokens for each new gram.+No latency, RSS, throughput or general retrieval-quality claim is made.++## Real private-node control and candidate++From repository root, `smoke.py --binary <path>` snapshots the executable,+starts a temporary lexical-mode node and seeds three synthetic documents.+It uses `match_all` to hold the candidate set fixed, judges `雷同`, checks+threshold counts/engine totals and original `_score`, compares HTTP/MCP,+flushes, and gracefully restarts from the same private data directory.+Only this temporary node uses a 99% disk flood watermark on the shared+laptop. Hosted rerank is disabled; no model, personal corpus or existing node+is used. No index format or reindex/migration is involved.++The original control snapshot from prior source base `7d3ca96b1` is+`bf8e611d814f935e5a313cb735f33fb3d7ff532b78c04a5fe3af87eb020f3853`.+That base and current `25b108084` have the identical original judge blob+`fd51419b804a9240fb2248256671851a7a480ee1`. The control is behavior evidence,+not a matched-build performance comparison. With `--expect-zero`, all three+documents score 0.0 before flush, after flush and after restart; a positive+threshold drops them all.++The candidate snapshot is+`9bef44e59d43b3403f15364452c3e8c7e67f6bdc5dcce4d40498d293754c0c87`.+Observed scores in this fixed page: exact-containing document+0.5900621118012421, partial single-character evidence 0.24340586742297435,+unrelated document 0.0. All HTTP/MCP, pruning/count, flush and restart checks+passed. These are lexical fixture scores, not probabilities calibrated+against human relevance judgments.++The same candidate executable passed the full ES-YAML gate on a fresh private+node: **1,451 passed, 0 failed, 10 skipped across 232 files**, with the+existing 75 catch-lenient cases. Runner and node exited 0. The subsequent+HTTP/MCP, threshold, flush and restart checks repeated the same scores.++```sh+# Run the scoped builds from engine/ before this repository-root command:+# cargo build --locked --offline --profile ci-test -j 8 -p xerj-server -p es-yaml-runner+python3 tools/cjk-judge/smoke.py \+ --binary engine/target/ci-test/xerj \+ --runner engine/target/ci-test/es-yaml-runner+```++## Contribution review and exclusions++Audited against `docs/CONTRIBUTION_REVIEW.md`: preserve opt-in behavior,+non-CJK word tokens, deterministic arithmetic, input ordinals, scorer name,+original engine scores, count semantics, egress policy and model path. The+HTTP test uses the real router/storage before and after flush. The private+binary harness adds MCP transport and durable restart coverage.++No dependency, workflow, stored schema, journal, migration, provider/model+execution, feature-gate or installation/deployment change. Durable-state+failure atomicity, optional accelerator fallback and distributed replication+are not applicable. The ordinary first-stage search path is unchanged.++Not tested: the original six-file Windows corpus, broad CJK relevance,+cross-language threshold calibration, dictionary segmentation, half-width+kana/jamo normalization, or Linux/Windows execution. In particular, first-stage+candidate misses and rank order without `judge` in #2580 remain separate.+The existing model scorer/tokenizer is unchanged; feature tests use its tiny+synthetic fixture, not a real trained-model quality evaluation. Hosted CI+remains separate from local receipts.diff --git a/tools/cjk-judge/smoke.py b/tools/cjk-judge/smoke.pynew file mode 100644index 000000000..6b384f19a--- /dev/null+++ b/tools/cjk-judge/smoke.py@@ -0,0 +1,142 @@+#!/usr/bin/env python3+"""Check #2580 on a private node, holding first-stage candidates fixed.++Accept --expect-zero for an original lexical-judge binary, or --runner for a+fresh-node compatibility gate before the fixture. No model or personal data.+"""+import argparse+import hashlib+import json+import os+from pathlib import Path+import shutil+import signal+import socket+import subprocess+import tempfile+import time+from urllib.request import build_opener, ProxyHandler, Request+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument('--binary', type=Path, required=True)+ parser.add_argument('--expect-zero', action='store_true')+ parser.add_argument('--runner', type=Path)+ args = parser.parse_args()+ original = args.binary.resolve(strict=True)+ runner = args.runner.resolve(strict=True) if args.runner else None+ repo = Path(__file__).resolve().parents[2]+ env = {k: v for k, v in os.environ.items() if not k.startswith('XERJ_')+ and k.lower() not in ('http_proxy', 'https_proxy', 'all_proxy', 'no_proxy')}+ env.update(XERJ_DISABLE_FEEDBACK='true', XERJ_NUM_CPUS='4')+ opener = build_opener(ProxyHandler({}))+ with tempfile.TemporaryDirectory(prefix='xerj-cjk-judge-') as temp:+ root = Path(temp)+ binary = root / ('xerj.exe' if os.name == 'nt' else 'xerj')+ shutil.copyfile(original, binary)+ binary.chmod(0o700)+ print('Executable snapshot SHA-256:', hashlib.sha256(binary.read_bytes()).hexdigest(), flush=True)+ config = root / 'node.toml'+ # Only this tiny temporary fixture changes its volume reserve.+ config.write_text('[server]\nauto_port = false\n\n[limits]\ndisk_flood_stage_percent = 99\n\n[rerank]\nenabled = false\n')+ env['XERJ_CONFIG'] = str(config)+ with socket.socket() as sock:+ sock.bind(('127.0.0.1', 0))+ port = sock.getsockname()[1]+ assert port <= 65533+ url = f'http://127.0.0.1:{port}'++ def run(command, stdin=None, timeout=120):+ result = subprocess.run([str(x) for x in command], input=stdin, capture_output=True,+ text=True, cwd=root, env=env, timeout=timeout)+ assert result.returncode == 0, (command, result.returncode, result.stdout, result.stderr)+ return result.stdout++ def request(method, path, body=None):+ data = json.dumps(body).encode() if body is not None else None+ req = Request(url + path, data=data, method=method, headers={'Content-Type': 'application/json'})+ with opener.open(req, timeout=30) as response:+ return json.load(response)++ def query(judge=None):+ body = {'query': {'match_all': {}}, 'size': 3}+ if judge is not None:+ body['judge'] = judge+ return request('POST', '/cjkjudge-docs/_search', body)++ def check():+ base = query()+ assert 'judged' not in base+ assert all('_p_relevant' not in h for h in base['hits']['hits'])+ judged = query({'query': '雷同'})+ hits = judged['hits']['hits']+ assert len(hits) == 3 and judged['judged']['scorer'] == 'lexical', judged+ scores = {h['_id']: h['_p_relevant'] for h in hits}+ print('Observed lexical scores:', json.dumps(scores, ensure_ascii=False), flush=True)+ assert {h['_id']: h['_score'] for h in hits} == {h['_id']: h['_score'] for h in base['hits']['hits']}+ if args.expect_zero:+ assert all(p == 0.0 for p in scores.values()), scores+ assert not query({'query': '雷同', 'min_p': 0.01})['hits']['hits']+ return+ assert scores['exact'] > scores['partial'] > 0.0 == scores['unrelated'], judged+ assert hits[0]['_id'] == 'exact', judged+ cut = (scores['exact'] + scores['partial']) / 2+ kept = query({'query': '雷同', 'min_p': cut})+ assert [h['_id'] for h in kept['hits']['hits']] == ['exact'], kept+ assert kept['hits']['total'] == base['hits']['total']+ assert [kept['judged'][k] for k in ['judged', 'kept', 'dropped']] == [3, 1, 2], kept+ messages = [+ {'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': {}},+ {'jsonrpc': '2.0', 'id': 2, 'method': 'tools/call', 'params': {+ 'name': 'xerj_search', 'arguments': {'index': 'cjkjudge-docs',+ 'query_dsl': {'match_all': {}}, 'size': 3, 'judge': {'query': '雷同'}}}},+ ]+ raw = run([binary, 'mcp', '--url', url], ''.join(json.dumps(m) + '\n' for m in messages))+ reply = next(json.loads(line)['result'] for line in raw.splitlines() if json.loads(line).get('id') == 2)+ assert not reply.get('isError'), reply+ mcp = json.loads(reply['content'][0]['text'])+ assert [(h['_id'], h['_p_relevant']) for h in mcp['hits']['hits']] == [(h['_id'], h['_p_relevant']) for h in hits], mcp+ print('PASS HTTP/MCP order, CJK evidence, threshold counts, total and original _score', flush=True)++ print(run([binary, '--version']), end='', flush=True)+ for phase in ['first-run', 'restart']:+ with (root / f'{phase}.log').open('w') as log:+ node = subprocess.Popen([str(binary), '--insecure', '--port', str(port),+ '--data-dir', str(root / 'data'), '--embed-mode', 'lexical'],+ cwd=root, env=env, stdout=log, stderr=log)+ try:+ deadline = time.monotonic() + 60+ while True:+ assert node.poll() is None, (root / f'{phase}.log').read_text()+ try:+ request('GET', '/')+ break+ except OSError:+ if time.monotonic() >= deadline:+ raise TimeoutError('node readiness')+ time.sleep(0.1)+ if phase == 'first-run':+ if runner:+ print(run([runner, '--url', url, '--dir', repo / 'engine/tests/es-compat-yaml/yaml'], timeout=900), flush=True)+ request('PUT', '/cjkjudge-docs', {'mappings': {'properties': {'body': {'type': 'text'}}}})+ for id_, body in [('exact', '这里发现雷同内容雷同内容'), ('partial', '部分雷声同时出现'), ('unrelated', '今天阳光明媚风景宜人')]:+ request('PUT', f'/cjkjudge-docs/_doc/{id_}?refresh=true', {'body': body})+ check()+ request('POST', '/cjkjudge-docs/_flush', {})+ check()+ finally:+ if node.poll() is None:+ node.send_signal(signal.SIGTERM)+ try:+ node.wait(timeout=30)+ except subprocess.TimeoutExpired:+ node.kill()+ node.wait()+ raise+ assert node.returncode == 0, (root / f'{phase}.log').read_text()+ print('PASS buffered, flushed and restarted fixture' + (' (original zero-score control)' if args.expect_zero else ''), flush=True)+++if __name__ == '__main__':+ main()