AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(xerj-code): preserve match scores in passage windows
Amakurai80e2e5059c65
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex e16214a95..4048e9e65 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -96,6 +96,9 @@ jobs:- name: Corpus manifest round-triprun: tools/xerj-code/tests/test_xc_corpus.sh+ - name: Matching passage window regressions+ run: python3 tools/xerj-code/tests/test_passage_window.py+- name: Vetted hub manifests validaterun: python3 tools/xerj-code/tests/validate_manifest.py --hub tools/xerj-code/hub/*.jsondiff --git a/CHANGELOG.md b/CHANGELOG.mdindex 16a4a91c7..e31eb1b20 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -26,6 +26,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0exposes the same two parameters. Raised by @Vinz2168 from a shared-memoryagent integration where neither single mode was enough.+### Fixed++- **Reference-code passage windows preserve their match score.** `xc.py`+ could select relevant source and then discard its matches while aligning+ the excerpt to line boundaries, including on long source lines. Line+ alignment now keeps the selected term score or falls back to the bounded+ original window. Window scoring also uses original-source offsets when+ Unicode lowercasing expands characters. This affects the fallback when no+ matching symbol is available and explicit `--no-symbol` output.+## [1.0.0-rc.74] - 2026-09-08### Addeddiff --git a/tools/xerj-code/README.md b/tools/xerj-code/README.mdindex c785fe981..22e9cb966 100644--- a/tools/xerj-code/README.md+++ b/tools/xerj-code/README.md@@ -88,8 +88,19 @@ already knows, it is overhead. `SKILL.md` and[`docs/case-studies/reference-coding/`](../../docs/case-studies/reference-coding/)carry the measured numbers, including the cases where this loses.+## Passage windows++When no matching symbol is available, or with `--no-symbol`, `xc.py` selects a+window of source around query terms, capped at 800 characters by default.+It aligns the window to whole lines only when doing so preserves+the selected window's term score; a partial line is preferable to dropping the+match. Window offsets refer to the original source, including when Unicode+lowercasing changes its length. `--full N` changes the character limit, and+`--full 0` keeps the explicit file-head mode.+## Tests```shtools/xerj-code/tests/test_xc_corpus.sh # offline; local git fixtures over file://+python3 tools/xerj-code/tests/test_passage_window.py # offline passage extraction regressions```diff --git a/tools/xerj-code/scripts/xc.py b/tools/xerj-code/scripts/xc.pyindex 645b4afb0..85b1e5b8f 100755--- a/tools/xerj-code/scripts/xc.py+++ b/tools/xerj-code/scripts/xc.py@@ -470,7 +470,7 @@ def symbol_passage(body, src, query, width):def best_window(body, query, width):- """The `width`-char slice of `body` densest in query terms, snapped to lines.+ """The densest sampled `width`-char slice, snapped to lines when safe.Taking the HEAD of the file instead is the single worst bug this tool had.Measured on a real valkey+memcached corpus: retrieval ranked the correct@@ -492,23 +492,29 @@ def best_window(body, query, width):terms = [t.lower() for t in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}", query)]if not terms:return body[:width], 0, total- low = body.lower()# Score every candidate start on a coarse stride: dense term hits win. The# stride keeps this linear-ish on multi-hundred-KB sources.stride = max(1, width // 8)best_start, best_score = 0, -1for start in range(0, total - width + stride, stride):- chunk = low[start:start + width]+ # Slice before lowercasing: Unicode lowercase can expand a character,+ # so offsets in a lowercased copy need not be offsets in the source.+ chunk = body[start:start + width].lower()score = sum(chunk.count(t) for t in terms)if score > best_score:best_start, best_score = start, scoreif best_score <= 0:return body[:width], 0, total- # Snap to line boundaries so the excerpt is readable code, not a torn line.+ # Prefer complete lines, but never discard query evidence just to align+ # them. The final partial line may hold the match, or the previous newline+ # may be far away in a source line longer than the requested width.nl = body.rfind("\n", 0, best_start)start = nl + 1 if nl != -1 else best_startend = body.rfind("\n", start, start + width)end = end if end > start else min(total, start + width)+ snapped = body[start:end].lower()+ if sum(snapped.count(t) for t in terms) < best_score:+ return body[best_start:best_start + width], best_start, totalreturn body[start:end], start, totaldiff --git a/tools/xerj-code/tests/test_passage_window.py b/tools/xerj-code/tests/test_passage_window.pynew file mode 100644index 000000000..2a1c482e3--- /dev/null+++ b/tools/xerj-code/tests/test_passage_window.py@@ -0,0 +1,137 @@+#!/usr/bin/env python3+"""Offline regressions for preserving query evidence in xc.py excerpts."""+import importlib.util+import io+import json+import os+from pathlib import Path+import sys+import tempfile+import unittest+from contextlib import redirect_stderr, redirect_stdout+from unittest.mock import patch+++XC_PATH = Path(__file__).resolve().parents[1] / "scripts" / "xc.py"+SPEC = importlib.util.spec_from_file_location("xc_passage_under_test", XC_PATH)+xc = importlib.util.module_from_spec(SPEC)+SPEC.loader.exec_module(xc)+++def tail_match_body():+ # The only match fits in the first 800 characters, but its newline does not.+ return ("padding 123456789012\n" * 37+ + "needle(); more_source_text_here();\n"+ + "// trailing text\n" * 30)+++class PassageWindowTests(unittest.TestCase):+ def window(self, body, query="needle", width=800):+ text, start, total = xc.best_window(body, query, width)+ self.assertEqual(total, len(body))+ self.assertGreaterEqual(start, 0)+ self.assertLessEqual(len(text), width)+ self.assertEqual(text, body[start:start + len(text)])+ return text, start++ def test_end_snap_keeps_the_matching_partial_line(self):+ text, _ = self.window(tail_match_body())+ self.assertIn("needle", text)++ def test_start_snap_does_not_jump_back_over_a_long_line(self):+ body = "// header\n" + "x" * 1500 + "needle" + "x" * 500+ text, _ = self.window(body)+ self.assertIn("needle", text)++ def test_alignment_preserves_all_selected_query_occurrences(self):+ body = ("needle\n" + "padding 123456789012\n" * 36+ + "padding abcde\nneedle(); more_source_text_here();\n"+ + "// trailing text\n" * 30)+ self.assertEqual(body[:800].count("needle"), 2)+ text, _ = self.window(body)+ self.assertEqual(text.count("needle"), 2)++ def test_lowercase_expansion_keeps_source_offsets(self):+ # U+0130 lowercases to two code points; source offsets must not shift.+ body = "\u0130" * 900 + "needle" + "x" * 1500+ text, _ = self.window(body)+ self.assertIn("needle", text)++ def test_readable_line_boundaries_when_the_match_is_preserved(self):+ body = "// padding line\n" * 70 + "needle();\n" + "// tail\n" * 100+ text, start = self.window(body)+ self.assertIn("needle", text)+ self.assertTrue(start == 0 or body[start - 1] == "\n")+ self.assertTrue(start + len(text) == len(body)+ or body[start + len(text)] == "\n")++ def test_equal_scores_keep_the_first_candidate(self):+ body = "needle" + "x" * 1600 + "needle" + "x" * 1000+ text, start = self.window(body)+ self.assertEqual(start, 0)+ self.assertEqual(text, body[:800])++ def test_empty_and_short_bodies_are_returned_unchanged(self):+ for body in ("", "needle();\n", "x" * 800):+ with self.subTest(length=len(body)):+ self.assertEqual(self.window(body), (body, 0))++ def test_no_terms_and_no_matches_keep_the_head_fallback(self):+ body = "// plain source\n" * 100+ for query in ("", "$-1", "needle"):+ with self.subTest(query=query):+ self.assertEqual(self.window(body, query), (body[:800], 0))++ def test_custom_width_preserves_a_match_within_the_budget(self):+ body = "padding\n" * 3 + "needle(); more_source();\n" + "tail\n" * 30+ text, _ = self.window(body, width=32)+ self.assertIn("needle", text)++ def test_cli_emits_the_matching_source_with_and_without_symbols(self):+ body = tail_match_body()+ # Exercise the real argument parsing, HTTP request handling, and output+ # formatter without a live XERJ or changes to the user's corpus state.+ with tempfile.TemporaryDirectory(prefix="xc-passage-test-") as home:+ state = Path(home) / "state"+ state.mkdir()+ (state / "fixture.json").write_text(json.dumps({+ "corpus": "fixture", "prefix": "xc-fixture",+ }), encoding="utf-8")+ for extra in ([], ["--no-symbol"]):+ with self.subTest(extra=extra):+ source = {"path": "acme/source.txt", "body": body}+ if extra:+ source["symbols"] = [+ {"name": "needle", "kind": "fn", "line": 38},+ ]++ def urlopen(req, timeout=None):+ if "/_cat/indices/" in req.full_url:+ payload = [{"index": "xc-fixture-1"}]+ elif req.full_url.endswith("/_mapping"):+ payload = {"xc-fixture-1": {"mappings": {+ "properties": {"body": {"type": "text"}},+ }}}+ elif req.full_url.endswith("/_search"):+ payload = {"hits": {"hits": [{+ "_source": source, "_score": 1.0,+ }]}}+ else:+ self.fail(f"unexpected request: {req.full_url}")+ return io.BytesIO(json.dumps(payload).encode())++ out, err = io.StringIO(), io.StringIO()+ with (patch.object(xc, "ROOT", home),+ patch.dict(xc._FIELDS_CACHE, {}, clear=True),+ patch.dict(os.environ, {"XERJ_CODE_NO_NUDGE": "1"}),+ patch.object(xc.urllib.request, "urlopen", urlopen),+ patch.object(sys, "argv", ["xc.py", "fixture", "needle"] + extra),+ redirect_stdout(out), redirect_stderr(err)):+ xc.main()+ self.assertIn("needle", out.getvalue())+ self.assertIn("1 passages from 'fixture'", out.getvalue())+ self.assertEqual(err.getvalue(), "")+++if __name__ == "__main__":+ unittest.main()