Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(xerj-code): preserve match scores in passage windows

Amakurai80e2e5059c65
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e16214a95..4048e9e65 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -96,6 +96,9 @@ jobs:
- name: Corpus manifest round-trip
run: tools/xerj-code/tests/test_xc_corpus.sh
+ - name: Matching passage window regressions
+ run: python3 tools/xerj-code/tests/test_passage_window.py
+
- name: Vetted hub manifests validate
run: python3 tools/xerj-code/tests/validate_manifest.py --hub tools/xerj-code/hub/*.json
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 16a4a91c7..e31eb1b20 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
exposes the same two parameters. Raised by @Vinz2168 from a shared-memory
agent integration where neither single mode was enough.
+### Fixed
+
+- **Reference-code passage windows preserve their match score.** `xc.py`
+ could select relevant source and then discard its matches while aligning
+ the excerpt to line boundaries, including on long source lines. Line
+ alignment now keeps the selected term score or falls back to the bounded
+ original window. Window scoring also uses original-source offsets when
+ Unicode lowercasing expands characters. This affects the fallback when no
+ matching symbol is available and explicit `--no-symbol` output.
+
## [1.0.0-rc.74] - 2026-09-08
### Added
diff --git a/tools/xerj-code/README.md b/tools/xerj-code/README.md
index c785fe981..22e9cb966 100644
--- a/tools/xerj-code/README.md
+++ b/tools/xerj-code/README.md
@@ -88,8 +88,19 @@ already knows, it is overhead. `SKILL.md` and
[`docs/case-studies/reference-coding/`](../../docs/case-studies/reference-coding/)
carry the measured numbers, including the cases where this loses.
+## Passage windows
+
+When no matching symbol is available, or with `--no-symbol`, `xc.py` selects a
+window of source around query terms, capped at 800 characters by default.
+It aligns the window to whole lines only when doing so preserves
+the selected window's term score; a partial line is preferable to dropping the
+match. Window offsets refer to the original source, including when Unicode
+lowercasing changes its length. `--full N` changes the character limit, and
+`--full 0` keeps the explicit file-head mode.
+
## Tests
```sh
tools/xerj-code/tests/test_xc_corpus.sh # offline; local git fixtures over file://
+python3 tools/xerj-code/tests/test_passage_window.py # offline passage extraction regressions
```
diff --git a/tools/xerj-code/scripts/xc.py b/tools/xerj-code/scripts/xc.py
index 645b4afb0..85b1e5b8f 100755
--- a/tools/xerj-code/scripts/xc.py
+++ b/tools/xerj-code/scripts/xc.py
@@ -470,7 +470,7 @@ def symbol_passage(body, src, query, width):
def best_window(body, query, width):
- """The `width`-char slice of `body` densest in query terms, snapped to lines.
+ """The densest sampled `width`-char slice, snapped to lines when safe.
Taking the HEAD of the file instead is the single worst bug this tool had.
Measured on a real valkey+memcached corpus: retrieval ranked the correct
@@ -492,23 +492,29 @@ def best_window(body, query, width):
terms = [t.lower() for t in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}", query)]
if not terms:
return body[:width], 0, total
- low = body.lower()
# Score every candidate start on a coarse stride: dense term hits win. The
# stride keeps this linear-ish on multi-hundred-KB sources.
stride = max(1, width // 8)
best_start, best_score = 0, -1
for start in range(0, total - width + stride, stride):
- chunk = low[start:start + width]
+ # Slice before lowercasing: Unicode lowercase can expand a character,
+ # so offsets in a lowercased copy need not be offsets in the source.
+ chunk = body[start:start + width].lower()
score = sum(chunk.count(t) for t in terms)
if score > best_score:
best_start, best_score = start, score
if best_score <= 0:
return body[:width], 0, total
- # Snap to line boundaries so the excerpt is readable code, not a torn line.
+ # Prefer complete lines, but never discard query evidence just to align
+ # them. The final partial line may hold the match, or the previous newline
+ # may be far away in a source line longer than the requested width.
nl = body.rfind("\n", 0, best_start)
start = nl + 1 if nl != -1 else best_start
end = body.rfind("\n", start, start + width)
end = end if end > start else min(total, start + width)
+ snapped = body[start:end].lower()
+ if sum(snapped.count(t) for t in terms) < best_score:
+ return body[best_start:best_start + width], best_start, total
return body[start:end], start, total
diff --git a/tools/xerj-code/tests/test_passage_window.py b/tools/xerj-code/tests/test_passage_window.py
new file mode 100644
index 000000000..2a1c482e3
--- /dev/null
+++ b/tools/xerj-code/tests/test_passage_window.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""Offline regressions for preserving query evidence in xc.py excerpts."""
+import importlib.util
+import io
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from contextlib import redirect_stderr, redirect_stdout
+from unittest.mock import patch
+
+
+XC_PATH = Path(__file__).resolve().parents[1] / "scripts" / "xc.py"
+SPEC = importlib.util.spec_from_file_location("xc_passage_under_test", XC_PATH)
+xc = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(xc)
+
+
+def tail_match_body():
+ # The only match fits in the first 800 characters, but its newline does not.
+ return ("padding 123456789012\n" * 37
+ + "needle(); more_source_text_here();\n"
+ + "// trailing text\n" * 30)
+
+
+class PassageWindowTests(unittest.TestCase):
+ def window(self, body, query="needle", width=800):
+ text, start, total = xc.best_window(body, query, width)
+ self.assertEqual(total, len(body))
+ self.assertGreaterEqual(start, 0)
+ self.assertLessEqual(len(text), width)
+ self.assertEqual(text, body[start:start + len(text)])
+ return text, start
+
+ def test_end_snap_keeps_the_matching_partial_line(self):
+ text, _ = self.window(tail_match_body())
+ self.assertIn("needle", text)
+
+ def test_start_snap_does_not_jump_back_over_a_long_line(self):
+ body = "// header\n" + "x" * 1500 + "needle" + "x" * 500
+ text, _ = self.window(body)
+ self.assertIn("needle", text)
+
+ def test_alignment_preserves_all_selected_query_occurrences(self):
+ body = ("needle\n" + "padding 123456789012\n" * 36
+ + "padding abcde\nneedle(); more_source_text_here();\n"
+ + "// trailing text\n" * 30)
+ self.assertEqual(body[:800].count("needle"), 2)
+ text, _ = self.window(body)
+ self.assertEqual(text.count("needle"), 2)
+
+ def test_lowercase_expansion_keeps_source_offsets(self):
+ # U+0130 lowercases to two code points; source offsets must not shift.
+ body = "\u0130" * 900 + "needle" + "x" * 1500
+ text, _ = self.window(body)
+ self.assertIn("needle", text)
+
+ def test_readable_line_boundaries_when_the_match_is_preserved(self):
+ body = "// padding line\n" * 70 + "needle();\n" + "// tail\n" * 100
+ text, start = self.window(body)
+ self.assertIn("needle", text)
+ self.assertTrue(start == 0 or body[start - 1] == "\n")
+ self.assertTrue(start + len(text) == len(body)
+ or body[start + len(text)] == "\n")
+
+ def test_equal_scores_keep_the_first_candidate(self):
+ body = "needle" + "x" * 1600 + "needle" + "x" * 1000
+ text, start = self.window(body)
+ self.assertEqual(start, 0)
+ self.assertEqual(text, body[:800])
+
+ def test_empty_and_short_bodies_are_returned_unchanged(self):
+ for body in ("", "needle();\n", "x" * 800):
+ with self.subTest(length=len(body)):
+ self.assertEqual(self.window(body), (body, 0))
+
+ def test_no_terms_and_no_matches_keep_the_head_fallback(self):
+ body = "// plain source\n" * 100
+ for query in ("", "$-1", "needle"):
+ with self.subTest(query=query):
+ self.assertEqual(self.window(body, query), (body[:800], 0))
+
+ def test_custom_width_preserves_a_match_within_the_budget(self):
+ body = "padding\n" * 3 + "needle(); more_source();\n" + "tail\n" * 30
+ text, _ = self.window(body, width=32)
+ self.assertIn("needle", text)
+
+ def test_cli_emits_the_matching_source_with_and_without_symbols(self):
+ body = tail_match_body()
+ # Exercise the real argument parsing, HTTP request handling, and output
+ # formatter without a live XERJ or changes to the user's corpus state.
+ with tempfile.TemporaryDirectory(prefix="xc-passage-test-") as home:
+ state = Path(home) / "state"
+ state.mkdir()
+ (state / "fixture.json").write_text(json.dumps({
+ "corpus": "fixture", "prefix": "xc-fixture",
+ }), encoding="utf-8")
+ for extra in ([], ["--no-symbol"]):
+ with self.subTest(extra=extra):
+ source = {"path": "acme/source.txt", "body": body}
+ if extra:
+ source["symbols"] = [
+ {"name": "needle", "kind": "fn", "line": 38},
+ ]
+
+ def urlopen(req, timeout=None):
+ if "/_cat/indices/" in req.full_url:
+ payload = [{"index": "xc-fixture-1"}]
+ elif req.full_url.endswith("/_mapping"):
+ payload = {"xc-fixture-1": {"mappings": {
+ "properties": {"body": {"type": "text"}},
+ }}}
+ elif req.full_url.endswith("/_search"):
+ payload = {"hits": {"hits": [{
+ "_source": source, "_score": 1.0,
+ }]}}
+ else:
+ self.fail(f"unexpected request: {req.full_url}")
+ return io.BytesIO(json.dumps(payload).encode())
+
+ out, err = io.StringIO(), io.StringIO()
+ with (patch.object(xc, "ROOT", home),
+ patch.dict(xc._FIELDS_CACHE, {}, clear=True),
+ patch.dict(os.environ, {"XERJ_CODE_NO_NUDGE": "1"}),
+ patch.object(xc.urllib.request, "urlopen", urlopen),
+ patch.object(sys, "argv", ["xc.py", "fixture", "needle"] + extra),
+ redirect_stdout(out), redirect_stderr(err)):
+ xc.main()
+ self.assertIn("needle", out.getvalue())
+ self.assertIn("1 passages from 'fixture'", out.getvalue())
+ self.assertEqual(err.getvalue(), "")
+
+
+if __name__ == "__main__":
+ unittest.main()