Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(xc): make the documented reference-coding loop work end to end

xerj-orgcfba3389551f
diff --git a/tools/xerj-code/scripts/xc-index.sh b/tools/xerj-code/scripts/xc-index.sh
index b667d9ebb..1cddf516b 100755
--- a/tools/xerj-code/scripts/xc-index.sh
+++ b/tools/xerj-code/scripts/xc-index.sh
@@ -38,6 +38,14 @@ curl -fsS -m 5 "$URL/_cluster/health" >/dev/null 2>&1 || {
echo "indexing corpus '$name' from $dir"
+# Recorded before the run so a failure can tell "this run wrote records" apart
+# from "an earlier run's records are still lying around". Salvaging the latter
+# would date stale data to now, which is the failure mode SKILL.md calls worse
+# than no index at all.
+docs_before="$(curl -fsS -m 5 "$URL/xc-$name*/_count" 2>/dev/null \
+ | sed 's/.*"count":\([0-9]*\).*/\1/' || echo "")"
+[ -n "$docs_before" ] || docs_before=0
+
# --no-graph: reference code needs ranked passages, not a relationship map. The
# graph detectors cost real time on a large tree and nothing here consumes edges.
set +e
@@ -49,20 +57,58 @@ set +e
rc=$?
set -e
+# How many records actually landed. This is the ground truth about whether the
+# corpus is usable, and it is the check SKILL.md already tells people to run by
+# hand ("Always verify: curl -s "$URL/xc-<corpus>*/_count" must be > 0").
+count_records() {
+ curl -fsS -m 5 "$URL/xc-$name*/_count" 2>/dev/null \
+ | sed 's/.*"count":\([0-9]*\).*/\1/' || echo ""
+}
+
# Exit 3 is "completed, some files were unparseable" — expected on any real
# repository (binaries, fixtures, vendored blobs). Treat it as success.
+#
+# Any other non-zero code is NOT taken at face value, because autoindex can
+# abort in finalisation *after* every document has been written — a whole-corpus
+# read-back or generation-cutover reconciliation failure leaves a complete,
+# queryable index behind and still exits 1 (see issue #367; observed on
+# apache/lucene on both embedding modes, 6,032 java records live). Discarding
+# that corpus sends the user back to a step that will fail again, so ask the
+# engine what is actually there before deciding. An empty or unreachable index
+# still fails, and a salvaged run is reported loudly rather than silently.
+docs=""
+salvaged=false
case $rc in
0) echo "indexed cleanly" ;;
3) echo "indexed (exit 3: some files skipped as junk — normal for real repos)" ;;
- *) echo "xc-index: autoindex failed with exit $rc" >&2; exit $rc ;;
+ *)
+ docs="$(count_records)"
+ if [ -n "$docs" ] && [ "$docs" -gt "$docs_before" ] 2>/dev/null; then
+ salvaged=true
+ echo "xc-index: WARNING — autoindex exited $rc, but this run wrote records" >&2
+ echo "xc-index: ($docs_before -> $docs). The corpus is queryable and is being" >&2
+ echo "xc-index: recorded as indexed, with autoindex_exit=$rc in its state file." >&2
+ echo "xc-index: That exit usually means finalisation failed AFTER the writes." >&2
+ echo "xc-index: Coverage is not guaranteed — please report the error above." >&2
+ elif [ -n "$docs" ] && [ "$docs" -gt 0 ] 2>/dev/null; then
+ echo "xc-index: autoindex failed with exit $rc and wrote no new records" >&2
+ echo "xc-index: ($docs already present from an earlier run — not treating" >&2
+ echo "xc-index: those as this run's output; a stale corpus dated to now is" >&2
+ echo "xc-index: worse than none)." >&2
+ exit $rc
+ else
+ echo "xc-index: autoindex failed with exit $rc and left no records" >&2
+ exit $rc
+ fi
+ ;;
esac
mkdir -p "$ROOT/state"
cat > "$ROOT/state/$name.json" <<EOF
-{"corpus":"$name","indexed_at":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","prefix":"xc-$name","url":"$URL","autoindex_exit":$rc}
+{"corpus":"$name","indexed_at":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","prefix":"xc-$name","url":"$URL","autoindex_exit":$rc,"salvaged":$salvaged}
EOF
-docs=$(curl -fsS -m 5 "$URL/xc-$name*/_count" 2>/dev/null \
- | sed 's/.*"count":\([0-9]*\).*/\1/' || echo "?")
+[ -n "$docs" ] || docs="$(count_records)"
+[ -n "$docs" ] || docs="?"
echo "corpus '$name' searchable: $docs records"
echo "next: xc.py $name \"<what you need>\""
diff --git a/tools/xerj-code/scripts/xc.py b/tools/xerj-code/scripts/xc.py
index 568698626..3377ca4b2 100755
--- a/tools/xerj-code/scripts/xc.py
+++ b/tools/xerj-code/scripts/xc.py
@@ -448,10 +448,16 @@ def main():
# Reference coding needs the actual implementation, not a keyword-in-context
# snippet. A 320-char highlight tells the agent a file is relevant; it does
# not tell it how the algorithm works.
- ap.add_argument("--full", type=int, metavar="CHARS",
- help="emit up to CHARS of each passage's real source instead of highlights")
+ # Defaulted, not opt-in. This used to be None, which meant the documented
+ # invocation `xc.py <corpus> "<query>"` skipped the symbol path entirely and
+ # fell through to a branch that prints body[:400] — the licence header, on
+ # every Apache/BSD/MIT file in every corpus (issue #368). Retrieving the
+ # definition IS the feature; capping its length is the knob.
+ ap.add_argument("--full", type=int, metavar="CHARS", default=800,
+ help="cap each passage at CHARS of real source (default 800; "
+ "0 prints only the file head)")
ap.add_argument("--no-symbol", action="store_true",
- help="with --full, use a raw byte window instead of the matching definition")
+ help="use a raw byte window instead of the matching definition")
# Default is BM25, decided on TWO corpora after an earlier default (hybrid)
# was chosen on one and turned out to be corpus-specific.
#
@@ -559,15 +565,15 @@ def main():
f"chars from offset {start:,}, window centred on the match]")
print(window)
else:
- frags = []
- for vals in (h.get("highlight") or {}).values():
- frags.extend(vals)
- if frags:
- for f in frags[:2]:
- print(" " + f.replace("\n", "\n ")[:600])
- else:
- body = src.get("body") or ""
- print(" " + str(body)[:400].replace("\n", "\n "))
+ # Only reachable via `--full 0`. `highlight` is never requested
+ # (issue #177 — it reorders hits), so this is the file head and is
+ # labelled as such rather than passed off as a matching passage.
+ body = str(src.get("body") or "")
+ head = body[:400]
+ if len(body) > len(head):
+ print(f" [file head; {len(head):,} of {len(body):,} chars — "
+ f"pass --full N for the matching definition]")
+ print(" " + head.replace("\n", "\n "))
if lic and any(r in lic for r in RESTRICTED):
print(f" !! {lic}: adapt the APPROACH, do not copy the code")