AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(xc): make the documented reference-coding loop work end to end
xerj-orgcfba3389551f
diff --git a/tools/xerj-code/scripts/xc-index.sh b/tools/xerj-code/scripts/xc-index.shindex b667d9ebb..1cddf516b 100755--- a/tools/xerj-code/scripts/xc-index.sh+++ b/tools/xerj-code/scripts/xc-index.sh@@ -38,6 +38,14 @@ curl -fsS -m 5 "$URL/_cluster/health" >/dev/null 2>&1 || {echo "indexing corpus '$name' from $dir"+# Recorded before the run so a failure can tell "this run wrote records" apart+# from "an earlier run's records are still lying around". Salvaging the latter+# would date stale data to now, which is the failure mode SKILL.md calls worse+# than no index at all.+docs_before="$(curl -fsS -m 5 "$URL/xc-$name*/_count" 2>/dev/null \+ | sed 's/.*"count":\([0-9]*\).*/\1/' || echo "")"+[ -n "$docs_before" ] || docs_before=0+# --no-graph: reference code needs ranked passages, not a relationship map. The# graph detectors cost real time on a large tree and nothing here consumes edges.set +e@@ -49,20 +57,58 @@ set +erc=$?set -e+# How many records actually landed. This is the ground truth about whether the+# corpus is usable, and it is the check SKILL.md already tells people to run by+# hand ("Always verify: curl -s "$URL/xc-<corpus>*/_count" must be > 0").+count_records() {+ curl -fsS -m 5 "$URL/xc-$name*/_count" 2>/dev/null \+ | sed 's/.*"count":\([0-9]*\).*/\1/' || echo ""+}+# Exit 3 is "completed, some files were unparseable" — expected on any real# repository (binaries, fixtures, vendored blobs). Treat it as success.+#+# Any other non-zero code is NOT taken at face value, because autoindex can+# abort in finalisation *after* every document has been written — a whole-corpus+# read-back or generation-cutover reconciliation failure leaves a complete,+# queryable index behind and still exits 1 (see issue #367; observed on+# apache/lucene on both embedding modes, 6,032 java records live). Discarding+# that corpus sends the user back to a step that will fail again, so ask the+# engine what is actually there before deciding. An empty or unreachable index+# still fails, and a salvaged run is reported loudly rather than silently.+docs=""+salvaged=falsecase $rc in0) echo "indexed cleanly" ;;3) echo "indexed (exit 3: some files skipped as junk — normal for real repos)" ;;- *) echo "xc-index: autoindex failed with exit $rc" >&2; exit $rc ;;+ *)+ docs="$(count_records)"+ if [ -n "$docs" ] && [ "$docs" -gt "$docs_before" ] 2>/dev/null; then+ salvaged=true+ echo "xc-index: WARNING — autoindex exited $rc, but this run wrote records" >&2+ echo "xc-index: ($docs_before -> $docs). The corpus is queryable and is being" >&2+ echo "xc-index: recorded as indexed, with autoindex_exit=$rc in its state file." >&2+ echo "xc-index: That exit usually means finalisation failed AFTER the writes." >&2+ echo "xc-index: Coverage is not guaranteed — please report the error above." >&2+ elif [ -n "$docs" ] && [ "$docs" -gt 0 ] 2>/dev/null; then+ echo "xc-index: autoindex failed with exit $rc and wrote no new records" >&2+ echo "xc-index: ($docs already present from an earlier run — not treating" >&2+ echo "xc-index: those as this run's output; a stale corpus dated to now is" >&2+ echo "xc-index: worse than none)." >&2+ exit $rc+ else+ echo "xc-index: autoindex failed with exit $rc and left no records" >&2+ exit $rc+ fi+ ;;esacmkdir -p "$ROOT/state"cat > "$ROOT/state/$name.json" <<EOF-{"corpus":"$name","indexed_at":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","prefix":"xc-$name","url":"$URL","autoindex_exit":$rc}+{"corpus":"$name","indexed_at":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","prefix":"xc-$name","url":"$URL","autoindex_exit":$rc,"salvaged":$salvaged}EOF-docs=$(curl -fsS -m 5 "$URL/xc-$name*/_count" 2>/dev/null \- | sed 's/.*"count":\([0-9]*\).*/\1/' || echo "?")+[ -n "$docs" ] || docs="$(count_records)"+[ -n "$docs" ] || docs="?"echo "corpus '$name' searchable: $docs records"echo "next: xc.py $name \"<what you need>\""diff --git a/tools/xerj-code/scripts/xc.py b/tools/xerj-code/scripts/xc.pyindex 568698626..3377ca4b2 100755--- a/tools/xerj-code/scripts/xc.py+++ b/tools/xerj-code/scripts/xc.py@@ -448,10 +448,16 @@ def main():# Reference coding needs the actual implementation, not a keyword-in-context# snippet. A 320-char highlight tells the agent a file is relevant; it does# not tell it how the algorithm works.- ap.add_argument("--full", type=int, metavar="CHARS",- help="emit up to CHARS of each passage's real source instead of highlights")+ # Defaulted, not opt-in. This used to be None, which meant the documented+ # invocation `xc.py <corpus> "<query>"` skipped the symbol path entirely and+ # fell through to a branch that prints body[:400] — the licence header, on+ # every Apache/BSD/MIT file in every corpus (issue #368). Retrieving the+ # definition IS the feature; capping its length is the knob.+ ap.add_argument("--full", type=int, metavar="CHARS", default=800,+ help="cap each passage at CHARS of real source (default 800; "+ "0 prints only the file head)")ap.add_argument("--no-symbol", action="store_true",- help="with --full, use a raw byte window instead of the matching definition")+ help="use a raw byte window instead of the matching definition")# Default is BM25, decided on TWO corpora after an earlier default (hybrid)# was chosen on one and turned out to be corpus-specific.#@@ -559,15 +565,15 @@ def main():f"chars from offset {start:,}, window centred on the match]")print(window)else:- frags = []- for vals in (h.get("highlight") or {}).values():- frags.extend(vals)- if frags:- for f in frags[:2]:- print(" " + f.replace("\n", "\n ")[:600])- else:- body = src.get("body") or ""- print(" " + str(body)[:400].replace("\n", "\n "))+ # Only reachable via `--full 0`. `highlight` is never requested+ # (issue #177 — it reorders hits), so this is the file head and is+ # labelled as such rather than passed off as a matching passage.+ body = str(src.get("body") or "")+ head = body[:400]+ if len(body) > len(head):+ print(f" [file head; {len(head):,} of {len(body):,} chars — "+ f"pass --full N for the matching definition]")+ print(" " + head.replace("\n", "\n "))if lic and any(r in lic for r in RESTRICTED):print(f" !! {lic}: adapt the APPROACH, do not copy the code")