Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

bench(#874): set _RJEM_MALLOC_CONF too — the unprefixed MALLOC_CONF was silently ignored

xerj-orge82bdfd7cadc
diff --git a/benchmarks/idle-budget/README.md b/benchmarks/idle-budget/README.md
index 2125f08e7..f58ab70fd 100644
--- a/benchmarks/idle-budget/README.md
+++ b/benchmarks/idle-budget/README.md
@@ -31,7 +31,7 @@ across `/proc/<pid>/task/*/status`. The three are never mixed into one number.
| XERJ | v1.0.0-rc.77 + branch `bench/issue-874-idle-budget-gate`, `cargo build --release -p xerj-server` |
| Node | throwaway data dir, private port in 9610..9639, `--insecure` (TLS+auth off), `--embed-mode lexical` (the default feature-hashing embedder — not neural), no config file — `Config::default()` flush/merge/sampler behaviour is the thing measured |
| Box load | **shared**: other agents were building and running on it during every window (load average is recorded inside each result file: 38–45 in the run of record, up to 63 in the before-fix artifact). Idle-CPU and wakeup figures are process-wide counters, so a busy neighbour inflates *nonvoluntary* switches (scheduler preemption) but not the process's own CPU time; RSS is unaffected. |
-| THP | The fixture pins `MALLOC_CONF=thp:never,dirty_decay_ms:0,muzzy_decay_ms:0` on the measured server. THP: jemalloc follows the kernel's transparent-hugepage mode; on a kernel with `THP=always` (GitHub `ubuntu-latest` runners) each index's small boot allocations land in distinct 2 MB extents and pin a full hugepage — the first CI run measured **2075 kB anon per idle index** (≈ one hugepage) vs 299 kB on a `madvise` host, a page-granularity artifact that fails the 0.2 MB/index budget even for a perfect engine. Decay: without `dirty/muzzy_decay_ms:0` the baseline arm alone wobbled 31–76 MB run-to-run on the same host (whatever freed pages the allocator retained at sample time), a 1.5× swing in the per-index subtraction — with the pins, same host, ci-test profile: **181.6 kB/idx**, gate green (result `citest-check2`). The kernel's THP mode and the pin are printed in the measured section of every run. |
+| THP | The fixture pins `thp:never,dirty_decay_ms:0,muzzy_decay_ms:0` on the measured server, via **both** `_RJEM_MALLOC_CONF` and `MALLOC_CONF`. The var name matters: the engine's allocator is `tikv-jemalloc-sys`, which builds jemalloc `--with-jemalloc-prefix=_rjem_`, and a prefixed build reads `_RJEM_MALLOC_CONF` — the plain `MALLOC_CONF` set alone was **silently ignored** (the second CI run measured the same 1835 kB/idx as the first with the pin printed in the log). Verified on this box: `MALLOC_CONF=bogus_opt:1` boots with no allocator comment; `_RJEM_MALLOC_CONF=bogus_opt:1` boots with `<jemalloc>: Invalid conf pair: bogus_opt:1` — and `run.sh` now aborts if the boot log contains that line. THP: jemalloc follows the kernel's transparent-hugepage mode by default; on a kernel with `THP=always` (GitHub `ubuntu-latest` runners) each index's small boot allocations land in distinct 2 MB extents and pin a full hugepage — the first CI run measured **2075 kB anon per idle index** (≈ one hugepage) vs 299 kB on a `madvise` host, a page-granularity artifact that fails the 0.2 MB/index budget even for a perfect engine. Decay: without `dirty/muzzy_decay_ms:0` the baseline arm alone wobbled 31–76 MB run-to-run on the same host (whatever freed pages the allocator retained at sample time), a 1.5× swing in the per-index subtraction — with the pins, same host, ci-test profile: **181.6 kB/idx**, gate green (result `citest-check2`). The kernel's THP mode, the pin, and the kernel-side `AnonHugePages` reading (~0 = pin live) are printed in the measured section of every run. |
## What the fixture measures
diff --git a/benchmarks/idle-budget/results/result-citest-check3.json b/benchmarks/idle-budget/results/result-citest-check3.json
new file mode 100644
index 000000000..99a6af016
--- /dev/null
+++ b/benchmarks/idle-budget/results/result-citest-check3.json
@@ -0,0 +1,102 @@
+{
+ "label": "citest-check3",
+ "date_utc": "2026-09-21T23:58:33+00:00",
+ "binary": {
+ "version": "xerj v1.0.0-rc.77",
+ "path": "/root/xerj-874/engine/target/ci-test/xerj"
+ },
+ "corpus": {
+ "shape": "15x30",
+ "indices": 450,
+ "docs_per_index": 3
+ },
+ "port": 9610,
+ "window_s": 120.0,
+ "settle_target_s": 120.0,
+ "boot_ms": {
+ "empty_node": 168,
+ "baseline_arm": 26,
+ "n_indices": 9
+ },
+ "wal_replay_lines_after_clean_flush": 0,
+ "doc_probe_hits": 3,
+ "baseline": {
+ "pid": 1721823,
+ "window_s": 120.0,
+ "clk_tck": 100,
+ "cpu_ticks": 14,
+ "cpu_percent_of_one_core": 0.1167,
+ "wakeups_per_s": 25.8,
+ "voluntary_per_s": 24.2,
+ "nonvoluntary_per_s": 1.7,
+ "rss_kb": 76236,
+ "hwm_kb": 76236,
+ "rss_anon_kb": 50192,
+ "rss_file_kb": 26044,
+ "anon_hugepages_kb": 0,
+ "threads": 340,
+ "fds": 15,
+ "per_thread_cpu": [
+ {
+ "comm": "xerj-mem-sample",
+ "cpu_ticks": 8,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.067
+ },
+ {
+ "comm": "xerj-memtable-s",
+ "cpu_ticks": 3,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.025
+ }
+ ],
+ "loadavg_start": [
+ "31.60",
+ "36.01",
+ "39.74"
+ ]
+ },
+ "measurement": {
+ "pid": 4099232,
+ "window_s": 120.0,
+ "clk_tck": 100,
+ "cpu_ticks": 27,
+ "cpu_percent_of_one_core": 0.225,
+ "wakeups_per_s": 27.4,
+ "voluntary_per_s": 24.1,
+ "nonvoluntary_per_s": 3.3,
+ "rss_kb": 164088,
+ "hwm_kb": 164088,
+ "rss_anon_kb": 136152,
+ "rss_file_kb": 27936,
+ "anon_hugepages_kb": 0,
+ "threads": 336,
+ "fds": 15,
+ "per_thread_cpu": [
+ {
+ "comm": "xerj-memtable-s",
+ "cpu_ticks": 17,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.142
+ },
+ {
+ "comm": "xerj-mem-sample",
+ "cpu_ticks": 9,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.075
+ },
+ {
+ "comm": "xerj-rt",
+ "cpu_ticks": 2,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.017
+ }
+ ],
+ "loadavg_start": [
+ "39.76",
+ "38.11",
+ "39.94"
+ ]
+ },
+ "per_index_rss_kb": 195.2
+}
\ No newline at end of file
diff --git a/benchmarks/idle-budget/run.sh b/benchmarks/idle-budget/run.sh
index c2bc7053a..99d969773 100755
--- a/benchmarks/idle-budget/run.sh
+++ b/benchmarks/idle-budget/run.sh
@@ -166,7 +166,18 @@ node_start() { # $1 = data dir, $2 = log file; sets SERVER_PID
# whatever freed pages the allocator happens to be retaining at sample
# time — without them the baseline arm alone wobbled 31-76 MB across runs
# on this host, which is a 1.5x swing in the per-index subtraction.
- MALLOC_CONF="thp:never,dirty_decay_ms:0,muzzy_decay_ms:0" \
+ #
+ # BOTH spellings are set: the engine's allocator is tikv-jemalloc-sys,
+ # which builds jemalloc with --with-jemalloc-prefix=_rjem_, and a prefixed
+ # build reads <PREFIX>MALLOC_CONF (_RJEM_MALLOC_CONF), NOT MALLOC_CONF.
+ # The unprefixed MALLOC_CONF was set alone first and silently did nothing
+ # — the second CI run measured the same 1835 kB/idx as the first, with the
+ # pin printed in the log. Verified on this box: MALLOC_CONF=bogus_opt:1
+ # boots with no allocator comment, _RJEM_MALLOC_CONF=bogus_opt:1 boots with
+ # "<jemalloc>: Invalid conf pair: bogus_opt:1". The unprefixed spelling
+ # stays for the day the engine links an unprefixed jemalloc.
+ CONF="thp:never,dirty_decay_ms:0,muzzy_decay_ms:0"
+ MALLOC_CONF="$CONF" _RJEM_MALLOC_CONF="$CONF" \
"$XERJ_BIN" --insecure --port "$PORT" --data-dir "$1" \
--embed-mode lexical > "$2" 2>&1 &
SERVER_PID=$!
@@ -174,6 +185,14 @@ node_start() { # $1 = data dir, $2 = log file; sets SERVER_PID
if ! kill -0 "$SERVER_PID" 2>/dev/null; then
echo "node failed to start; log:"; tail -20 "$2"; exit 1
fi
+ # A rejected pair means the measurement would run uncontrolled — the exact
+ # failure that cost two CI runs. jemalloc writes the complaint to stderr
+ # at allocator init, well inside the 0.4 s above.
+ if grep -q 'Invalid conf pair' "$2"; then
+ echo "::error::allocator rejected the MALLOC_CONF pin — refusing to measure uncontrolled"
+ grep 'jemalloc' "$2" || true
+ exit 1
+ fi
}
wait_green() { # $1 = log for context; sets BOOT_MS (process start -> health green)
@@ -232,6 +251,15 @@ def sample():
def cnt(path):
try: return len(os.listdir(f"/proc/{pid}/{path}"))
except OSError: return -1
+ # AnonHugePages (smaps_rollup): THP-backed anon RSS, straight from the
+ # kernel. Under thp:never it reads ~0 even on a THP=always host; a dead
+ # pin shows up here as hundreds of MB. Printed as evidence, not gated —
+ # the per-index RSS gate is the gate; this line is the one-grep diagnosis.
+ try:
+ thp_kb = next(int(l.split()[1]) for l in open(f"/proc/{pid}/smaps_rollup")
+ if l.startswith("AnonHugePages:"))
+ except (OSError, StopIteration, IndexError, ValueError):
+ thp_kb = -1
vol = non = 0
tasks = {}
try: tids = os.listdir(f"/proc/{pid}/task")
@@ -254,6 +282,7 @@ def sample():
vol=vol, non=non, tasks=tasks,
rss_kb=num("VmRSS"), hwm_kb=num("VmHWM"),
rss_anon_kb=num("RssAnon"), rss_file_kb=num("RssFile"),
+ anon_hugepages_kb=thp_kb,
threads=cnt("task"), fds=cnt("fd"))
a = sample(); time.sleep(secs); b = sample()
@@ -285,6 +314,7 @@ res = dict(
nonvoluntary_per_s=round(non / elapsed, 1),
rss_kb=b["rss_kb"], hwm_kb=b["hwm_kb"],
rss_anon_kb=b["rss_anon_kb"], rss_file_kb=b["rss_file_kb"],
+ anon_hugepages_kb=b["anon_hugepages_kb"],
threads=b["threads"], fds=b["fds"],
per_thread_cpu=top_threads,
loadavg_start=open("/proc/loadavg").read().split()[:3],
@@ -469,6 +499,8 @@ print(f" wakeups/s, {N:>4} indices {n['wakeups_per_s']:>8} /s (volu
print(f" wakeups/s, baseline node {base['wakeups_per_s']:>8} /s")
print(f" VmRSS, {N:>4} indices {n['rss_kb']:>8} kB (VmHWM {n['hwm_kb']} kB; anon {n['rss_anon_kb']} kB + file {n['rss_file_kb']} kB)")
print(f" VmRSS, baseline node {base['rss_kb']:>8} kB")
+print(f" AnonHugePages, {N:>4} indices {n['anon_hugepages_kb']:>8} kB (~0 = the thp:never pin is live at the kernel, any THP mode)")
+print(f" AnonHugePages, baseline node {base['anon_hugepages_kb']:>8} kB")
print(f" per-index idle RSS {per_index_kb:>8.1f} kB (({n['rss_kb']} - {base['rss_kb']}) / {N})")
print(f" threads {n['threads']}, fds {n['fds']}, loadavg {n['loadavg_start']}")
print(f" results: {path}")