Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

bench(#874): pin THP + allocator decay off for the measured node — the runner's hugepage floor was the first CI red

xerj-org5b9542bbcd1e
diff --git a/benchmarks/idle-budget/README.md b/benchmarks/idle-budget/README.md
index 478c529ac..2125f08e7 100644
--- a/benchmarks/idle-budget/README.md
+++ b/benchmarks/idle-budget/README.md
@@ -31,6 +31,7 @@ across `/proc/<pid>/task/*/status`. The three are never mixed into one number.
| XERJ | v1.0.0-rc.77 + branch `bench/issue-874-idle-budget-gate`, `cargo build --release -p xerj-server` |
| Node | throwaway data dir, private port in 9610..9639, `--insecure` (TLS+auth off), `--embed-mode lexical` (the default feature-hashing embedder — not neural), no config file — `Config::default()` flush/merge/sampler behaviour is the thing measured |
| Box load | **shared**: other agents were building and running on it during every window (load average is recorded inside each result file: 38–45 in the run of record, up to 63 in the before-fix artifact). Idle-CPU and wakeup figures are process-wide counters, so a busy neighbour inflates *nonvoluntary* switches (scheduler preemption) but not the process's own CPU time; RSS is unaffected. |
+| THP | The fixture pins `MALLOC_CONF=thp:never,dirty_decay_ms:0,muzzy_decay_ms:0` on the measured server. THP: jemalloc follows the kernel's transparent-hugepage mode; on a kernel with `THP=always` (GitHub `ubuntu-latest` runners) each index's small boot allocations land in distinct 2 MB extents and pin a full hugepage — the first CI run measured **2075 kB anon per idle index** (≈ one hugepage) vs 299 kB on a `madvise` host, a page-granularity artifact that fails the 0.2 MB/index budget even for a perfect engine. Decay: without `dirty/muzzy_decay_ms:0` the baseline arm alone wobbled 31–76 MB run-to-run on the same host (whatever freed pages the allocator retained at sample time), a 1.5× swing in the per-index subtraction — with the pins, same host, ci-test profile: **181.6 kB/idx**, gate green (result `citest-check2`). The kernel's THP mode and the pin are printed in the measured section of every run. |
## What the fixture measures
diff --git a/benchmarks/idle-budget/results/result-citest-check.json b/benchmarks/idle-budget/results/result-citest-check.json
new file mode 100644
index 000000000..59e369361
--- /dev/null
+++ b/benchmarks/idle-budget/results/result-citest-check.json
@@ -0,0 +1,106 @@
+{
+ "label": "citest-check",
+ "date_utc": "2026-09-21T23:12:41+00:00",
+ "binary": {
+ "version": "xerj v1.0.0-rc.77",
+ "path": "/root/xerj-874/engine/target/ci-test/xerj"
+ },
+ "corpus": {
+ "shape": "15x30",
+ "indices": 450,
+ "docs_per_index": 3
+ },
+ "port": 9610,
+ "window_s": 120.0,
+ "settle_target_s": 120.0,
+ "boot_ms": {
+ "empty_node": 54664,
+ "baseline_arm": 2214,
+ "n_indices": 1963
+ },
+ "wal_replay_lines_after_clean_flush": 0,
+ "doc_probe_hits": 3,
+ "baseline": {
+ "pid": 1861561,
+ "window_s": 121.5,
+ "clk_tck": 100,
+ "cpu_ticks": 13,
+ "cpu_percent_of_one_core": 0.107,
+ "wakeups_per_s": 25.2,
+ "voluntary_per_s": 24.3,
+ "nonvoluntary_per_s": 0.9,
+ "rss_kb": 31012,
+ "hwm_kb": 117824,
+ "rss_anon_kb": 1460,
+ "rss_file_kb": 29552,
+ "threads": 340,
+ "fds": 15,
+ "per_thread_cpu": [
+ {
+ "comm": "xerj-mem-sample",
+ "cpu_ticks": 7,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.058
+ },
+ {
+ "comm": "xerj-memtable-s",
+ "cpu_ticks": 3,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.025
+ },
+ {
+ "comm": "xerj-rt",
+ "cpu_ticks": 1,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.008
+ }
+ ],
+ "loadavg_start": [
+ "96.55",
+ "50.05",
+ "40.61"
+ ]
+ },
+ "measurement": {
+ "pid": 1709095,
+ "window_s": 120.0,
+ "clk_tck": 100,
+ "cpu_ticks": 25,
+ "cpu_percent_of_one_core": 0.2083,
+ "wakeups_per_s": 26.6,
+ "voluntary_per_s": 24.2,
+ "nonvoluntary_per_s": 2.5,
+ "rss_kb": 166272,
+ "hwm_kb": 180108,
+ "rss_anon_kb": 136556,
+ "rss_file_kb": 29716,
+ "threads": 336,
+ "fds": 15,
+ "per_thread_cpu": [
+ {
+ "comm": "xerj-memtable-s",
+ "cpu_ticks": 16,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.133
+ },
+ {
+ "comm": "xerj-mem-sample",
+ "cpu_ticks": 6,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.05
+ },
+ {
+ "comm": "xerj-rt",
+ "cpu_ticks": 1,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.008
+ }
+ ],
+ "loadavg_start": [
+ "35.77",
+ "67.91",
+ "60.14"
+ ]
+ },
+ "per_index_rss_kb": 300.6
+}
\ No newline at end of file
diff --git a/benchmarks/idle-budget/results/result-citest-check2.json b/benchmarks/idle-budget/results/result-citest-check2.json
new file mode 100644
index 000000000..2247a3722
--- /dev/null
+++ b/benchmarks/idle-budget/results/result-citest-check2.json
@@ -0,0 +1,106 @@
+{
+ "label": "citest-check2",
+ "date_utc": "2026-09-21T23:23:17+00:00",
+ "binary": {
+ "version": "xerj v1.0.0-rc.77",
+ "path": "/root/xerj-874/engine/target/ci-test/xerj"
+ },
+ "corpus": {
+ "shape": "15x30",
+ "indices": 450,
+ "docs_per_index": 3
+ },
+ "port": 9610,
+ "window_s": 120.0,
+ "settle_target_s": 120.0,
+ "boot_ms": {
+ "empty_node": 64,
+ "baseline_arm": 824,
+ "n_indices": 38
+ },
+ "wal_replay_lines_after_clean_flush": 0,
+ "doc_probe_hits": 3,
+ "baseline": {
+ "pid": 2027123,
+ "window_s": 120.0,
+ "clk_tck": 100,
+ "cpu_ticks": 12,
+ "cpu_percent_of_one_core": 0.1,
+ "wakeups_per_s": 26.0,
+ "voluntary_per_s": 24.4,
+ "nonvoluntary_per_s": 1.6,
+ "rss_kb": 82084,
+ "hwm_kb": 122280,
+ "rss_anon_kb": 54592,
+ "rss_file_kb": 27492,
+ "threads": 340,
+ "fds": 15,
+ "per_thread_cpu": [
+ {
+ "comm": "xerj-mem-sample",
+ "cpu_ticks": 7,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.058
+ },
+ {
+ "comm": "xerj-memtable-s",
+ "cpu_ticks": 3,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.025
+ },
+ {
+ "comm": "xerj-rt",
+ "cpu_ticks": 1,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.008
+ }
+ ],
+ "loadavg_start": [
+ "42.05",
+ "48.07",
+ "53.00"
+ ]
+ },
+ "measurement": {
+ "pid": 3926987,
+ "window_s": 120.0,
+ "clk_tck": 100,
+ "cpu_ticks": 26,
+ "cpu_percent_of_one_core": 0.2166,
+ "wakeups_per_s": 23.7,
+ "voluntary_per_s": 20.3,
+ "nonvoluntary_per_s": 3.3,
+ "rss_kb": 163820,
+ "hwm_kb": 178144,
+ "rss_anon_kb": 136272,
+ "rss_file_kb": 27548,
+ "threads": 336,
+ "fds": 15,
+ "per_thread_cpu": [
+ {
+ "comm": "xerj-memtable-s",
+ "cpu_ticks": 16,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.133
+ },
+ {
+ "comm": "xerj-mem-sample",
+ "cpu_ticks": 8,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.067
+ },
+ {
+ "comm": "xerj-rt",
+ "cpu_ticks": 1,
+ "threads": 1,
+ "cpu_percent_of_one_core": 0.008
+ }
+ ],
+ "loadavg_start": [
+ "35.14",
+ "41.42",
+ "49.37"
+ ]
+ },
+ "per_index_rss_kb": 181.6
+}
\ No newline at end of file
diff --git a/benchmarks/idle-budget/run.sh b/benchmarks/idle-budget/run.sh
index 4dd3ea851..c2bc7053a 100755
--- a/benchmarks/idle-budget/run.sh
+++ b/benchmarks/idle-budget/run.sh
@@ -154,7 +154,20 @@ node_start() { # $1 = data dir, $2 = log file; sets SERVER_PID
# no -c => Config::default() — default flush/merge/sampler behaviour is what
# this fixture measures. Lexical embedder, explicitly (the default
# feature-hashing embedder, never called neural).
- "$XERJ_BIN" --insecure --port "$PORT" --data-dir "$1" \
+ #
+ # THP + decay pins: jemalloc follows the kernel's THP mode by default. On a
+ # kernel with transparent_hugepage=always (GitHub runners), every index's
+ # small boot allocations land in distinct 2 MB extents and each pins a full
+ # hugepage — measured 2075 kB anon per idle index on the runner vs 299 kB
+ # on a madvise host, i.e. a ~2 MB/index kernel page-granularity artifact
+ # that would fail a 0.2 MB/index budget even for a perfect engine. Pinning
+ # thp:never for the MEASURED process removes the artifact. The decay pins
+ # (dirty/muzzy -> 0) make both arms read true in-use memory instead of
+ # whatever freed pages the allocator happens to be retaining at sample
+ # time — without them the baseline arm alone wobbled 31-76 MB across runs
+ # on this host, which is a 1.5x swing in the per-index subtraction.
+ MALLOC_CONF="thp:never,dirty_decay_ms:0,muzzy_decay_ms:0" \
+ "$XERJ_BIN" --insecure --port "$PORT" --data-dir "$1" \
--embed-mode lexical > "$2" 2>&1 &
SERVER_PID=$!
sleep 0.4
@@ -247,7 +260,10 @@ a = sample(); time.sleep(secs); b = sample()
elapsed = b["t"] - a["t"]
clk = os.sysconf("SC_CLK_TCK")
ticks = (b["utime"] - a["utime"]) + (b["stime"] - a["stime"])
-vol = b["vol"] - a["vol"]; non = b["non"] - a["non"]
+# max(0, …): a thread exiting between the two passes drops its counters from
+# the second sum, which would otherwise print a negative rate (seen as the
+# CI run's "nonvoluntary -0.0/s").
+vol = max(0, b["vol"] - a["vol"]); non = max(0, b["non"] - a["non"])
per_comm = {}
for tid, (comm, ticks_then) in a["tasks"].items():
if tid in b["tasks"]:
@@ -439,6 +455,11 @@ json.dump(out, open(path, "w"), indent=1)
print()
print("== measured ==")
+try:
+ _thp = next(l.strip() for l in open("/sys/kernel/mm/transparent_hugepage/enabled") if "[" in l)
+except (OSError, StopIteration):
+ _thp = "?"
+print(f" kernel THP mode {_thp}, allocator MALLOC_CONF=thp:never,decay=0 (page-granularity artifact removed; in-use memory)")
print(f" boot-to-green, empty node {out['boot_ms']['empty_node']:>8} ms")
print(f" boot-to-green, {N:>4} indices {out['boot_ms']['n_indices']:>8} ms (cleanly-flushed corpus)")
print(f" WAL replay lines after flush {out['wal_replay_lines_after_clean_flush']:>8} (0 = nothing to replay)")