Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

bench: execute the binary snapshot recorded in evaluation receipts

Jarvisf016e078d61b
diff --git a/tools/benchmarks/batch-retrieval/README.md b/tools/benchmarks/batch-retrieval/README.md
index 8ab194ed3..b96ca1705 100644
--- a/tools/benchmarks/batch-retrieval/README.md
+++ b/tools/benchmarks/batch-retrieval/README.md
@@ -28,7 +28,9 @@ The offline Cargo build needs cached dependencies; omit `--offline` on a first
build. Cargo runs from `engine/` to load its target-specific configuration.
Use a release binary built from the
declared engine commit; the harness records its SHA-256 and version but cannot
-prove its source ancestry. Indexing and timing use that same binary.
+prove its source ancestry. Indexing and timing execute a private copy of the
+exact byte snapshot that was hashed; changing the original binary mid-run
+cannot change the measured executable.
The archive is pinned by its GitHub release asset SHA-256:
`54f0419e6d944635eeaf9199923064278c36fd578213db60ca77cc85d2a69e3d`.
diff --git a/tools/benchmarks/batch-retrieval/run.py b/tools/benchmarks/batch-retrieval/run.py
index 8fdd012cd..450e6796b 100644
--- a/tools/benchmarks/batch-retrieval/run.py
+++ b/tools/benchmarks/batch-retrieval/run.py
@@ -35,6 +35,13 @@ def sha(data):
return hashlib.sha256(data).hexdigest()
+def freeze_binary(data, work):
+ binary = work / 'xerj'
+ binary.write_bytes(data)
+ binary.chmod(0o700)
+ return binary
+
+
def read_pack(path):
# Read once: both the digest and records refer to these exact bytes. No extraction.
data = path.read_bytes()
@@ -355,14 +362,13 @@ def main():
case_bytes = (HERE / 'cases.json').read_bytes()
cases = json.loads(case_bytes)
validate_cases(cases, records)
- binary = args.binary.resolve(strict=True)
- binary_sha = sha(binary.read_bytes())
+ binary_data = args.binary.resolve(strict=True).read_bytes()
+ binary_sha = sha(binary_data)
env = child_env()
- version = subprocess.check_output([str(binary), '--version'], text=True, env=env).strip()
host = hardware()
args.output.mkdir(parents=True, exist_ok=False)
report = {'schema_version': 1, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
- 'binary_version': version, 'binary_sha256': binary_sha,
+ 'binary_sha256': binary_sha,
'engine_source_commit_declared': args.engine_commit,
'harness_commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(),
'harness_dirty': bool(subprocess.check_output(['git', 'status', '--porcelain'], cwd=ROOT)),
@@ -383,6 +389,11 @@ def main():
with tempfile.TemporaryDirectory(prefix='xerj-batch-retrieval-') as temp:
work = Path(temp)
try:
+ # Execute the same private byte snapshot we hashed, even if Cargo
+ # or another operator replaces the original executable during a run.
+ binary = freeze_binary(binary_data, work)
+ report['binary_version'] = subprocess.check_output(
+ [str(binary), '--version'], text=True, cwd=work, env=env).strip()
with node(binary, work, env) as url:
ingest(url, records)
reference = None
diff --git a/tools/benchmarks/batch-retrieval/test_run.py b/tools/benchmarks/batch-retrieval/test_run.py
index af4863462..c5eb50fac 100644
--- a/tools/benchmarks/batch-retrieval/test_run.py
+++ b/tools/benchmarks/batch-retrieval/test_run.py
@@ -14,6 +14,18 @@ def hit(identifier='known', citation_id=None):
class ScoringTests(unittest.TestCase):
+ def test_executed_snapshot_does_not_reopen_mutable_binary(self):
+ with tempfile.TemporaryDirectory() as temp:
+ work = Path(temp)
+ original = work / 'input'
+ original.write_bytes(b'first binary')
+ data = original.read_bytes()
+ original.write_bytes(b'replacement binary')
+ frozen = eval.freeze_binary(data, work)
+ self.assertEqual(frozen.read_bytes(), data)
+ self.assertEqual(eval.sha(frozen.read_bytes()), eval.sha(data))
+ self.assertTrue(frozen.stat().st_mode & 0o100)
+
def test_wrong_archive_is_refused_before_parsing(self):
with tempfile.TemporaryDirectory() as temp:
pack = Path(temp) / 'pack.zip'