Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

docs(air-gapped): make the recipe fail closed, and lint it so it stays that way

xerj-org14170ff755c4
diff --git a/.github/scripts/recipe-shell-lint.py b/.github/scripts/recipe-shell-lint.py
new file mode 100644
index 000000000..920aa79d9
--- /dev/null
+++ b/.github/scripts/recipe-shell-lint.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Lint the shell that ships inside docs/recipes/*.md and its published copy.
+
+Two properties, each of which has already been violated in this repo:
+
+1. No block chains an integrity check into the action it is supposed to gate
+ with ``&&`` while relying on ``set -e``. ``set -e`` does not stop a command
+ that is part of an ``&&`` list unless it is the last one, so::
+
+ set -eu
+ sha256sum -c "$ASSET.sha256" \\
+ && tar -xzf "$ASSET"
+
+ exits 0 on a bad digest and lets the caller continue into the next section.
+ Writing each step on its own line is what makes errexit load-bearing.
+ Only integrity verifiers are flagged: ``[ "$n" -eq 0 ] && break`` is
+ ordinary flow control, not a gate that must fail closed.
+
+2. Every shell block in a recipe appears verbatim on the published page. The
+ two files are edited by hand and have drifted before: one copy of the
+ air-gapped model block was guarded and the other was not, so a reader of
+ xerj.org and a reader of the repo ran different code. Line wrapping and
+ HTML entities are normalised away; the sequence of commands is not. The
+ page may carry *extra* blocks — sample output is also in ``<pre>`` — so this
+ is containment, not equality.
+
+``bash -n`` is used to tell shell from sample output on the published page,
+which has no language marker. Blocks in the markdown are trusted to be shell
+because the author wrote ```` ```bash ```` and are parsed as such.
+"""
+
+from __future__ import annotations
+
+import html
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[2]
+MD_DIR = REPO / "docs" / "recipes"
+HTML_DIR = REPO / "landing" / "docs" / "recipes"
+
+MD_BLOCK = re.compile(r"^```(?:bash|sh)\n(.*?)^```$", re.S | re.M)
+HTML_BLOCK = re.compile(r'<pre class="code">(.*?)</pre>', re.S)
+
+# Commands whose whole purpose is to refuse the next step.
+VERIFIERS = ("sha256sum", "sha512sum", "shasum", "sha256", "gpg", "gpgv", "cosign", "cmp")
+
+# Recipes whose published page must carry every command in the markdown.
+#
+# This is not yet the whole tree. Several pages legitimately predate their
+# recipe or were edited independently and are missing blocks outright; making
+# them match is editorial work, not a lint fix, and doing it silently here would
+# hide how much of the tree is unchecked. The recipes listed below are the ones
+# where a divergence is a *security* difference rather than a formatting one, so
+# they are held in lockstep now and the rest are reported as uncovered.
+LOCKSTEP = frozenset({"air-gapped-deployment"})
+
+
+def parses(text: str) -> tuple[bool, str]:
+ proc = subprocess.run(
+ ["bash", "-n"], input=text, text=True, capture_output=True, check=False
+ )
+ return proc.returncode == 0, proc.stderr.strip()
+
+
+def normalise(text: str) -> list[str]:
+ """Commands, stripped of the formatting the two copies may differ in."""
+ joined = re.sub(r"\\\n\s*", " ", text)
+ return [" ".join(line.split()) for line in joined.splitlines() if line.strip()]
+
+
+def check_gate_chaining(text: str, where: str, errors: list[str]) -> None:
+ if not re.search(r"^\s*set -[a-z]*e", text, re.M):
+ return
+ for stmt in re.sub(r"\\\n\s*", " ", text).splitlines():
+ if "&&" not in stmt:
+ continue
+ head = stmt.strip().lstrip("( ").split()
+ if head and head[0] in VERIFIERS:
+ errors.append(
+ f"{where}: `{head[0]}` gates the rest of an && list inside a "
+ f"`set -e` block, which exempts it from errexit — put each step "
+ f"on its own line\n {stmt.strip()}"
+ )
+
+
+def main() -> int:
+ errors: list[str] = []
+ md_checked = 0
+ paired = 0
+ uncovered: list[str] = []
+
+ for md_path in sorted(MD_DIR.glob("*.md")):
+ md_rel = md_path.relative_to(REPO)
+ md_blocks = MD_BLOCK.findall(md_path.read_text())
+
+ for i, block in enumerate(md_blocks):
+ where = f"{md_rel} block {i + 1}"
+ ok, err = parses(block)
+ if not ok:
+ errors.append(f"{where}: does not parse\n {err}")
+ check_gate_chaining(block, where, errors)
+ md_checked += 1
+
+ html_path = HTML_DIR / (md_path.stem + ".html")
+ if not html_path.exists():
+ continue
+ if md_path.stem not in LOCKSTEP:
+ uncovered.append(md_path.stem)
+ continue
+
+ published = []
+ for raw in HTML_BLOCK.findall(html_path.read_text()):
+ block = html.unescape(raw)
+ if parses(block)[0]:
+ published.append(normalise(block))
+ check_gate_chaining(block, f"{html_path.relative_to(REPO)}", errors)
+
+ for i, block in enumerate(md_blocks):
+ paired += 1
+ if normalise(block) not in published:
+ errors.append(
+ f"{md_rel} block {i + 1} is not on the published page "
+ f"{html_path.relative_to(REPO)} — the two copies have drifted "
+ f"and readers get different instructions\n"
+ f" recipe: {normalise(block)}"
+ )
+
+ if errors:
+ print(f"recipe-shell-lint: {len(errors)} problem(s)\n", file=sys.stderr)
+ for e in errors:
+ print(f" {e}\n", file=sys.stderr)
+ return 1
+
+ print(
+ f"recipe-shell-lint: {md_checked} recipe shell blocks OK, "
+ f"{paired} checked against their published copy"
+ )
+ if uncovered:
+ print(
+ f"recipe-shell-lint: published copies NOT checked for drift "
+ f"({len(uncovered)}): {', '.join(sorted(uncovered))}"
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b2d59565f..039e96e41 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -487,6 +487,9 @@ jobs:
- name: Installer script lint (landing/get, landing/get.ps1)
run: bash .github/scripts/installer-lint.sh
+ - name: Recipe shell lint (docs/recipes + published copies)
+ run: python3 .github/scripts/recipe-shell-lint.py
+
- uses: actions/setup-node@v4
with:
node-version: '20'
diff --git a/docs/recipes/air-gapped-deployment.md b/docs/recipes/air-gapped-deployment.md
index 92f546d78..47138a867 100644
--- a/docs/recipes/air-gapped-deployment.md
+++ b/docs/recipes/air-gapped-deployment.md
@@ -98,15 +98,31 @@ extracting:
```bash
set -eu
+
: "${TAG:?set TAG to the same operator-approved release tag}"
VERSION="${TAG#v}"
-ASSET="xerj-${VERSION}-x86_64-unknown-linux-musl.tar.gz"
-cd /opt/xerj-staging/release
-sha256sum -c "$ASSET.sha256" \
- && tar -xzf "$ASSET" \
- && test -x "xerj-${VERSION}-x86_64-unknown-linux-musl/xerj"
+STAGE="xerj-${VERSION}-x86_64-unknown-linux-musl"
+ASSET="${STAGE}.tar.gz"
+
+(
+ cd /opt/xerj-staging/release
+ grep -qF " $ASSET" "$ASSET.sha256"
+ sha256sum -c "$ASSET.sha256"
+ rm -rf "$STAGE"
+ tar -xzf "$ASSET"
+ test -x "$STAGE/xerj"
+)
```
+Each step is on its own line for a reason. `set -e` does not stop a command
+that is part of an `&&` list unless it is the last one, so writing the same
+sequence as `sha256sum -c … && tar -xzf …` leaves the block exiting `0` after a
+failed digest and lets a wrapping script continue into section 2. The `grep`
+line is what makes the digest apply to this asset: `sha256sum -c` reports
+success for whatever filenames the `.sha256` happens to list, including a file
+that is not the archive you are about to extract. `rm -rf "$STAGE"` removes any
+tree left behind by an earlier run, so `test -x` cannot pass on a stale binary.
+
## 2. Install the base lexical node
The account and directories below are an example of the required ownership;
@@ -114,6 +130,9 @@ use the equivalent account-management command on the target Linux image. No
model directory is created for the lexical deployment.
```bash
+set -eu
+
+: "${VERSION:?run section 1 in this shell first, or set VERSION to the verified release}"
if ! getent passwd xerj >/dev/null; then
sudo useradd --system --user-group --home-dir /var/lib/xerj --shell /sbin/nologin xerj
fi
@@ -160,6 +179,9 @@ machine, download the three files consumed by the built-in loader and checksum
them for the transfer:
```bash
+set -eu
+
+: "${OUT:?run section 1 in this shell first, or set OUT to the staging directory}"
MODEL="$OUT/model/all-MiniLM-L6-v2"
mkdir -p "$MODEL"
for FILE in config.json tokenizer.json model.safetensors; do
@@ -172,10 +194,26 @@ done
)
```
-Transfer `OUT/model` with the release and verify `model.sha256` inside the
-enclave. Then install the files and change only the embedding block:
+Transfer `OUT/model` with the release. Verify it inside the enclave before
+installing anything — the same fail-closed rule as section 1, and the reason
+the check is a command here rather than a sentence:
+
+```bash
+set -eu
+
+(
+ cd /opt/xerj-staging/model/all-MiniLM-L6-v2
+ sha256sum -c ../model.sha256
+)
+```
+
+`model.sha256` names all three files, so a transfer that dropped or truncated
+one fails this check rather than installing a partial model. Then install the
+files and change only the embedding block:
```bash
+set -eu
+
sudo install -d -o xerj -g xerj -m 0750 /opt/xerj/models/all-MiniLM-L6-v2
sudo install -o xerj -g xerj -m 0640 \
/opt/xerj-staging/model/all-MiniLM-L6-v2/{config.json,tokenizer.json,model.safetensors} \
@@ -199,6 +237,8 @@ documentation was live-tested against a disconnected firewall. They verify the
actual binary and local files in your enclave.
```bash
+set -eu
+
XERJ=/opt/xerj/bin/xerj
DATA=/var/lib/xerj
KEY="$(sudo cat "$DATA/admin.key")"
@@ -215,6 +255,8 @@ local lexical embedder and needs zero model files; assert that it returns
`_id` `1`.
```bash
+set -eu
+
curl -fsS -X PUT "http://127.0.0.1:9200/offline-demo" \
-H "Authorization: ApiKey $KEY" -H 'Content-Type: application/json' \
-d '{"mappings":{"properties":{"body":{"type":"semantic_text"}}}}'
@@ -255,6 +297,8 @@ data directory is reused. Inspect and remove that overlay before an offline run
if the directory is not fresh:
```bash
+set -eu
+
curl -fsS -H "Authorization: ApiKey $KEY" \
http://127.0.0.1:9200/_xerj/wal_tap
curl -fsS -X DELETE -H "Authorization: ApiKey $KEY" \
diff --git a/landing/docs/recipes/air-gapped-deployment.html b/landing/docs/recipes/air-gapped-deployment.html
index b4c112c0c..5b63bdf64 100644
--- a/landing/docs/recipes/air-gapped-deployment.html
+++ b/landing/docs/recipes/air-gapped-deployment.html
@@ -101,17 +101,28 @@ curl -fL --retry 3 -o "$OUT/release/$ASSET.sha256" "$BASE/$ASSET.sha256"
)</pre>
<p>The checksum proves archive integrity against the digest published beside it in the same release; it is not a signature, provenance statement, or attestation. After transfer, set the same approved <code class="inline">TAG</code> and verify before extracting:</p>
<pre class="code">set -eu
+
: "${TAG:?set TAG to the same operator-approved release tag}"
VERSION="${TAG#v}"
-ASSET="xerj-${VERSION}-x86_64-unknown-linux-musl.tar.gz"
-cd /opt/xerj-staging/release
-sha256sum -c "$ASSET.sha256" \
- && tar -xzf "$ASSET" \
- && test -x "xerj-${VERSION}-x86_64-unknown-linux-musl/xerj"</pre>
+STAGE="xerj-${VERSION}-x86_64-unknown-linux-musl"
+ASSET="${STAGE}.tar.gz"
+
+(
+ cd /opt/xerj-staging/release
+ grep -qF " $ASSET" "$ASSET.sha256"
+ sha256sum -c "$ASSET.sha256"
+ rm -rf "$STAGE"
+ tar -xzf "$ASSET"
+ test -x "$STAGE/xerj"
+)</pre>
+ <p>Each step is on its own line for a reason. <code class="inline">set -e</code> does not stop a command that is part of an <code class="inline">&amp;&amp;</code> list unless it is the last one, so writing this as <code class="inline">sha256sum -c &hellip; &amp;&amp; tar -xzf &hellip;</code> leaves the block exiting <code class="inline">0</code> after a failed digest and lets a wrapping script continue into section 2. The <code class="inline">grep</code> line is what makes the digest apply to this asset: <code class="inline">sha256sum -c</code> reports success for whatever filenames the <code class="inline">.sha256</code> happens to list, including a file that is not the archive you are about to extract. <code class="inline">rm -rf "$STAGE"</code> removes any tree left by an earlier run, so <code class="inline">test -x</code> cannot pass on a stale binary.</p>
<h2>2. Install the base lexical node</h2>
<p>Create the service account first, then install only the staged binary. No model directory is needed for lexical mode:</p>
-<pre class="code">if ! getent passwd xerj &gt;/dev/null; then
+<pre class="code">set -eu
+
+: "${VERSION:?run section 1 in this shell first, or set VERSION to the verified release}"
+if ! getent passwd xerj &gt;/dev/null; then
sudo useradd --system --user-group --home-dir /var/lib/xerj --shell /sbin/nologin xerj
fi
sudo install -d -m 0755 /opt/xerj/bin /etc/xerj
@@ -137,15 +148,30 @@ default_model = ""</pre>
<h2>3. Optional neural model</h2>
<p>Only when neural semantics are needed, stage the three loader files on the connected machine:</p>
-<pre class="code">MODEL="$OUT/model/all-MiniLM-L6-v2"
+<pre class="code">set -eu
+
+: "${OUT:?run section 1 in this shell first, or set OUT to the staging directory}"
+MODEL="$OUT/model/all-MiniLM-L6-v2"
mkdir -p "$MODEL"
for FILE in config.json tokenizer.json model.safetensors; do
curl -fL --retry 3 -o "$MODEL/$FILE" \
"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/$FILE"
done
-( cd "$MODEL" &amp;&amp; sha256sum config.json tokenizer.json model.safetensors &gt; ../model.sha256 )</pre>
- <p>Transfer and verify <code class="inline">model.sha256</code>, then install the files:</p>
-<pre class="code">sudo install -d -o xerj -g xerj -m 0750 /opt/xerj/models/all-MiniLM-L6-v2
+(
+ cd "$MODEL"
+ sha256sum config.json tokenizer.json model.safetensors &gt; ../model.sha256
+)</pre>
+ <p>Transfer <code class="inline">OUT/model</code> with the release. Verify it inside the enclave before installing anything &mdash; the same fail-closed rule as section 1, and the reason the check is a command here rather than a sentence:</p>
+<pre class="code">set -eu
+
+(
+ cd /opt/xerj-staging/model/all-MiniLM-L6-v2
+ sha256sum -c ../model.sha256
+)</pre>
+ <p><code class="inline">model.sha256</code> names all three files, so a transfer that dropped or truncated one fails this check rather than installing a partial model. Then install the files:</p>
+<pre class="code">set -eu
+
+sudo install -d -o xerj -g xerj -m 0750 /opt/xerj/models/all-MiniLM-L6-v2
sudo install -o xerj -g xerj -m 0640 \
/opt/xerj-staging/model/all-MiniLM-L6-v2/{config.json,tokenizer.json,model.safetensors} \
/opt/xerj/models/all-MiniLM-L6-v2/</pre>
@@ -156,7 +182,9 @@ local_model_dir = "/opt/xerj/models/all-MiniLM-L6-v2"</pre>
<p>XERJ does not independently verify model checksums; the transferred digest is an operator check, not a XERJ signature. Omit this section and all model assets for lexical search.</p>
<h2>4. Authenticate local clients and verify</h2>
-<pre class="code">XERJ=/opt/xerj/bin/xerj
+<pre class="code">set -eu
+
+XERJ=/opt/xerj/bin/xerj
DATA=/var/lib/xerj
KEY="$(sudo cat "$DATA/admin.key")"
export XERJ_API_KEY="$KEY"
@@ -164,13 +192,15 @@ export XERJ_API_KEY="$KEY"
curl -fsS -H "Authorization: ApiKey $KEY" \
http://127.0.0.1:9200/_cluster/health</pre>
<p>With the lexical node running, this semantic query uses the local lexical embedder and needs zero model files. Assert that it returns <code class="inline">_id</code> <code class="inline">1</code>:</p>
-<pre class="code">curl -fsS -X PUT http://127.0.0.1:9200/offline-demo \
+<pre class="code">set -eu
+
+curl -fsS -X PUT "http://127.0.0.1:9200/offline-demo" \
-H "Authorization: ApiKey $KEY" -H 'Content-Type: application/json' \
-d '{"mappings":{"properties":{"body":{"type":"semantic_text"}}}}'
-curl -fsS -X POST 'http://127.0.0.1:9200/offline-demo/_doc/1?refresh=true' \
+curl -fsS -X POST "http://127.0.0.1:9200/offline-demo/_doc/1?refresh=true" \
-H "Authorization: ApiKey $KEY" -H 'Content-Type: application/json' \
-d '{"body":"A local lexical node can answer this without a network service."}'
-curl -fsS -X POST http://127.0.0.1:9200/offline-demo/_search \
+curl -fsS -X POST "http://127.0.0.1:9200/offline-demo/_search" \
-H "Authorization: ApiKey $KEY" -H 'Content-Type: application/json' \
-d '{"query":{"semantic":{"field":"body","query":"local lexical"}}}'</pre>
<p>Repeat health and query after restarting the same data directory. For the stronger boundary check, run the sequence with host egress disabled and inspect firewall/log counters; this recipe does not claim that test was run here.</p>
@@ -183,7 +213,9 @@ curl -fsS -X POST http://127.0.0.1:9200/offline-demo/_search \
<h2>Existing data directories and local clients</h2>
<p>WAL tap is disabled by default, but <code class="inline">PUT /_xerj/wal_tap</code> persists a runtime overlay and reapplies it over TOML on restart. Inspect and remove an old overlay before reusing a data directory:</p>
-<pre class="code">curl -fsS -H "Authorization: ApiKey $KEY" http://127.0.0.1:9200/_xerj/wal_tap
+<pre class="code">set -eu
+
+curl -fsS -H "Authorization: ApiKey $KEY" http://127.0.0.1:9200/_xerj/wal_tap
curl -fsS -X DELETE -H "Authorization: ApiKey $KEY" http://127.0.0.1:9200/_xerj/wal_tap</pre>
<p>Cluster mode remains disabled unless explicitly enabled and supplied a shared <code class="inline">cluster.auth_secret</code>. <code class="inline">xerj mcp</code> is a local stdio client to <code class="inline">XERJ_URL</code>; <code class="inline">xerj autoindex</code> also defaults to the local ES-compatible endpoint. Neither creates an external listener or silently uploads corpus data.</p>