Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

ci: skip the heavy Rust jobs on docs-only pull requests (fail open) (#1477)

xerj-org7041db0454b9
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 82b63de7f..bec6f99d6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -33,6 +33,77 @@ env:
# point there, so only its ACTION ref is pinned, not what "nightly" resolves
# to on the day the job runs.
jobs:
+ # Change scoping. Docs-only pull requests used to run all ~22 jobs, and with
+ # the org's concurrent-job limit that queued docs PRs behind Rust builds for
+ # about an hour (53 queued runs on 2026-10-15). `changes` classifies the
+ # diff with scripts/ci/change_scope.sh (tested by
+ # scripts/ci/test_change_scope.sh, run first below) and the heavy Rust jobs
+ # run only when `code == 'true'`. The light guards (cla-config,
+ # reference-coding, landing-constants-guard, workflow-timeouts,
+ # release-notes-gate) never depend on it.
+ #
+ # It FAILS OPEN: an empty diff, a failed diff, an all-zeros `before`, or any
+ # path not on the narrow docs allowlist all mean code=true (run everything).
+ # If `changes` itself fails, the gated jobs still run: their condition is
+ # `!cancelled() && (needs.changes.result != 'success' || ...)`, because a
+ # plain `needs:` would SKIP them, and a skipped required check reads green.
+ # The allowlist excludes every docs file a test reads by name; see the
+ # script header for the list and the tests that read them.
+ #
+ # Pushes to main always run everything. Build + Test also carries doc-claim
+ # tests that walk EVERY .md/.html/.txt in the tree (security_tooling_claims,
+ # docs_capability_lists), which no path allowlist can exclude. On a PR that
+ # risk is accepted; on main a docs merge that breaks one of them shows up red
+ # on its own merge commit instead of on the next unrelated code PR.
+ changes:
+ name: Change scope
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ outputs:
+ code: ${{ steps.scope.outputs.code }}
+ ux: ${{ steps.scope.outputs.ux }}
+ steps:
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+ with:
+ fetch-depth: 0
+
+ - name: Classifier unit tests
+ run: bash scripts/ci/test_change_scope.sh
+
+ - name: Classify changed files
+ id: scope
+ env:
+ EVENT: ${{ github.event_name }}
+ PR_BASE: ${{ github.event.pull_request.base.sha }}
+ PUSH_BEFORE: ${{ github.event.before }}
+ HEAD_SHA: ${{ github.sha }}
+ run: |
+ fail_open() {
+ echo "::notice::change scope: $1 -> running everything"
+ printf 'code=true\nux=true\n' >> "$GITHUB_OUTPUT"
+ exit 0
+ }
+ if [ "$EVENT" = "pull_request" ]; then
+ range="${PR_BASE}...${HEAD_SHA}"
+ else
+ if [ -z "$PUSH_BEFORE" ] || [ "$PUSH_BEFORE" = "0000000000000000000000000000000000000000" ]; then
+ fail_open "no usable push base"
+ fi
+ range="${PUSH_BEFORE}..${HEAD_SHA}"
+ fi
+ if ! files="$(git diff --name-only "$range")"; then
+ fail_open "git diff $range failed"
+ fi
+ echo "Changed files ($range):"
+ printf '%s\n' "$files" | sed 's/^/ /'
+ result="$(printf '%s\n' "$files" | bash scripts/ci/change_scope.sh)" || fail_open "classifier errored"
+ if [ "$EVENT" != "pull_request" ]; then
+ echo "Push event: classifier said [${result//$'\n'/ }], running everything (see job comment)"
+ fail_open "push to $GITHUB_REF"
+ fi
+ echo "$result"
+ echo "$result" >> "$GITHUB_OUTPUT"
+
cla-config:
name: CLA gate config
runs-on: ubuntu-latest
@@ -238,6 +309,8 @@ jobs:
lint:
name: Format + Clippy
+ needs: [changes]
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
@@ -275,6 +348,8 @@ jobs:
# line. (#749)
workspace-resolves:
name: Workspace resolves (links + lockfile)
+ needs: [changes]
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
@@ -300,6 +375,8 @@ jobs:
# They are still in the log on every run, which is the point.
security-audit:
name: Dependency advisory audit (cargo-audit)
+ needs: [changes]
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
@@ -337,6 +414,8 @@ jobs:
# FUZZ_SECONDS=3600 bash .github/scripts/fuzz-smoke.sh
fuzz:
name: Fuzz harnesses (cargo-fuzz)
+ needs: [changes]
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
@@ -385,6 +464,8 @@ jobs:
build-test:
name: Build + Test
+ needs: [changes]
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
runs-on: ubuntu-latest
# #751's second-order damage was the whole job hanging against GitHub's 6h
# default while main's `cancel-in-progress: false` never auto-cancels — main
@@ -499,6 +580,8 @@ jobs:
# with the tests.
release-build:
name: Release build (fat LTO, all members link)
+ needs: [changes]
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
@@ -518,6 +601,8 @@ jobs:
onnx-feature:
name: ONNX feature compile + tests
+ needs: [changes]
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
@@ -551,6 +636,8 @@ jobs:
api-checks:
name: API smoke + liveness + benchmark
+ needs: [changes]
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
@@ -592,6 +679,8 @@ jobs:
conformance:
name: ES-compat YAML conformance
+ needs: [changes]
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
@@ -688,6 +777,8 @@ jobs:
# script that needs a live server and a real browser, i.e. by nothing.
ux-tests:
name: UX pipeline tests
+ needs: [changes]
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true' || needs.changes.outputs.ux == 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
@@ -724,6 +815,8 @@ jobs:
# breaks an install, and it needs nothing but Node.
usecase-smoke:
name: Use-case harnesses (brain + MCP + autoindex)
+ needs: [changes]
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
@@ -807,6 +900,8 @@ jobs:
# under.
autoindex-fd-smoke:
name: Autoindex FD smoke (${{ matrix.os }})
+ needs: [changes]
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
strategy:
fail-fast: false
matrix:
@@ -858,6 +953,8 @@ jobs:
# arrive as multiples, not percents.
idle-budget:
name: Idle budget (many-index at-rest CPU/RSS)
+ needs: [changes]
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
runs-on: ubuntu-latest
# One tier up from the no-compile tier: compiles one crate (xerj-server)
# on the shared ci-test cache, then boots and measures. The fixture itself
diff --git a/scripts/ci/change_scope.sh b/scripts/ci/change_scope.sh
new file mode 100755
index 000000000..ffefb6f45
--- /dev/null
+++ b/scripts/ci/change_scope.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Classify a change set for .github/workflows/ci.yml's `changes` job.
+#
+# Reads changed file paths (repo-relative, one per line) on stdin and prints:
+# code=true|false false ONLY when every path is on the docs-only allowlist
+# ux=true|false true when anything under xerj-ux/ changed
+#
+# It FAILS OPEN: empty input, an unknown path, or any doubt means code=true,
+# i.e. run every job. Skipping a job wrongly costs a red main later; running
+# it needlessly only costs runner minutes. Tests: scripts/ci/test_change_scope.sh
+#
+# The allowlist is deliberately narrow. Most of this repo's prose is NOT inert:
+# Build + Test carries doc-claim tests that read named docs surfaces and assert
+# on their contents (engine/crates/xerj-engine/tests/docs_capability_lists.rs,
+# docs_do_not_promise_the_next_release.rs, security_tooling_claims.rs,
+# engine/crates/xerj-rerank/tests/egress_inventory.rs,
+# engine/crates/xerj-api/tests/scroll_snapshot_cap_is_documented.rs,
+# engine/crates/xerj-common/src/config.rs's include_str! of landing/docs/config.html,
+# engine/crates/xerj-mcp/tests/published_schema_drift.rs, ...), and the
+# use-case smoke job lints landing/get{,.ps1}. Every file those read BY NAME is
+# listed in is_test_input below and always counts as code. Add to that list
+# whenever a test starts reading a new docs file.
+set -u
+
+is_test_input() {
+ case "$1" in
+ # Root/engine prose that doc-claim tests read by name.
+ README.md|ROADMAP.md|SECURITY.md|CHANGELOG.md|*/CHANGELOG.md) return 0 ;;
+ docs/RERANK.md|docs/ZERO_TOKEN_DIRECTION.md|docs/ARCHITECTURE.md) return 0 ;;
+ docs/recipes/air-gapped-deployment.md|docs/media/*) return 0 ;;
+ user-feedback/09-security/cves-and-vulnerabilities.md) return 0 ;;
+ # Published surfaces with generated/asserted sections.
+ landing/llms.txt|landing/llms-full.txt|landing/openapi.json) return 0 ;;
+ landing/docs/agents/schemas/*) return 0 ;;
+ landing/docs/queries.html|landing/docs/aggregations.html) return 0 ;;
+ landing/docs/api-es-compat.html|landing/docs/migration-from-es.html) return 0 ;;
+ landing/docs/config.html|landing/docs/env.html) return 0 ;;
+ landing/docs/recipes/air-gapped-deployment.html) return 0 ;;
+ landing/answers/rerank-search-results-calibrated-judge.*) return 0 ;;
+ landing/pricing/index.html|landing/demo/index.html) return 0 ;;
+ # Installer scripts: linted + exercised by the use-case smoke job.
+ landing/get|landing/get.ps1) return 0 ;;
+ esac
+ return 1
+}
+
+is_docs_only() {
+ local p="$1"
+ is_test_input "$p" && return 1
+ case "$p" in
+ # Never docs, whatever the extension: code trees, test fixtures, the
+ # reference-coding toolkit, use-case harnesses, the console (embedded into
+ # the server binary by xerj-console-api/build.rs), SEO sources.
+ engine/*|xerj-ux/*|tools/*|demo/usecases/*|content/*|scripts/*|.github/*) return 1 ;;
+ */fixtures/*|*/testdata/*|*/test/*|*/tests/*) return 1 ;;
+ esac
+ case "$p" in
+ docs/*|user-feedback/*|landing/*) return 0 ;;
+ .contributors|.gitignore|LICENSE*|README*) return 0 ;;
+ benchmarks/*.md) return 0 ;;
+ *.md) return 0 ;;
+ esac
+ return 1
+}
+
+code=false
+ux=false
+seen=0
+while IFS= read -r path || [ -n "$path" ]; do
+ path="${path%$'\r'}"
+ [ -z "$path" ] && continue
+ seen=$((seen + 1))
+ case "$path" in xerj-ux/*) ux=true ;; esac
+ if ! is_docs_only "$path"; then
+ code=true
+ fi
+done
+
+# Nothing to classify (empty diff, failed diff upstream): fail open.
+[ "$seen" -eq 0 ] && code=true
+
+echo "code=$code"
+echo "ux=$ux"
diff --git a/scripts/ci/test_change_scope.sh b/scripts/ci/test_change_scope.sh
new file mode 100755
index 000000000..f73a3186e
--- /dev/null
+++ b/scripts/ci/test_change_scope.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Tests for scripts/ci/change_scope.sh. Run by the `changes` job in ci.yml
+# before the classifier is trusted, and runnable locally:
+# bash scripts/ci/test_change_scope.sh
+set -u
+HERE="$(cd "$(dirname "$0")" && pwd)"
+SCOPE="$HERE/change_scope.sh"
+fails=0
+
+check() { # name expected-code expected-ux files...
+ local name="$1" want_code="$2" want_ux="$3"
+ shift 3
+ local out
+ out="$(printf '%s\n' "$@" | bash "$SCOPE")"
+ local want
+ want="$(printf 'code=%s\nux=%s' "$want_code" "$want_ux")"
+ if [ "$out" = "$want" ]; then
+ echo "ok - $name"
+ else
+ echo "FAIL - $name: want [$(echo $want)] got [$(echo $out)]"
+ fails=$((fails + 1))
+ fi
+}
+
+check "docs-only" false false docs/WHY_XERJ.md landing/blog/index.html user-feedback/x/report.md .contributors LICENSE benchmarks/idle-budget/README.md notes/thing.md
+check "mixed docs + rust" true false docs/WHY_XERJ.md engine/crates/xerj-engine/src/index.rs
+check "rust-only" true false engine/crates/xerj-engine/src/index.rs
+check "xerj-ux-only" true true xerj-ux/console.js
+check "CHANGELOG" true false CHANGELOG.md
+check "tools/xerj-code" true false tools/xerj-code/SKILL.md
+check "workflow" true false .github/workflows/ci.yml
+check "test-read doc" true false docs/RERANK.md
+check "test-read landing" true false landing/llms.txt
+check "root README" true false README.md
+check "installer" true false landing/get
+check "test fixture .md" true false engine/crates/xerj-autoindex/tests/fixtures/a.md
+
+# Empty input (an empty or failed diff) must fail open.
+out="$(printf '' | bash "$SCOPE")"
+if [ "$out" = "$(printf 'code=true\nux=false')" ]; then
+ echo "ok - empty input"
+else
+ echo "FAIL - empty input: got [$(echo $out)]"
+ fails=$((fails + 1))
+fi
+
+if [ "$fails" -ne 0 ]; then
+ echo "$fails change_scope test(s) failed"
+ exit 1
+fi
+echo "all change_scope tests passed"