Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

ci: run evidence for gstack clear --run: files/tests in tree.json, pregate-remote-gate.sh --list

garrytan3323e8ece9af
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index b334a72a91..1452efe115 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -1375,7 +1375,7 @@ jobs:
# Event parity (docs/TESTING.md#event-parity): the same job on every event;
# EVENT only labels the recorded tree.
e2e-full-suite:
- needs: [prepare-e2e, e2e-status]
+ needs: [prepare-e2e, e2e-status, executed-receipts]
if: ${{ needs.prepare-e2e.outputs.reuse != 'true' && always() }}
runs-on: ubicloud-standard-2-ubuntu-2404
timeout-minutes: 5
@@ -1393,9 +1393,18 @@ jobs:
persist-credentials: false
sparse-checkout: scripts/ci-full-suite-tree.sh
sparse-checkout-cone-mode: false
+ # The executed-file list in tree.json (run evidence, docs/ci-event-parity.md#remote-gate-reuse)
+ # comes from this run's manifest; without it the record carries no files.
+ - name: Download the execution manifest
+ continue-on-error: true
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+ with:
+ name: ci-manifest
+ path: ${{ runner.temp }}/ci-manifest
- name: Record the tested tree
env:
ARTIFACT_DIR: ${{ runner.temp }}/full-suite
+ MANIFEST: ${{ runner.temp }}/ci-manifest/ci-manifest.json
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
WORKFLOW: ${{ github.workflow }}
diff --git a/.github/workflows/persistence-validation.yml b/.github/workflows/persistence-validation.yml
index 2564896815..d8b2fa2136 100644
--- a/.github/workflows/persistence-validation.yml
+++ b/.github/workflows/persistence-validation.yml
@@ -453,12 +453,24 @@ jobs:
DATABASE_URL: postgres://gbrain_test:[email protected]:5432/gbrain_test
GBRAIN_TEST_ALLOW_DATABASE_URL: '1'
GBRAIN_TEST_RECONCILE_CRASH_MANIFEST_DIR: .context/reconcile-crashes
+ # Receipts put these files in the Test run's ci-manifest, so the run's
+ # tree.json lists them as executed (docs/ci-event-parity.md#remote-gate-reuse).
+ GBRAIN_TEST_RECEIPT_DIR: ${{ runner.temp }}/receipts
run: |
if [ '${{ matrix.engine }}' = 'postgres' ]; then
- bun --no-env-file test --timeout=180000 test/e2e/reconcile-crash.test.ts test/e2e/reconcile-crash-unactivated.test.ts
+ bash scripts/run-with-receipt.sh reconciliation postgres-bun-${{ matrix.bun }} -- bun --no-env-file test --timeout=180000 test/e2e/reconcile-crash.test.ts test/e2e/reconcile-crash-unactivated.test.ts
else
- env -u DATABASE_URL -u GBRAIN_DATABASE_URL bun --no-env-file test --timeout=180000 test/reconcile-crash-*.slow.test.ts
+ env -u DATABASE_URL -u GBRAIN_DATABASE_URL bash scripts/run-with-receipt.sh reconciliation pglite-bun-${{ matrix.bun }} -- bun --no-env-file test --timeout=180000 test/reconcile-crash-*.slow.test.ts
fi
+ - name: Upload receipts
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+ with:
+ name: receipts-reconciliation-${{ matrix.engine }}-bun-${{ matrix.bun }}
+ path: ${{ runner.temp }}/receipts
+ retention-days: 14
+ if-no-files-found: warn
+ overwrite: true
- name: Upload reconciliation crash manifests
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8a4cf38182..fd2a82d79d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1299,7 +1299,7 @@ jobs:
# a pull request gets it from the full-ci dispatch on its head.
full-suite:
name: ${{ (github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && (inputs.native_only == true || inputs.race_hunt == true || inputs.stress_files != '' || inputs.stress_base != ''))) && 'full-suite-not-run' || 'full-suite' }}
- needs: [changes, test-status]
+ needs: [changes, test-status, executed-receipts]
if: ${{ needs.changes.outputs.reuse != 'true' && always() && github.event_name != 'pull_request' && (github.event_name != 'workflow_dispatch' || (inputs.native_only != true && inputs.race_hunt != true && !inputs.stress_files && !inputs.stress_base)) }}
runs-on: ubicloud-standard-2-ubuntu-2404
timeout-minutes: 5
@@ -1317,9 +1317,18 @@ jobs:
persist-credentials: false
sparse-checkout: scripts/ci-full-suite-tree.sh
sparse-checkout-cone-mode: false
+ # The executed-file list in tree.json (run evidence, docs/ci-event-parity.md#remote-gate-reuse)
+ # comes from this run's manifest; without it the record carries no files.
+ - name: Download the execution manifest
+ continue-on-error: true
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+ with:
+ name: ci-manifest
+ path: ${{ runner.temp }}/ci-manifest
- name: Record the tested tree
env:
ARTIFACT_DIR: ${{ runner.temp }}/full-suite
+ MANIFEST: ${{ runner.temp }}/ci-manifest/ci-manifest.json
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
WORKFLOW: ${{ github.workflow }}
diff --git a/docs/ci-event-parity.md b/docs/ci-event-parity.md
index 74154bec68..86bed415eb 100644
--- a/docs/ci-event-parity.md
+++ b/docs/ci-event-parity.md
@@ -131,6 +131,40 @@ reaches the head through gstack's stamp rule (`gstack-pregate clear --receipt`
with the stamp files in `.gstack/ship-policy.json`), never through this check.
Merge still requires `full-suite` and `e2e-full-suite` on the exact head.
+### Run evidence for gstack (`gstack-pregate clear --run`)
+
+gstack can take the same GitHub runs as evidence itself, so a slot runs one full
+suite per tree instead of a full-ci pass and a gate
+([gstack docs/pregate.md#run-evidence](https://github.com/garrytan/gstack/blob/main/docs/pregate.md#run-evidence)).
+`gstack-pregate clear --remote pregate.json/remote_gate --run <Test run> --run <E2E run> --allow-repo-commands`
+reads each run's `full-suite-tree` / `e2e-full-suite-tree` artifact and accepts the pair
+when both passed on pregate.json's tree and their executed files cover the gate's
+inventory:
+
+- `tree.json` carries `files` (every test file the run executed, repo-relative)
+ and `tests` (its executed tests). `scripts/ci-full-suite-tree.sh` takes them from
+ the run's `ci-manifest.json`, and only when that manifest is complete and names
+ the same run, attempt and commit. Otherwise both are left out with a warning:
+ the run still counts for merge, but gstack refuses it (`inventory_uncovered`).
+ A push that reused a run has no `full-suite` job, so it is never evidence.
+- `bash scripts/pregate-remote-gate.sh --list` prints the inventory: the gate
+ command with `--list` (`scripts/ci-ubicloud.ts --list`), the same lanes and
+ discovery the gate runs, one file per line. It exits non-zero when the
+ inventory fails or is empty (a doc-only tree under `:diff` runs no test file,
+ so it clears through the gate as before). The gate leaves out the keyed-only
+ Tier 2 file (`test/e2e/skills.test.ts`): a VM has neither the keys nor
+ openclaw, so every test in it would skip.
+- gstack accepts only `push` and `workflow_dispatch` runs whose newest attempt
+ succeeded. **A flaky job means re-running the whole workflow.** "Re-run failed
+ jobs" keeps every job that passed at its older attempt, so the new attempt's
+ manifest is incomplete and its `tree.json` has no `files`
+ (`inventory_uncovered`); a `tree.json` left from an older attempt is refused
+ too (`artifact_mismatch`).
+- The profiles live in the base branch's `.gstack/pregate.json` (`run_profiles`
+ for `test.yml` / `full-suite` and `e2e.yml` / `e2e-full-suite`, and
+ `remote_gate_runs` with this inventory command). gstack reads them only from
+ `origin/<base>`, so they apply to a branch once they are on master.
+
## Dependency-audit rule
`dependency-audit` blocks on pushes, schedules, manual runs and pull requests
diff --git a/scripts/ci-full-suite-tree.sh b/scripts/ci-full-suite-tree.sh
index ab5112606b..b87e60bf57 100755
--- a/scripts/ci-full-suite-tree.sh
+++ b/scripts/ci-full-suite-tree.sh
@@ -1,15 +1,48 @@
#!/usr/bin/env bash
-# Record the tree a full-profile run tested (docs/ci-event-parity.md#full-ci-on-pull-requests).
+# Record the tree a full-profile run tested and the test files it executed
+# (docs/ci-event-parity.md#full-ci-on-pull-requests, #remote-gate-reuse).
# Called by test.yml `full-suite` and e2e.yml `e2e-full-suite` after their
-# aggregate succeeded. scripts/ci-reuse.ts reads the artifact on a push to master.
-# Input (environment): ARTIFACT_DIR, RUN_ID, RUN_ATTEMPT, WORKFLOW, EVENT.
-# Output: $ARTIFACT_DIR/tree.json and one step-summary line.
+# aggregate succeeded. scripts/ci-reuse.ts reads the artifact on a push to master;
+# `gstack-pregate clear --run` reads it as run evidence for the remote gate.
+# Input (environment): ARTIFACT_DIR, RUN_ID, RUN_ATTEMPT, WORKFLOW, EVENT, and
+# optionally MANIFEST, the run's ci-manifest.json (scripts/ci-manifest.ts).
+# Output: $ARTIFACT_DIR/tree.json and step-summary lines.
+# `files` (every test file the run executed, repo-relative) and `tests` (its
+# executed test count) come only from a complete manifest of this run, attempt
+# and commit. Otherwise both are left out with a warning: the record still says
+# the full suite passed, but gstack cannot take it as run evidence.
set -euo pipefail
mkdir -p "$ARTIFACT_DIR"
tree=$(git rev-parse 'HEAD^{tree}')
-jq -n --arg tree "$tree" --arg sha "$(git rev-parse HEAD)" --arg workflow "$WORKFLOW" --arg event "$EVENT" \
- --argjson run_id "$RUN_ID" --argjson run_attempt "$RUN_ATTEMPT" \
- '{tree: $tree, sha: $sha, workflow: $workflow, event: $event, run_id: $run_id, run_attempt: $run_attempt, profile: "full"}' \
- | tee "$ARTIFACT_DIR/tree.json"
-echo "$WORKFLOW full suite passed on tree $tree (commit $(git rev-parse HEAD), run $RUN_ID attempt $RUN_ATTEMPT)." >> "${GITHUB_STEP_SUMMARY:-/dev/null}"
+sha=$(git rev-parse HEAD)
+evidence="$ARTIFACT_DIR/evidence.json"
+echo '{}' > "$evidence"
+problem=""
+if [ -z "${MANIFEST:-}" ] || [ ! -s "$MANIFEST" ]; then
+ problem="no ci-manifest.json from this run"
+else
+ problem=$(jq -r --argjson id "$RUN_ID" --argjson attempt "$RUN_ATTEMPT" --arg sha "$sha" '
+ if .schema != "gbrain-ci-manifest/v1" then "ci-manifest.json has schema \(.schema)"
+ elif .run_id != $id or .run_attempt != $attempt then "ci-manifest.json belongs to run \(.run_id) attempt \(.run_attempt), not run \($id) attempt \($attempt) (re-run the whole workflow, not only its failed jobs)"
+ elif .sha != $sha then "ci-manifest.json belongs to commit \(.sha), not \($sha)"
+ elif .complete != true then "ci-manifest.json is incomplete: \(.problems | join("; "))"
+ else "" end' "$MANIFEST" 2>&1) || problem="ci-manifest.json is unreadable: $problem"
+ if [ -z "$problem" ]; then
+ jq -c '{files: ([.files[].file | select(test("\\.test\\.[cm]?[jt]sx?$"))] | unique), tests: ([.files[].executed] | add // 0)}' "$MANIFEST" > "$evidence"
+ [ "$(jq '.files | length' "$evidence")" -gt 0 ] || { problem="ci-manifest.json lists no test file"; echo '{}' > "$evidence"; }
+ fi
+fi
+jq -n --arg tree "$tree" --arg sha "$sha" --arg workflow "$WORKFLOW" --arg event "$EVENT" \
+ --argjson run_id "$RUN_ID" --argjson run_attempt "$RUN_ATTEMPT" --slurpfile evidence "$evidence" \
+ '{tree: $tree, sha: $sha, workflow: $workflow, event: $event, run_id: $run_id, run_attempt: $run_attempt, profile: "full"} + $evidence[0]' \
+ > "$ARTIFACT_DIR/tree.json"
+rm -f "$evidence"
+jq -c 'del(.files)' "$ARTIFACT_DIR/tree.json"
+echo "$WORKFLOW full suite passed on tree $tree (commit $sha, run $RUN_ID attempt $RUN_ATTEMPT)." >> "${GITHUB_STEP_SUMMARY:-/dev/null}"
+if [ -n "$problem" ]; then
+ echo "::warning::tree.json records no executed test files: $problem. The full suite still counts for merge, but not as gstack run evidence."
+ echo "No executed-file list: $problem." >> "${GITHUB_STEP_SUMMARY:-/dev/null}"
+else
+ echo "Run evidence: $(jq '.files | length' "$ARTIFACT_DIR/tree.json") test files, $(jq .tests "$ARTIFACT_DIR/tree.json") executed tests." >> "${GITHUB_STEP_SUMMARY:-/dev/null}"
+fi
diff --git a/scripts/ci-ubicloud.ts b/scripts/ci-ubicloud.ts
index aed0a4ffe0..ce848989c9 100755
--- a/scripts/ci-ubicloud.ts
+++ b/scripts/ci-ubicloud.ts
@@ -25,6 +25,10 @@
* audit Postgres 15 (#7112); the catalog goldens hold
* Postgres 16 output and fail on 15 by design
* --diff doc-only diffs run only the doc checks and gitleaks (ci:local --diff)
+ * --list print the test files this selection runs (every selected
+ * lane's inventory, keyed-only E2E files left out), one per
+ * line, and exit; exit 1 when it is empty. Needs no key or
+ * VM (scripts/pregate-remote-gate.sh --list)
* --record-weights write measured durations to scripts/ubicloud/weights.json
* --keep leave the VMs running (destroy with ubi-runner.sh down NAME)
* --ttl-hours N VM TTL written into each name as -t<N> (default 2,
@@ -108,6 +112,11 @@ const BATCH_TIMEOUT_MS = 20 * 60 * 1000;
const VM_LIST_TIMEOUT_MS = 120_000;
// Test timeouts and the mined shard weights are calibrated on this location's VMs.
const GATE_LOCATION = "eu-central-h1";
+// Keyed-only E2E files: e2e.yml's Tier 2 job runs them with the provider keys and
+// openclaw on the nightly schedule. A gate VM has neither (setup-ci-vm.sh drops the
+// keys), so every test in them would skip; the gate leaves them out instead of
+// counting a run that tested nothing (GBRA-237 removed the same keyless run from CI).
+export const KEYED_E2E = ["test/e2e/skills.test.ts"];
// Items this heavy are the run's long poles: spread them one per VM so two of
// them never compete for the same VM's CPUs.
const HEAVY_MS = 60_000;
@@ -139,6 +148,8 @@ interface Opts {
recordWeights: boolean;
keep: boolean;
ttlHours: number;
+ /** Print the gate's test files and exit; starts nothing. */
+ list: boolean;
}
function parseArgs(argv: string[]): Opts {
@@ -154,6 +165,7 @@ function parseArgs(argv: string[]): Opts {
recordWeights: false,
keep: false,
ttlHours: 2,
+ list: false,
};
for (let i = 0; i < argv.length; i++) {
const arg = argv[i]!;
@@ -173,6 +185,7 @@ function parseArgs(argv: string[]): Opts {
else if (arg === "--record-weights") opts.recordWeights = true;
else if (arg === "--keep") opts.keep = true;
else if (arg === "--ttl-hours") opts.ttlHours = Number(value());
+ else if (arg === "--list") opts.list = true;
else usage(`unknown argument ${arg}`);
}
for (const lane of opts.lanes) if (!(ALL_LANES as readonly string[]).includes(lane)) usage(`unknown lane ${lane}`);
@@ -192,7 +205,7 @@ function parseArgs(argv: string[]): Opts {
function usage(message: string): never {
console.error(`ci-ubicloud: ${message}`);
- console.error("usage: bun run scripts/ci-ubicloud.ts [--vms N] [--size SIZE] [--slots N] [--location LOC] [--image IMAGE] [--lanes a,b] [--pg-image IMAGE] [--diff] [--record-weights] [--keep] [--ttl-hours N]");
+ console.error("usage: bun run scripts/ci-ubicloud.ts [--vms N] [--size SIZE] [--slots N] [--location LOC] [--image IMAGE] [--lanes a,b] [--pg-image IMAGE] [--diff] [--record-weights] [--keep] [--ttl-hours N] [--list]");
process.exit(2);
}
@@ -233,50 +246,70 @@ interface Vm {
control: boolean;
}
-async function main() {
- const opts = parseArgs(process.argv.slice(2));
- if (!process.env.UBICLOUD_API_KEY && !process.env.UBICLOUD_API_TOKEN) {
- console.error("ci-ubicloud: UBICLOUD_API_KEY (or UBICLOUD_API_TOKEN) is not set");
- process.exit(2);
- }
- if (!process.env.UBI_OWNER) {
- console.error("ci-ubicloud: UBI_OWNER is not set; set it to your thread code in lowercase (e.g. UBI_OWNER=gbra106) so every VM carries its owner (UBI_OWNER_REQUIRED)");
- process.exit(2);
- }
- const owner = sh("bash", [RUNNER, "owner"]).trim();
- const sha = sh("git", ["rev-parse", "HEAD"]).trim();
- const runId = `${new Date().toISOString().replace(/[:.]/g, "-")}-${sha.slice(0, 8)}`;
- const runDir = join(ROOT, ".context/ci-ubicloud", runId);
- mkdirSync(join(runDir, "logs"), { recursive: true });
- mkdirSync(join(runDir, "failures"), { recursive: true });
-
- // ── Inventory: the same discovery the ci:local wrappers use ───────────────
- const e2eFiles = lines(sh("bash", ["scripts/run-e2e.sh", "--dry-run-list"]));
+/**
+ * The gate's test selection, the same discovery the ci:local wrappers use: the
+ * lanes that run, each lane's files, and the serial and slow lanes'
+ * machine-exclusive files. --list prints exactly these files.
+ */
+function gateInventory(opts: Opts, note: (message: string) => void) {
if (opts.diff) {
const classification = spawnSync("bun", ["run", "scripts/select-e2e.ts", "--classify-only"], { cwd: ROOT, encoding: "utf8" }).stdout?.trim();
if (classification === "DOC_ONLY") {
- log("--diff: doc-only diff — running the doc checks locally, then gitleaks (ci:local doc-only fast-path)");
- if (spawnSync("bash", ["scripts/ci-doc-checks.sh"], { cwd: ROOT, stdio: "inherit" }).status !== 0) {
+ note("--diff: doc-only diff — running the doc checks locally, then gitleaks (ci:local doc-only fast-path)");
+ if (!opts.list && spawnSync("bash", ["scripts/ci-doc-checks.sh"], { cwd: ROOT, stdio: "inherit" }).status !== 0) {
process.exit(1);
}
opts.lanes = new Set(["gitleaks"]);
opts.vms = 1;
} else {
- log(`--diff: E2E narrowing is retired; running the full E2E corpus (see docs/TESTING.md#e2e-selection). Classification: ${classification || "unknown"}`);
+ note(`--diff: E2E narrowing is retired; running the full E2E corpus (see docs/TESTING.md#e2e-selection). Classification: ${classification || "unknown"}`);
}
}
const exclusive = lines(sh("bash", ["scripts/run-serial-tests.sh", "--dry-run-list-exclusive"]));
- const serialPool = lines(sh("bash", ["scripts/run-serial-tests.sh", "--dry-run-list"])).filter((f) => !exclusive.includes(f));
const slowExclusive = lines(sh("bash", ["scripts/run-slow-tests.sh", "--dry-run-list-exclusive"]));
const arms = readArmsList(ROOT);
if (arms.errors.length) throw new Error(`test/postgres-unit-arms.txt: ${arms.errors.join("; ")}`);
const inventory: Record<Lane, string[]> = {
unit: lines(sh("bash", ["scripts/run-unit-shard.sh", "--dry-run-list"])),
- serial: serialPool,
+ serial: lines(sh("bash", ["scripts/run-serial-tests.sh", "--dry-run-list"])).filter((f) => !exclusive.includes(f)),
slow: lines(sh("bash", ["scripts/run-slow-tests.sh", "--dry-run-list"])).filter((f) => !slowExclusive.includes(f)),
"pg-arms": arms.files,
- e2e: e2eFiles,
+ e2e: lines(sh("bash", ["scripts/run-e2e.sh", "--dry-run-list"])).filter((f) => !KEYED_E2E.includes(f)),
};
+ return { inventory, exclusive, slowExclusive };
+}
+
+async function main() {
+ const opts = parseArgs(process.argv.slice(2));
+ if (opts.list) {
+ const { inventory, exclusive, slowExclusive } = gateInventory(opts, (message) => console.error(`ci-ubicloud --list: ${message}`));
+ const files = new Set<string>();
+ for (const [lane, laneFiles] of Object.entries(inventory)) if (opts.lanes.has(lane)) laneFiles.forEach((f) => files.add(f));
+ if (opts.lanes.has("serial")) exclusive.forEach((f) => files.add(f));
+ if (opts.lanes.has("slow")) slowExclusive.forEach((f) => files.add(f));
+ if (!files.size) {
+ console.error(`ci-ubicloud --list: the gate runs no test file on this tree (lanes: ${[...opts.lanes].join(",")})`);
+ process.exit(1);
+ }
+ console.log([...files].sort().join("\n"));
+ return;
+ }
+ if (!process.env.UBICLOUD_API_KEY && !process.env.UBICLOUD_API_TOKEN) {
+ console.error("ci-ubicloud: UBICLOUD_API_KEY (or UBICLOUD_API_TOKEN) is not set");
+ process.exit(2);
+ }
+ if (!process.env.UBI_OWNER) {
+ console.error("ci-ubicloud: UBI_OWNER is not set; set it to your thread code in lowercase (e.g. UBI_OWNER=gbra106) so every VM carries its owner (UBI_OWNER_REQUIRED)");
+ process.exit(2);
+ }
+ const owner = sh("bash", [RUNNER, "owner"]).trim();
+ const sha = sh("git", ["rev-parse", "HEAD"]).trim();
+ const runId = `${new Date().toISOString().replace(/[:.]/g, "-")}-${sha.slice(0, 8)}`;
+ const runDir = join(ROOT, ".context/ci-ubicloud", runId);
+ mkdirSync(join(runDir, "logs"), { recursive: true });
+ mkdirSync(join(runDir, "failures"), { recursive: true });
+
+ const { inventory, exclusive, slowExclusive } = gateInventory(opts, log);
const tables: WeightTable[] = [
readWeightTable(LOCAL_WEIGHTS),
@@ -600,7 +633,7 @@ async function main() {
// workflow's own partition, its ARM_WORKERS workers' makespan over this run's
// measured file durations (both arms, as this lane runs them).
const armShardReport = opts.lanes.has("pg-arms")
- ? armShards(ROOT, arms.files, ARM_SHARDS).map((files, i) => ({
+ ? armShards(ROOT, inventory["pg-arms"], ARM_SHARDS).map((files, i) => ({
shard: i + 1,
files: files.length,
ms: armMakespan(files, (file) => results.get(weightKey("pg-arms", file))?.ms ?? 0),
@@ -710,7 +743,7 @@ function sortedObject(obj: Record<string, number>): Record<string, number> {
if (import.meta.main) {
process.on("exit", () => {
- if (leftReported || vmRequested) return;
+ if (leftReported || vmRequested || process.argv.includes("--list")) return;
const line = `UBICLOUD: left=0 owner=${process.env.UBI_OWNER ?? ""} vms=0 others=unknown`;
console.log(line);
if (process.env.UBI_LEFT_FILE) appendFileSync(process.env.UBI_LEFT_FILE, line + "\n");
diff --git a/scripts/pregate-remote-gate.sh b/scripts/pregate-remote-gate.sh
index 2ed87704a7..e9a8ffc4a2 100755
--- a/scripts/pregate-remote-gate.sh
+++ b/scripts/pregate-remote-gate.sh
@@ -50,6 +50,8 @@
# environment. Extra arguments go to ci:ubicloud (for example --vms 8).
# Spend is not reported (no spent=), so gstack counts it at the cap.
#
+# `--list` (below) prints the gate's test inventory for gstack's run evidence.
+#
# Seams: GBRAIN_PREGATE_GATE_CMD replaces `bun run ci:ubicloud:diff`,
# GBRAIN_PREGATE_REUSE_CMD replaces `bun scripts/ci-reuse.ts --gate`, and
# GBRAIN_PREGATE_REPO names owner/repo instead of the origin remote
@@ -58,6 +60,22 @@ set -uo pipefail
fail() { echo "pregate-remote-gate: $1" >&2; exit "${2:-1}"; }
+# --list prints the test files the gate would run on the checked-out tree, one
+# per line: the gate command with --list (ci-ubicloud.ts --list, the same
+# selection and inventory), so gstack-pregate clear --run can check that the
+# full-ci runs it is given executed every one (remote_gate_runs.inventory,
+# docs/ci-event-parity.md#remote-gate-reuse). A failing or empty inventory exits
+# non-zero. It starts no VM and needs no UBI_OWNER.
+if [ "${1:-}" = "--list" ]; then
+ shift
+ TOP=$(git rev-parse --show-toplevel 2>/dev/null) || fail "not inside a git checkout" 2
+ cd "$TOP" || exit 2
+ listed=$(bash -c "${GBRAIN_PREGATE_GATE_CMD:-bun run ci:ubicloud:diff} --list \"\$@\"" pregate-remote-gate "$@") || fail "the gate's inventory failed (exit $?); no list" 1
+ [ -n "$listed" ] || fail "the gate's inventory is empty; no list" 1
+ printf '%s\n' "$listed"
+ exit 0
+fi
+
[ -n "${UBI_OWNER:-}" ] || fail "UBI_OWNER is not set; set it to your thread code in lowercase (e.g. UBI_OWNER=gbra106) so the gate's VMs are yours to tear down" 2
export UBI_OWNER
diff --git a/scripts/run-with-receipt.sh b/scripts/run-with-receipt.sh
index 075e42548e..886329e03c 100755
--- a/scripts/run-with-receipt.sh
+++ b/scripts/run-with-receipt.sh
@@ -8,7 +8,7 @@
# scripts/lib/test-env.sh (receipts_init) and scripts/ci-executed-counts.ts.
#
# Usage:
-# bash scripts/run-with-receipt.sh <lane> <tag> -- bun test --timeout=N [args] <files...>
+# bash scripts/run-with-receipt.sh <lane> <tag> -- bun [--no-env-file] test --timeout=N [args] <files...>
#
# <lane> names the identity lane (unit, slow, tier1, ...); <tag> must be unique
# per invocation inside that lane. Test files are the *.test.ts arguments.
@@ -17,8 +17,13 @@
set -euo pipefail
+bun_flag=""
+if [ "${5:-}" = "--no-env-file" ]; then
+ bun_flag=--no-env-file
+ set -- "$1" "$2" "$3" "$4" "${@:6}"
+fi
if [ "$#" -lt 5 ] || [ "$3" != "--" ] || [ "$4" != "bun" ] || [ "$5" != "test" ]; then
- echo "usage: bash scripts/run-with-receipt.sh <lane> <tag> -- bun test --timeout=N [args] <files...>" >&2
+ echo "usage: bash scripts/run-with-receipt.sh <lane> <tag> -- bun [--no-env-file] test --timeout=N [args] <files...>" >&2
exit 2
fi
lane="$1"
@@ -50,6 +55,6 @@ fi
receipt_begin primary "$tag" "" "" "" "${files[@]}"
rc=0
-bun test ${RECEIPT_ARGS[@]+"${RECEIPT_ARGS[@]}"} "$@" || rc=$? # "$@" carries --timeout (refused above otherwise)
+bun $bun_flag test ${RECEIPT_ARGS[@]+"${RECEIPT_ARGS[@]}"} "$@" || rc=$? # "$@" carries --timeout (refused above otherwise)
receipt_end "$rc"
exit "$rc"
diff --git a/test/scripts/ci-ubicloud-list.test.ts b/test/scripts/ci-ubicloud-list.test.ts
new file mode 100644
index 0000000000..7aca9bf400
--- /dev/null
+++ b/test/scripts/ci-ubicloud-list.test.ts
@@ -0,0 +1,74 @@
+/**
+ * scripts/ci-ubicloud.ts --list — the remote gate's test inventory.
+ *
+ * Protects: the file list gstack-pregate clear --run checks full-ci runs
+ * against (scripts/pregate-remote-gate.sh --list, docs/ci-event-parity.md#remote-gate-reuse).
+ * Fails when: --list stops printing exactly the files the gate's lanes run
+ * (the same dry-run discovery), prints anything else on stdout, needs a key or
+ * an owner, exits 0 with nothing to list, or the keyed-only E2E files the gate
+ * leaves out drift from the files e2e.yml's keyed Tier 2 job runs; or a gate file
+ * that GitHub runs outside the receipt lanes (the reconciliation crash suites)
+ * loses its receipt, so no full-ci run could cover it.
+ */
+import { describe, expect, test } from 'bun:test';
+import { spawnSync } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { load } from 'js-yaml';
+import { KEYED_E2E } from '../../scripts/ci-ubicloud.ts';
+import { readArmsList } from '../../scripts/postgres-unit-arms.ts';
+
+const root = join(import.meta.dir, '../..');
+const env = { ...process.env, UBICLOUD_API_KEY: undefined, UBICLOUD_API_TOKEN: undefined, UBI_OWNER: undefined, UBI_LEFT_FILE: undefined };
+const list = (...args: string[]) => spawnSync('bun', ['scripts/ci-ubicloud.ts', ...args, '--list'], { cwd: root, encoding: 'utf8', env });
+const dry = (script: string, flag = '--dry-run-list') => spawnSync('bash', [`scripts/${script}`, flag], { cwd: root, encoding: 'utf8' }).stdout.split(/\s+/).filter(Boolean);
+type Job = { steps?: Array<{ run?: string; uses?: string; with?: Record<string, string>; env?: Record<string, string> }> };
+const workflow = (name: string) => load(readFileSync(join(root, '.github/workflows', name), 'utf8')) as { jobs: Record<string, Job> };
+const testFiles = (job: Job) => (job.steps ?? []).flatMap(s => s.run?.match(/test\/\S+\.test\.ts/g) ?? []);
+
+describe('ci-ubicloud.ts --list', () => {
+ test('prints the files of every lane, sorted and once each, with no key, owner or VM', () => {
+ const r = list();
+ expect(r.status, r.stderr).toBe(0);
+ const expected = new Set([
+ ...dry('run-unit-shard.sh'), ...dry('run-serial-tests.sh'), ...dry('run-serial-tests.sh', '--dry-run-list-exclusive'),
+ ...dry('run-slow-tests.sh'), ...dry('run-slow-tests.sh', '--dry-run-list-exclusive'),
+ ...readArmsList(root).files, ...dry('run-e2e.sh').filter(f => !KEYED_E2E.includes(f)),
+ ]);
+ expect(r.stdout).toBe([...expected].sort().join('\n') + '\n');
+ expect(r.stdout).not.toContain('UBICLOUD:');
+ expect(r.stdout.split('\n').filter(Boolean).every(f => /^(test|evals)\/.+\.test\.ts$/.test(f))).toBe(true);
+ });
+
+ test('a selection with no test lane exits 1 with an empty list', () => {
+ const r = list('--lanes', 'gitleaks,verify');
+ expect(r.status).toBe(1);
+ expect(r.stdout).toBe('');
+ expect(r.stderr).toContain('the gate runs no test file on this tree (lanes: gitleaks,verify)');
+ });
+
+ test('the gate leaves out exactly the keyed-only files that e2e.yml runs only in its keyed Tier 2 job', () => {
+ expect(testFiles(workflow('e2e.yml').jobs.tier2!).sort()).toEqual([...KEYED_E2E].sort());
+ const e2e = list('--lanes', 'e2e');
+ expect(e2e.status, e2e.stderr).toBe(0);
+ for (const file of KEYED_E2E) {
+ expect(dry('run-e2e.sh')).toContain(file);
+ expect(e2e.stdout.split('\n')).not.toContain(file);
+ }
+ });
+
+ test('the reconciliation crash suites run with receipts, so the Test run manifest lists them', () => {
+ const job = workflow('persistence-validation.yml').jobs.reconciliation!;
+ const runs = job.steps!.filter(s => /reconcile-crash/.test(s.run ?? ''));
+ expect(runs.length).toBe(1);
+ const lines = runs[0]!.run!.split('\n').filter(l => /reconcile-crash/.test(l));
+ expect(lines.length).toBe(2);
+ for (const line of lines) expect(line).toMatch(/bash scripts\/run-with-receipt\.sh reconciliation (postgres|pglite)-bun-\$\{\{ matrix\.bun \}\} -- bun --no-env-file test /);
+ expect(runs[0]!.env!.GBRAIN_TEST_RECEIPT_DIR).toBe('${{ runner.temp }}/receipts');
+ const upload = job.steps!.find(s => s.uses?.startsWith('actions/upload-artifact@') && s.with?.name?.startsWith('receipts-'))!;
+ expect(upload.with!.path).toBe('${{ runner.temp }}/receipts');
+ const gate = list().stdout.split('\n').filter(f => f.includes('reconcile-crash'));
+ expect(gate.length).toBeGreaterThan(0);
+ for (const file of gate) expect(lines.some(l => l.includes(file) || (file.endsWith('.slow.test.ts') && l.includes('test/reconcile-crash-*.slow.test.ts')))).toBe(true);
+ });
+});
diff --git a/test/scripts/full-ci-workflow.test.ts b/test/scripts/full-ci-workflow.test.ts
index 953acb006f..524bfe37ed 100644
--- a/test/scripts/full-ci-workflow.test.ts
+++ b/test/scripts/full-ci-workflow.test.ts
@@ -105,6 +105,67 @@ describe('full CI on pull requests', () => {
}
});
+ test('tree.json lists the executed test files only from a complete manifest of the same run, attempt and commit', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'gbrain-full-suite-files-'));
+ try {
+ const git = (...args: string[]) => spawnSync('git', ['-c', 'user.name=t', '-c', '[email protected]', ...args], { cwd: dir, encoding: 'utf8' });
+ git('init', '-q');
+ writeFileSync(join(dir, 'a.txt'), 'a\n');
+ git('add', 'a.txt');
+ git('commit', '-qm', 'a');
+ const sha = git('rev-parse', 'HEAD').stdout.trim();
+ const manifest = (over: Record<string, unknown> = {}) => ({
+ schema: 'gbrain-ci-manifest/v1', workflow: 'Test', event: 'workflow_dispatch', sha, run_id: 42, run_attempt: 2, complete: true, problems: [],
+ files: [
+ { file: 'test/b.test.ts', lane: 'unit', arm: '', executed: 3, failed: 0, skipped: 1 },
+ { file: 'test/a.test.ts', lane: 'unit', arm: '', executed: 2, failed: 0, skipped: 0 },
+ { file: 'test/a.test.ts', lane: 'pg-arms-bun-1.4.4', arm: 'postgres', executed: 2, failed: 0, skipped: 0 },
+ { file: 'test/e2e/c.test.ts', lane: 'e2e', arm: '', executed: 0, failed: 0, skipped: 4 },
+ { file: 'verify', lane: 'verify', arm: '', executed: 1, failed: 0, skipped: 0 },
+ ],
+ failures: [], ...over,
+ });
+ const record = (doc: unknown) => {
+ const path = join(dir, 'ci-manifest.json');
+ if (doc === undefined) rmSync(path, { force: true }); else writeFileSync(path, JSON.stringify(doc));
+ const out = join(dir, 'out');
+ const result = bash(`bash ${join(root, 'scripts/ci-full-suite-tree.sh')}`,
+ { ARTIFACT_DIR: out, RUN_ID: '42', RUN_ATTEMPT: '2', WORKFLOW: 'Test', EVENT: 'workflow_dispatch', MANIFEST: path, GITHUB_STEP_SUMMARY: join(dir, 'summary.md') }, dir);
+ expect(result.status, result.stderr).toBe(0);
+ return { tree: JSON.parse(readFileSync(join(out, 'tree.json'), 'utf8')), stdout: result.stdout };
+ };
+ const base = { tree: git('rev-parse', 'HEAD^{tree}').stdout.trim(), sha, workflow: 'Test', event: 'workflow_dispatch', run_id: 42, run_attempt: 2, profile: 'full' };
+ expect(record(manifest()).tree).toEqual({ ...base, files: ['test/a.test.ts', 'test/b.test.ts', 'test/e2e/c.test.ts'], tests: 8 });
+ for (const [doc, why] of [
+ [undefined, 'no ci-manifest.json from this run'],
+ [manifest({ run_attempt: 1 }), 'belongs to run 42 attempt 1, not run 42 attempt 2 (re-run the whole workflow, not only its failed jobs)'],
+ [manifest({ run_id: 41 }), 'belongs to run 41 attempt 2'],
+ [manifest({ sha: 'f'.repeat(40) }), `belongs to commit ${'f'.repeat(40)}`],
+ [manifest({ complete: false, problems: ['unit shard 3 has no receipt'] }), 'is incomplete: unit shard 3 has no receipt'],
+ [manifest({ schema: 'other/v0' }), 'has schema other/v0'],
+ [manifest({ files: [{ file: 'verify', lane: 'verify', arm: '', executed: 1, failed: 0, skipped: 0 }] }), 'lists no test file'],
+ ] as const) {
+ const { tree, stdout } = record(doc);
+ expect(tree).toEqual(base);
+ expect(stdout).toContain(`::warning::tree.json records no executed test files: `);
+ expect(stdout).toContain(why);
+ }
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ for (const [w, job] of [[unit, 'full-suite'], [e2e, 'e2e-full-suite']] as const) {
+ expect(w.jobs[job]!.needs).toContain('executed-receipts');
+ const steps = w.jobs[job]!.steps!;
+ const download = steps.findIndex(s => s.name === 'Download the execution manifest');
+ expect(steps[download]!.with).toMatchObject({ name: 'ci-manifest', path: '${{ runner.temp }}/ci-manifest' });
+ expect(steps[download]!['continue-on-error']).toBe(true);
+ expect(download).toBeLessThan(steps.findIndex(s => s.name === 'Record the tested tree'));
+ expect(step(w.jobs[job]!, 'Record the tested tree').env!.MANIFEST).toBe('${{ runner.temp }}/ci-manifest/ci-manifest.json');
+ const ledger = w.jobs['executed-receipts']!.steps!.find(s => s.run?.includes('scripts/ci-manifest.ts'))!;
+ expect(ledger.run).toContain('--out "$RUNNER_TEMP/ci-manifest.json"');
+ }
+ });
+
test('E2E never runs on pull_request events; the full-ci label dispatches Test and E2E on a same-repository head', () => {
expect(e2e.on.pull_request).toBeUndefined();
expect(dispatcher.on.pull_request).toEqual({ branches: ['master'], types: ['labeled', 'synchronize', 'reopened'] });
diff --git a/test/scripts/pregate-remote-gate.test.ts b/test/scripts/pregate-remote-gate.test.ts
index 69867f4726..16d3312586 100644
--- a/test/scripts/pregate-remote-gate.test.ts
+++ b/test/scripts/pregate-remote-gate.test.ts
@@ -18,6 +18,9 @@
* Seams: GBRAIN_PREGATE_GATE_CMD replaces `bun run ci:ubicloud:diff`,
* GBRAIN_PREGATE_REUSE_CMD replaces `bun scripts/ci-reuse.ts --gate`,
* GBRAIN_PREGATE_REPO names the repository.
+ * --list: prints the gate command's --list output (the inventory gstack's
+ * clear --run checks runs against) without UBI_OWNER or a gate; a failing or
+ * empty inventory exits non-zero.
* VM cleanup: the gate's UBICLOUD: left= line (UBI_LEFT_FILE) must say 0, or
* the run is PREGATE_REMOTE_INFRA (exit 4, no receipt), never a test result;
* a missing line counts as not confirmed.
@@ -263,3 +266,52 @@ describe('pregate-remote-gate.sh', () => {
});
});
});
+
+describe('pregate-remote-gate.sh --list', () => {
+ /** Runs --list with `gate` as the gate command and no UBI_OWNER. */
+ const list = (gate: string | undefined, args: string[] = [], env: Record<string, string> = {}) => {
+ const r = spawnSync('bash', [SCRIPT, '--list', ...args], {
+ cwd: repo, encoding: 'utf8',
+ env: { ...process.env, GIT_CONFIG_NOSYSTEM: '1', UBI_OWNER: undefined, UBI_LEFT_FILE: undefined, GBRAIN_PREGATE_GATE_CMD: gate, ...env },
+ });
+ return { code: r.status, out: r.stdout, err: r.stderr };
+ };
+
+ it("prints the gate command's --list output at the checkout top, with the extra arguments, needing no UBI_OWNER", () => {
+ mkdirSync(join(repo, 'sub'));
+ const gate = join(repo, '.git', 'gate.sh');
+ writeFileSync(gate, 'echo "args: $*" >&2; echo "cwd: $PWD" >&2; printf "%s\\n" test/a.test.ts test/e2e/b.test.ts\n');
+ const r = spawnSync('bash', [SCRIPT, '--list', '--lanes', 'unit'], {
+ cwd: join(repo, 'sub'), encoding: 'utf8',
+ env: { ...process.env, GIT_CONFIG_NOSYSTEM: '1', UBI_OWNER: undefined, GBRAIN_PREGATE_GATE_CMD: `bash ${gate}` },
+ });
+ expect(r.status, r.stderr).toBe(0);
+ expect(r.stdout).toBe('test/a.test.ts\ntest/e2e/b.test.ts\n');
+ expect(r.stderr).toContain('args: --list --lanes unit');
+ expect(r.stderr).toContain(`cwd: ${repo}`);
+ expect(r.stderr).not.toContain('UBI_OWNER');
+ });
+
+ it('lists through the development gate, bun run ci:ubicloud:diff --list, by default', () => {
+ const bin = join(repo, '.git', 'fake-bin');
+ mkdirSync(bin);
+ writeFileSync(join(bin, 'bun'), '#!/usr/bin/env bash\necho "fake-bun: $*" >&2\necho test/a.test.ts\n');
+ chmodSync(join(bin, 'bun'), 0o755);
+ const r = list(undefined, [], { PATH: `${bin}:${process.env.PATH}` });
+ expect(r.code, r.err).toBe(0);
+ expect(r.err).toContain('fake-bun: run ci:ubicloud:diff --list');
+ expect(r.out).toBe('test/a.test.ts\n');
+ });
+
+ it('an empty or failing inventory exits non-zero with no list, and never starts the gate', () => {
+ const empty = list('true');
+ expect(empty.code).toBe(1);
+ expect(empty.out).toBe('');
+ expect(empty.err).toContain("the gate's inventory is empty");
+ const failing = list('echo test/a.test.ts; exit 3; echo');
+ expect(failing.code).toBe(1);
+ expect(failing.out).toBe('');
+ expect(failing.err).toContain("the gate's inventory failed (exit 3)");
+ for (const r of [empty, failing]) expect(r.out + r.err).not.toContain('PREGATE_REMOTE');
+ });
+});
diff --git a/test/scripts/run-with-receipt.test.ts b/test/scripts/run-with-receipt.test.ts
index 2724c68d62..18a1a9d04e 100644
--- a/test/scripts/run-with-receipt.test.ts
+++ b/test/scripts/run-with-receipt.test.ts
@@ -56,4 +56,17 @@ describe('run-with-receipt.sh', () => {
expect(run(root, ['slow', '--', 'bun', 'test']).status).toBe(2);
} finally { rmSync(root, { recursive: true, force: true }); }
});
+ test('passes the bun flag --no-env-file through and still records the receipt', () => {
+ const root = sandbox();
+ try {
+ const receipts = join(root, 'receipts');
+ const r = run(root, ['reconciliation', 'pglite', '--', 'bun', '--no-env-file', 'test', '--timeout=60000', 'test/a.test.ts'], { GBRAIN_TEST_RECEIPT_DIR: receipts });
+ expect(r.status, r.stdout + r.stderr).toBe(0);
+ const side = buildSide('local', loadReceiptDir(receipts));
+ expect(side.issues).toEqual([]);
+ expect([...side.identities.values()].map(i => [i.lane, i.file, i.status])).toEqual([['reconciliation', 'test/a.test.ts', 'pass'], ['reconciliation', 'test/a.test.ts', 'skip']]);
+ expect(run(root, ['slow', 'perf-a', '--', 'bun', '--no-env-file', 'test']).status).toBe(2);
+ expect(run(root, ['slow', 'perf-a', '--', 'bun', '--no-env-file', 'x', '--timeout=60000', 'test/a.test.ts']).status).toBe(2);
+ } finally { rmSync(root, { recursive: true, force: true }); }
+ });
});