Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci: name which command of a multi-command test step failed (#9779)

Cindy Zhang91d8d6fe7a4f
diff --git a/.github/scripts/ci-lab-only-routing.test.mjs b/.github/scripts/ci-lab-only-routing.test.mjs
index 6e66c88d21..9a0acac056 100644
--- a/.github/scripts/ci-lab-only-routing.test.mjs
+++ b/.github/scripts/ci-lab-only-routing.test.mjs
@@ -128,7 +128,7 @@ describe('Lab-only lane (spec:AST-030 FR26)', () => {
// The Lab tests, not the module graph's --changed set, then the guards.
expect(labBranch).not.toContain('pnpm test:ui --changed');
expect(labBranch).toContain(
- 'node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"',
+ '-- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1',
);
expect(labBranch).toContain(
'pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"',
diff --git a/.github/scripts/ci-test-routing.test.mjs b/.github/scripts/ci-test-routing.test.mjs
index b8414b76d0..fd4245e5bc 100644
--- a/.github/scripts/ci-test-routing.test.mjs
+++ b/.github/scripts/ci-test-routing.test.mjs
@@ -183,22 +183,22 @@ describe.each(Object.entries(WORKFLOWS))(
' # A change only to the Lab package runs every Lab test across the',
' # same legs, and the guards below as any scoped run does',
' # (spec:AST-030 FR26).',
- ` pnpm test:ui --passWithNoTests ${SHARD_ARG} packages/lab/`,
- ' node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"',
+ ` .github/scripts/run-named.sh "Lab tests" -- pnpm test:ui --passWithNoTests ${SHARD_ARG} packages/lab/`,
+ ' .github/scripts/run-named.sh "List the tests that read source files" --stdout "$RUNNER_TEMP/fs-guards.txt" -- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1',
' mapfile -t GUARDS < "$RUNNER_TEMP/fs-guards.txt"',
' if [ "${#GUARDS[@]}" -gt 0 ]; then',
- ` pnpm test:ui --passWithNoTests ${SHARD_ARG} "\${GUARDS[@]}"`,
+ ` .github/scripts/run-named.sh "Tests that read source files" -- pnpm test:ui --passWithNoTests ${SHARD_ARG} "\${GUARDS[@]}"`,
' fi',
'elif [ "$SCOPED" = true ]; then',
- ` pnpm test:ui --changed=HEAD^1 --passWithNoTests ${SHARD_ARG}`,
+ ` .github/scripts/run-named.sh "Tests the change reaches" -- pnpm test:ui --changed=HEAD^1 --passWithNoTests ${SHARD_ARG}`,
' # Tests no module graph relates to the change run too: guards',
' # that read sources, and tests that load a changed package from',
' # its build (by name or its dist/).',
' # The list goes through a file so a failing lister fails the step.',
- ' node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"',
+ ' .github/scripts/run-named.sh "List the tests that read source files" --stdout "$RUNNER_TEMP/fs-guards.txt" -- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1',
' mapfile -t GUARDS < "$RUNNER_TEMP/fs-guards.txt"',
' if [ "${#GUARDS[@]}" -gt 0 ]; then',
- ` pnpm test:ui --passWithNoTests ${SHARD_ARG} "\${GUARDS[@]}"`,
+ ` .github/scripts/run-named.sh "Tests that read source files" -- pnpm test:ui --passWithNoTests ${SHARD_ARG} "\${GUARDS[@]}"`,
' fi',
'else',
` pnpm test:ui ${SHARD_ARG}`,
@@ -215,15 +215,15 @@ describe.each(Object.entries(WORKFLOWS))(
expect(node.run.trim()).toBe(
[
'if [ "$SCOPED" = true ]; then',
- ` pnpm test:node --changed=HEAD^1 --passWithNoTests ${SHARD_ARG}`,
+ ` .github/scripts/run-named.sh "Tests the change reaches" -- pnpm test:node --changed=HEAD^1 --passWithNoTests ${SHARD_ARG}`,
' # Tests no module graph relates to the change run too: guards',
' # that read sources, and tests that load a changed package from',
' # its build (by name or its dist/).',
' # The list goes through a file so a failing lister fails the step.',
- ' node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"',
+ ' .github/scripts/run-named.sh "List the tests that read source files" --stdout "$RUNNER_TEMP/fs-guards.txt" -- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1',
' mapfile -t GUARDS < "$RUNNER_TEMP/fs-guards.txt"',
' if [ "${#GUARDS[@]}" -gt 0 ]; then',
- ` pnpm test:node --passWithNoTests ${SHARD_ARG} "\${GUARDS[@]}"`,
+ ` .github/scripts/run-named.sh "Tests that read source files" -- pnpm test:node --passWithNoTests ${SHARD_ARG} "\${GUARDS[@]}"`,
' fi',
'else',
` pnpm test:node ${SHARD_ARG}`,
diff --git a/.github/scripts/run-named.sh b/.github/scripts/run-named.sh
new file mode 100755
index 0000000000..24eed1b1eb
--- /dev/null
+++ b/.github/scripts/run-named.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Runs one command of a CI step that runs several, inside a named log group.
+# When the command fails, an error annotation names it, so a passing summary
+# from an earlier command never reads as the step's result.
+#
+# run-named.sh "<label>" [--stdout <file>] -- <command> [args...]
+#
+# With --stdout, the command's standard output goes to <file>; its standard
+# error stays in the log.
+set -u
+label=$1
+shift
+out=
+if [ "${1:-}" = --stdout ]; then
+ out=$2
+ shift 2
+fi
+if [ "${1:-}" = -- ]; then shift; fi
+echo "::group::${label}"
+if [ -n "$out" ]; then
+ "$@" > "$out"
+else
+ "$@"
+fi
+status=$?
+echo "::endgroup::"
+if [ "$status" -ne 0 ]; then
+ echo "::error title=${label} failed::${label} exited with status ${status}. Its output is in the \"${label}\" group of this step's log."
+fi
+exit "$status"
diff --git a/.github/scripts/run-named.test.mjs b/.github/scripts/run-named.test.mjs
new file mode 100644
index 0000000000..881d079f96
--- /dev/null
+++ b/.github/scripts/run-named.test.mjs
@@ -0,0 +1,99 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file run-named.sh names the failing command of a CI step that runs several,
+ * and the UI and Node test steps run each of their commands through it.
+ */
+
+import {spawnSync} from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import {describe, expect, it} from 'vitest';
+import YAML from 'yaml';
+
+const root = path.resolve(import.meta.dirname, '../..');
+const script = path.join(root, '.github/scripts/run-named.sh');
+const run = (...args) =>
+ spawnSync('bash', [script, ...args], {encoding: 'utf8'});
+
+describe('run-named.sh', () => {
+ it('groups a passing command and adds no error', () => {
+ const result = run('Tests the change reaches', '--', 'echo', 'all passed');
+ expect(result.status).toBe(0);
+ expect(result.stdout).toBe(
+ '::group::Tests the change reaches\nall passed\n::endgroup::\n',
+ );
+ });
+
+ it('names a failing command in an error annotation and keeps its exit status', () => {
+ const result = run(
+ 'Tests that read source files',
+ '--',
+ 'bash',
+ '-c',
+ 'exit 3',
+ );
+ expect(result.status).toBe(3);
+ expect(result.stdout).toContain(
+ '::error title=Tests that read source files failed::Tests that read source files exited with status 3.',
+ );
+ // The error follows the group, so the log view shows it outside it.
+ expect(result.stdout.indexOf('::endgroup::')).toBeLessThan(
+ result.stdout.indexOf('::error'),
+ );
+ });
+
+ it('sends standard output to a file with --stdout, and fails when the command fails', () => {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'run-named-'));
+ try {
+ const file = path.join(dir, 'list.txt');
+ const ok = run('List', '--stdout', file, '--', 'printf', 'a\\nb\\n');
+ expect(ok.status).toBe(0);
+ expect(fs.readFileSync(file, 'utf8')).toBe('a\nb\n');
+ const failed = run(
+ 'List',
+ '--stdout',
+ file,
+ '--',
+ 'bash',
+ '-c',
+ 'echo boom >&2; exit 1',
+ );
+ expect(failed.status).toBe(1);
+ expect(failed.stderr).toContain('boom');
+ expect(failed.stdout).toContain('::error title=List failed::');
+ } finally {
+ fs.rmSync(dir, {recursive: true, force: true});
+ }
+ });
+});
+
+describe('the UI and Node test steps', () => {
+ const workflow = YAML.parse(
+ fs.readFileSync(path.join(root, '.github/workflows/ci.yml'), 'utf8'),
+ );
+ const steps = [
+ ['test-ui', 'Run this shard of the UI tests'],
+ ['test-node', 'Run this shard of the Node tests'],
+ ].map(([job, name]) =>
+ workflow.jobs[job].steps.find(step => step.name === name),
+ );
+
+ it('run every command of a multi-command branch through run-named.sh', () => {
+ for (const step of steps) {
+ const commands = step.run
+ .split('\n')
+ .map(line => line.trim())
+ .filter(line => /(pnpm test:|fs-guard-tests\.mjs)/.test(line));
+ // The full run is one command; every other command is named.
+ const multi = commands.filter(
+ line => !/^pnpm test:(ui|node) --shard=/.test(line),
+ );
+ expect(multi.length).toBeGreaterThan(0);
+ for (const line of multi) {
+ expect(line).toMatch(/^\.github\/scripts\/run-named\.sh "[^"]+" /);
+ }
+ }
+ });
+});
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b97a2777f3..f5deec5d20 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -848,22 +848,22 @@ jobs:
# A change only to the Lab package runs every Lab test across the
# same legs, and the guards below as any scoped run does
# (spec:AST-030 FR26).
- pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} packages/lab/
- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"
+ .github/scripts/run-named.sh "Lab tests" -- pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} packages/lab/
+ .github/scripts/run-named.sh "List the tests that read source files" --stdout "$RUNNER_TEMP/fs-guards.txt" -- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1
mapfile -t GUARDS < "$RUNNER_TEMP/fs-guards.txt"
if [ "${#GUARDS[@]}" -gt 0 ]; then
- pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"
+ .github/scripts/run-named.sh "Tests that read source files" -- pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"
fi
elif [ "$SCOPED" = true ]; then
- pnpm test:ui --changed=HEAD^1 --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }}
+ .github/scripts/run-named.sh "Tests the change reaches" -- pnpm test:ui --changed=HEAD^1 --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }}
# Tests no module graph relates to the change run too: guards
# that read sources, and tests that load a changed package from
# its build (by name or its dist/).
# The list goes through a file so a failing lister fails the step.
- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"
+ .github/scripts/run-named.sh "List the tests that read source files" --stdout "$RUNNER_TEMP/fs-guards.txt" -- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1
mapfile -t GUARDS < "$RUNNER_TEMP/fs-guards.txt"
if [ "${#GUARDS[@]}" -gt 0 ]; then
- pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"
+ .github/scripts/run-named.sh "Tests that read source files" -- pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"
fi
else
pnpm test:ui --shard=${{ matrix.shard }}/${{ strategy.job-total }}
@@ -976,15 +976,15 @@ jobs:
SCOPED: ${{ github.event_name == 'pull_request' && steps.scope.outputs.full_suite == 'false' }}
run: |
if [ "$SCOPED" = true ]; then
- pnpm test:node --changed=HEAD^1 --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }}
+ .github/scripts/run-named.sh "Tests the change reaches" -- pnpm test:node --changed=HEAD^1 --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }}
# Tests no module graph relates to the change run too: guards
# that read sources, and tests that load a changed package from
# its build (by name or its dist/).
# The list goes through a file so a failing lister fails the step.
- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"
+ .github/scripts/run-named.sh "List the tests that read source files" --stdout "$RUNNER_TEMP/fs-guards.txt" -- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1
mapfile -t GUARDS < "$RUNNER_TEMP/fs-guards.txt"
if [ "${#GUARDS[@]}" -gt 0 ]; then
- pnpm test:node --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"
+ .github/scripts/run-named.sh "Tests that read source files" -- pnpm test:node --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"
fi
else
pnpm test:node --shard=${{ matrix.shard }}/${{ strategy.job-total }}