An open source design system that's fully customizable and agent ready
TypeScript
ci: name which command of a multi-command test step failed (#9779)
Cindy Zhang91d8d6fe7a4f
diff --git a/.github/scripts/ci-lab-only-routing.test.mjs b/.github/scripts/ci-lab-only-routing.test.mjsindex 6e66c88d21..9a0acac056 100644--- a/.github/scripts/ci-lab-only-routing.test.mjs+++ b/.github/scripts/ci-lab-only-routing.test.mjs@@ -128,7 +128,7 @@ describe('Lab-only lane (spec:AST-030 FR26)', () => {// The Lab tests, not the module graph's --changed set, then the guards.expect(labBranch).not.toContain('pnpm test:ui --changed');expect(labBranch).toContain(- 'node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"',+ '-- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1',);expect(labBranch).toContain('pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"',diff --git a/.github/scripts/ci-test-routing.test.mjs b/.github/scripts/ci-test-routing.test.mjsindex b8414b76d0..fd4245e5bc 100644--- a/.github/scripts/ci-test-routing.test.mjs+++ b/.github/scripts/ci-test-routing.test.mjs@@ -183,22 +183,22 @@ describe.each(Object.entries(WORKFLOWS))(' # A change only to the Lab package runs every Lab test across the',' # same legs, and the guards below as any scoped run does',' # (spec:AST-030 FR26).',- ` pnpm test:ui --passWithNoTests ${SHARD_ARG} packages/lab/`,- ' node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"',+ ` .github/scripts/run-named.sh "Lab tests" -- pnpm test:ui --passWithNoTests ${SHARD_ARG} packages/lab/`,+ ' .github/scripts/run-named.sh "List the tests that read source files" --stdout "$RUNNER_TEMP/fs-guards.txt" -- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1',' mapfile -t GUARDS < "$RUNNER_TEMP/fs-guards.txt"',' if [ "${#GUARDS[@]}" -gt 0 ]; then',- ` pnpm test:ui --passWithNoTests ${SHARD_ARG} "\${GUARDS[@]}"`,+ ` .github/scripts/run-named.sh "Tests that read source files" -- pnpm test:ui --passWithNoTests ${SHARD_ARG} "\${GUARDS[@]}"`,' fi','elif [ "$SCOPED" = true ]; then',- ` pnpm test:ui --changed=HEAD^1 --passWithNoTests ${SHARD_ARG}`,+ ` .github/scripts/run-named.sh "Tests the change reaches" -- pnpm test:ui --changed=HEAD^1 --passWithNoTests ${SHARD_ARG}`,' # Tests no module graph relates to the change run too: guards',' # that read sources, and tests that load a changed package from',' # its build (by name or its dist/).',' # The list goes through a file so a failing lister fails the step.',- ' node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"',+ ' .github/scripts/run-named.sh "List the tests that read source files" --stdout "$RUNNER_TEMP/fs-guards.txt" -- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1',' mapfile -t GUARDS < "$RUNNER_TEMP/fs-guards.txt"',' if [ "${#GUARDS[@]}" -gt 0 ]; then',- ` pnpm test:ui --passWithNoTests ${SHARD_ARG} "\${GUARDS[@]}"`,+ ` .github/scripts/run-named.sh "Tests that read source files" -- pnpm test:ui --passWithNoTests ${SHARD_ARG} "\${GUARDS[@]}"`,' fi','else',` pnpm test:ui ${SHARD_ARG}`,@@ -215,15 +215,15 @@ describe.each(Object.entries(WORKFLOWS))(expect(node.run.trim()).toBe(['if [ "$SCOPED" = true ]; then',- ` pnpm test:node --changed=HEAD^1 --passWithNoTests ${SHARD_ARG}`,+ ` .github/scripts/run-named.sh "Tests the change reaches" -- pnpm test:node --changed=HEAD^1 --passWithNoTests ${SHARD_ARG}`,' # Tests no module graph relates to the change run too: guards',' # that read sources, and tests that load a changed package from',' # its build (by name or its dist/).',' # The list goes through a file so a failing lister fails the step.',- ' node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"',+ ' .github/scripts/run-named.sh "List the tests that read source files" --stdout "$RUNNER_TEMP/fs-guards.txt" -- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1',' mapfile -t GUARDS < "$RUNNER_TEMP/fs-guards.txt"',' if [ "${#GUARDS[@]}" -gt 0 ]; then',- ` pnpm test:node --passWithNoTests ${SHARD_ARG} "\${GUARDS[@]}"`,+ ` .github/scripts/run-named.sh "Tests that read source files" -- pnpm test:node --passWithNoTests ${SHARD_ARG} "\${GUARDS[@]}"`,' fi','else',` pnpm test:node ${SHARD_ARG}`,diff --git a/.github/scripts/run-named.sh b/.github/scripts/run-named.shnew file mode 100755index 0000000000..24eed1b1eb--- /dev/null+++ b/.github/scripts/run-named.sh@@ -0,0 +1,32 @@+#!/usr/bin/env bash+# Copyright (c) Meta Platforms, Inc. and affiliates.+#+# Runs one command of a CI step that runs several, inside a named log group.+# When the command fails, an error annotation names it, so a passing summary+# from an earlier command never reads as the step's result.+#+# run-named.sh "<label>" [--stdout <file>] -- <command> [args...]+#+# With --stdout, the command's standard output goes to <file>; its standard+# error stays in the log.+set -u+label=$1+shift+out=+if [ "${1:-}" = --stdout ]; then+ out=$2+ shift 2+fi+if [ "${1:-}" = -- ]; then shift; fi+echo "::group::${label}"+if [ -n "$out" ]; then+ "$@" > "$out"+else+ "$@"+fi+status=$?+echo "::endgroup::"+if [ "$status" -ne 0 ]; then+ echo "::error title=${label} failed::${label} exited with status ${status}. Its output is in the \"${label}\" group of this step's log."+fi+exit "$status"diff --git a/.github/scripts/run-named.test.mjs b/.github/scripts/run-named.test.mjsnew file mode 100644index 0000000000..881d079f96--- /dev/null+++ b/.github/scripts/run-named.test.mjs@@ -0,0 +1,99 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file run-named.sh names the failing command of a CI step that runs several,+ * and the UI and Node test steps run each of their commands through it.+ */++import {spawnSync} from 'node:child_process';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import {describe, expect, it} from 'vitest';+import YAML from 'yaml';++const root = path.resolve(import.meta.dirname, '../..');+const script = path.join(root, '.github/scripts/run-named.sh');+const run = (...args) =>+ spawnSync('bash', [script, ...args], {encoding: 'utf8'});++describe('run-named.sh', () => {+ it('groups a passing command and adds no error', () => {+ const result = run('Tests the change reaches', '--', 'echo', 'all passed');+ expect(result.status).toBe(0);+ expect(result.stdout).toBe(+ '::group::Tests the change reaches\nall passed\n::endgroup::\n',+ );+ });++ it('names a failing command in an error annotation and keeps its exit status', () => {+ const result = run(+ 'Tests that read source files',+ '--',+ 'bash',+ '-c',+ 'exit 3',+ );+ expect(result.status).toBe(3);+ expect(result.stdout).toContain(+ '::error title=Tests that read source files failed::Tests that read source files exited with status 3.',+ );+ // The error follows the group, so the log view shows it outside it.+ expect(result.stdout.indexOf('::endgroup::')).toBeLessThan(+ result.stdout.indexOf('::error'),+ );+ });++ it('sends standard output to a file with --stdout, and fails when the command fails', () => {+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'run-named-'));+ try {+ const file = path.join(dir, 'list.txt');+ const ok = run('List', '--stdout', file, '--', 'printf', 'a\\nb\\n');+ expect(ok.status).toBe(0);+ expect(fs.readFileSync(file, 'utf8')).toBe('a\nb\n');+ const failed = run(+ 'List',+ '--stdout',+ file,+ '--',+ 'bash',+ '-c',+ 'echo boom >&2; exit 1',+ );+ expect(failed.status).toBe(1);+ expect(failed.stderr).toContain('boom');+ expect(failed.stdout).toContain('::error title=List failed::');+ } finally {+ fs.rmSync(dir, {recursive: true, force: true});+ }+ });+});++describe('the UI and Node test steps', () => {+ const workflow = YAML.parse(+ fs.readFileSync(path.join(root, '.github/workflows/ci.yml'), 'utf8'),+ );+ const steps = [+ ['test-ui', 'Run this shard of the UI tests'],+ ['test-node', 'Run this shard of the Node tests'],+ ].map(([job, name]) =>+ workflow.jobs[job].steps.find(step => step.name === name),+ );++ it('run every command of a multi-command branch through run-named.sh', () => {+ for (const step of steps) {+ const commands = step.run+ .split('\n')+ .map(line => line.trim())+ .filter(line => /(pnpm test:|fs-guard-tests\.mjs)/.test(line));+ // The full run is one command; every other command is named.+ const multi = commands.filter(+ line => !/^pnpm test:(ui|node) --shard=/.test(line),+ );+ expect(multi.length).toBeGreaterThan(0);+ for (const line of multi) {+ expect(line).toMatch(/^\.github\/scripts\/run-named\.sh "[^"]+" /);+ }+ }+ });+});diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex b97a2777f3..f5deec5d20 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -848,22 +848,22 @@ jobs:# A change only to the Lab package runs every Lab test across the# same legs, and the guards below as any scoped run does# (spec:AST-030 FR26).- pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} packages/lab/- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"+ .github/scripts/run-named.sh "Lab tests" -- pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} packages/lab/+ .github/scripts/run-named.sh "List the tests that read source files" --stdout "$RUNNER_TEMP/fs-guards.txt" -- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1mapfile -t GUARDS < "$RUNNER_TEMP/fs-guards.txt"if [ "${#GUARDS[@]}" -gt 0 ]; then- pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"+ .github/scripts/run-named.sh "Tests that read source files" -- pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"fielif [ "$SCOPED" = true ]; then- pnpm test:ui --changed=HEAD^1 --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }}+ .github/scripts/run-named.sh "Tests the change reaches" -- pnpm test:ui --changed=HEAD^1 --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }}# Tests no module graph relates to the change run too: guards# that read sources, and tests that load a changed package from# its build (by name or its dist/).# The list goes through a file so a failing lister fails the step.- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"+ .github/scripts/run-named.sh "List the tests that read source files" --stdout "$RUNNER_TEMP/fs-guards.txt" -- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1mapfile -t GUARDS < "$RUNNER_TEMP/fs-guards.txt"if [ "${#GUARDS[@]}" -gt 0 ]; then- pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"+ .github/scripts/run-named.sh "Tests that read source files" -- pnpm test:ui --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"fielsepnpm test:ui --shard=${{ matrix.shard }}/${{ strategy.job-total }}@@ -976,15 +976,15 @@ jobs:SCOPED: ${{ github.event_name == 'pull_request' && steps.scope.outputs.full_suite == 'false' }}run: |if [ "$SCOPED" = true ]; then- pnpm test:node --changed=HEAD^1 --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }}+ .github/scripts/run-named.sh "Tests the change reaches" -- pnpm test:node --changed=HEAD^1 --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }}# Tests no module graph relates to the change run too: guards# that read sources, and tests that load a changed package from# its build (by name or its dist/).# The list goes through a file so a failing lister fails the step.- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1 > "$RUNNER_TEMP/fs-guards.txt"+ .github/scripts/run-named.sh "List the tests that read source files" --stdout "$RUNNER_TEMP/fs-guards.txt" -- node .github/scripts/fs-guard-tests.mjs --changed HEAD^1mapfile -t GUARDS < "$RUNNER_TEMP/fs-guards.txt"if [ "${#GUARDS[@]}" -gt 0 ]; then- pnpm test:node --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"+ .github/scripts/run-named.sh "Tests that read source files" -- pnpm test:node --passWithNoTests --shard=${{ matrix.shard }}/${{ strategy.job-total }} "${GUARDS[@]}"fielsepnpm test:node --shard=${{ matrix.shard }}/${{ strategy.job-total }}