Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

only re-run the failed specs for SDK component tests (#82220)

Chris Vandoodewaardfe4e2beffa32
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
index 37fac3fe45f..c84cd430759 100644
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -64,6 +64,7 @@ jobs:
# workspace where the gate step runs; an absolute path anchored to the
# workspace makes both agree regardless of cwd.
QUARANTINE_FAILURES_FILE: ${{ github.workspace }}/target/quarantine-failures.jsonl
+ RERUN_ARTIFACT_BASE: failed-tests-${{ inputs.name }}-${{ inputs.edition }}
# Cypress records video for every spec but the passing specs' videos are
# deleted in after:spec so recording is ~5-10% slowdown per shard and almost
# never yields an artifact. Failure screenshots are unaffected, and the
@@ -119,17 +120,28 @@ jobs:
id: cypress-prep
uses: ./.github/actions/prepare-cypress
- - name: Download failed test artifacts
- uses: actions/download-artifact@v8
- if: github.run_attempt != '1'
+ - name: Resolve previous attempt
+ if: ${{ github.run_attempt != '1' }}
+ run: echo "RERUN_PREV_ATTEMPT=$((GITHUB_RUN_ATTEMPT - 1))" >> "$GITHUB_ENV"
+
+ - name: Restore previously-failed specs
+ if: ${{ github.run_attempt != '1' }}
continue-on-error: true
- id: failed-specs
+ uses: actions/download-artifact@v8
with:
- name: failed-tests-${{ inputs.name }}-${{ inputs.edition }}
+ pattern: ${{ env.RERUN_ARTIFACT_BASE }}-attempt-${{ env.RERUN_PREV_ATTEMPT }}
+ path: target/rerun
- name: Get failed specs
- if: github.run_attempt != '1'
- run: echo "specs=$(cat failed-specs)" >> "$GITHUB_ENV"
+ if: ${{ github.run_attempt != '1' }}
+ run: |
+ if [ -s target/rerun/failed-specs ]; then
+ echo "Re-running only the specs that failed on attempt $RERUN_PREV_ATTEMPT:"
+ cat target/rerun/failed-specs
+ echo "specs=$(cat target/rerun/failed-specs)" >> "$GITHUB_ENV"
+ else
+ echo "No failed-spec list from attempt $RERUN_PREV_ATTEMPT; running the full suite."
+ fi
# run-tests.yml compiles this in parallel with the uberjar build. Downloading it is ~10x faster than compiling it here.
# Callers without that job (stress tests, coverage shards, cross-run re-runs) fall through to compiling as before.
@@ -232,7 +244,7 @@ jobs:
if: |
!cancelled() && (steps.split-e2e.outcome == 'failure' || steps.tagged-e2e.outcome == 'failure')
with:
- name: cypress-recording-${{ inputs.name }}-${{ inputs.edition }}
+ name: cypress-recording-${{ inputs.name }}-${{ inputs.edition }}-attempt-${{ github.run_attempt }}
path: |
./cypress
./logs/test.log
@@ -242,8 +254,9 @@ jobs:
uses: actions/upload-artifact@v7
if: |
!cancelled() && (steps.split-e2e.outcome == 'failure' || steps.tagged-e2e.outcome == 'failure')
+ continue-on-error: true
with:
- name: failed-tests-${{ inputs.name }}-${{ inputs.edition }}
+ name: ${{ env.RERUN_ARTIFACT_BASE }}-attempt-${{ github.run_attempt }}
path: ./cypress/test-results
if-no-files-found: ignore
diff --git a/.github/workflows/embedding-sdk-component-tests.yml b/.github/workflows/embedding-sdk-component-tests.yml
index 0f795803ce7..f734a1d8540 100644
--- a/.github/workflows/embedding-sdk-component-tests.yml
+++ b/.github/workflows/embedding-sdk-component-tests.yml
@@ -26,6 +26,9 @@ jobs:
fail-fast: false
env:
CI_CONDUCTOR_TEST_SUITE: embedding-sdk-component-react-${{ matrix.react-version }}
+ # Base name of this leg's granular-rerun artifact. Each matrix leg needs its own, or a
+ # leg would restore the other leg's failed set. `-attempt-N` is appended below.
+ RERUN_ARTIFACT_BASE: failed-specs-embedding-sdk-component-react-${{ matrix.react-version }}
# Sensitive env vars (tokens, secrets) starting with `CYPRESS_` are accessed via async `cy.env()`
# Example: `cy.env(["FOO"]).then(({ FOO }) => ...)`
# For non-sensitive config, use `Cypress.expose()` in cypress config instead (synchronous)
@@ -73,6 +76,30 @@ jobs:
id: cypress-prep
uses: ./.github/actions/prepare-cypress
+ - name: Resolve previous attempt
+ if: ${{ github.run_attempt != '1' }}
+ run: echo "RERUN_PREV_ATTEMPT=$((GITHUB_RUN_ATTEMPT - 1))" >> "$GITHUB_ENV"
+
+ - name: Restore previously-failed specs
+ if: ${{ github.run_attempt != '1' }}
+ continue-on-error: true
+ uses: actions/download-artifact@v8
+ with:
+ # use pattern instead of name since this should still pass if the artifact doesn't exist
+ pattern: ${{ env.RERUN_ARTIFACT_BASE }}-attempt-${{ env.RERUN_PREV_ATTEMPT }}
+ path: target/rerun
+
+ - name: Get failed specs
+ if: ${{ github.run_attempt != '1' }}
+ run: |
+ if [ -s target/rerun/failed-specs ]; then
+ echo "Re-running failed specs from attempt $RERUN_PREV_ATTEMPT:"
+ cat target/rerun/failed-specs
+ echo "specs=$(cat target/rerun/failed-specs)" >> "$GITHUB_ENV"
+ else
+ echo "No failed-spec list from attempt $RERUN_PREV_ATTEMPT; running the full suite."
+ fi
+
# esbuild resolves every import in the spec bundle's graph, even the ones tree shaking then drops.
# The cljs imports resolve into target/cljs_dev, so the compiled CLJS has to exist before Cypress runs.
# run-tests.yml compiles it in parallel with the uberjar build, so downloading is ~10x faster than compiling here.
@@ -107,9 +134,18 @@ jobs:
- name: Run component tests for Embedding SDK
id: run-tests
+ shell: bash
env:
CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}
- run: node e2e/runner/run_cypress_ci.js component
+ SPECS: ${{ env.specs }}
+ # record video for re-runs of previously failed specs
+ CYPRESS_VIDEO: ${{ env.specs && 'true' || 'false' }}
+ run: |
+ SPEC_ARGS=()
+ if [ -n "$SPECS" ]; then
+ SPEC_ARGS=(--spec "$SPECS")
+ fi
+ node e2e/runner/run_cypress_ci.js component "${SPEC_ARGS[@]}"
# Keep this step continue-on-error because the quarantine logic controls whether it ultimately fails.
continue-on-error: true
@@ -140,12 +176,21 @@ jobs:
uses: actions/upload-artifact@v7
if: ${{ !cancelled() && steps.run-tests.outcome == 'failure' }}
with:
- name: cypress-recording-embedding-sdk-react-${{ matrix.react-version }}
+ name: cypress-recording-embedding-sdk-react-${{ matrix.react-version }}-attempt-${{ github.run_attempt }}
path: |
./cypress
./logs/test.log
if-no-files-found: ignore
+ - name: "SDK component tests > Upload failed specs for rerun"
+ uses: actions/upload-artifact@v7
+ if: ${{ !cancelled() && steps.run-tests.outcome == 'failure' }}
+ continue-on-error: true
+ with:
+ name: ${{ env.RERUN_ARTIFACT_BASE }}-attempt-${{ github.run_attempt }}
+ path: ./cypress/test-results
+ if-no-files-found: ignore
+
- name: "SDK component tests > Publish Summary"
if: ${{ !cancelled() && steps.run-tests.outcome == 'failure' }}
uses: actions/github-script@v9
@@ -195,6 +240,7 @@ jobs:
MB_WAREHOUSE_ALLOWED_NETWORKS: "allow-all"
METASTORE_DEV_SERVER_URL: ${{ secrets.METASTORE_DEV_SERVER_URL }}
CI_CONDUCTOR_TEST_SUITE: embedding-sdk-component-backward-compatibility
+ RERUN_ARTIFACT_BASE: failed-specs-embedding-sdk-component-backward-compatibility
# Sensitive env vars (tokens, secrets) starting with `CYPRESS_` are accessed via async `cy.env()`
# Example: `cy.env(["FOO"]).then(({ FOO }) => ...)`
# For non-sensitive config, use `Cypress.expose()` in cypress config instead (synchronous)
@@ -261,6 +307,30 @@ jobs:
id: cypress-prep
uses: ./.github/actions/prepare-cypress
+ - name: Resolve previous attempt
+ if: ${{ github.run_attempt != '1' }}
+ run: echo "RERUN_PREV_ATTEMPT=$((GITHUB_RUN_ATTEMPT - 1))" >> "$GITHUB_ENV"
+
+ - name: Restore previously-failed specs
+ if: ${{ github.run_attempt != '1' }}
+ continue-on-error: true
+ uses: actions/download-artifact@v8
+ with:
+ # use pattern instead of name since this should still pass if the artifact doesn't exist
+ pattern: ${{ env.RERUN_ARTIFACT_BASE }}-attempt-${{ env.RERUN_PREV_ATTEMPT }}
+ path: target/rerun
+
+ - name: Get failed specs
+ if: ${{ github.run_attempt != '1' }}
+ run: |
+ if [ -s target/rerun/failed-specs ]; then
+ echo "Re-running only the specs that failed on attempt $RERUN_PREV_ATTEMPT:"
+ cat target/rerun/failed-specs
+ echo "specs=$(cat target/rerun/failed-specs)" >> "$GITHUB_ENV"
+ else
+ echo "No failed-spec list from attempt $RERUN_PREV_ATTEMPT; running the full suite."
+ fi
+
# esbuild resolves every import in the spec bundle's graph, even the ones tree shaking then drops.
# The cljs imports resolve into target/cljs_dev, so the compiled CLJS has to exist before Cypress runs.
# The checked-out tree is the previous commit, which the current run's CLJS artifact was not built from,
@@ -283,11 +353,20 @@ jobs:
- name: Run component tests for Embedding SDK
id: run-tests
+ shell: bash
env:
CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}
+ SPECS: ${{ env.specs }}
+ # record video for re-runs of previously failed specs
+ CYPRESS_VIDEO: ${{ env.specs && 'true' || 'false' }}
run: |
+ SPEC_ARGS=()
+ if [ -n "$SPECS" ]; then
+ SPEC_ARGS=(--spec "$SPECS")
+ fi
node e2e/runner/run_cypress_ci.js component \
- --expose grepTags="-@skip-backward-compatibility"
+ --expose grepTags="-@skip-backward-compatibility" \
+ "${SPEC_ARGS[@]}"
# Keep this step continue-on-error because the quarantine logic controls whether it ultimately fails.
continue-on-error: true
@@ -318,12 +397,21 @@ jobs:
uses: actions/upload-artifact@v7
if: ${{ !cancelled() && steps.run-tests.outcome == 'failure' }}
with:
- name: cypress-recording-embedding-sdk-cross-version
+ name: cypress-recording-embedding-sdk-cross-version-attempt-${{ github.run_attempt }}
path: |
./cypress
./logs/test.log
if-no-files-found: ignore
+ - name: Upload failed specs for rerun
+ uses: actions/upload-artifact@v7
+ if: ${{ !cancelled() && steps.run-tests.outcome == 'failure' }}
+ continue-on-error: true
+ with:
+ name: ${{ env.RERUN_ARTIFACT_BASE }}-attempt-${{ github.run_attempt }}
+ path: ./cypress/test-results
+ if-no-files-found: ignore
+
- name: Publish Summary
if: ${{ !cancelled() && steps.run-tests.outcome == 'failure' }}
uses: actions/github-script@v9
diff --git a/e2e/support/cypress-embedding-sdk-component-test.config.js b/e2e/support/cypress-embedding-sdk-component-test.config.js
index edb09b7bf22..155beb72d54 100644
--- a/e2e/support/cypress-embedding-sdk-component-test.config.js
+++ b/e2e/support/cypress-embedding-sdk-component-test.config.js
@@ -6,7 +6,7 @@ const embeddingSdkComponentTestConfig = {
baseUrl: undefined, // baseUrl should not be set for component tests,
defaultCommandTimeout: 10000,
requestTimeout: 10000,
- video: false,
+ video: process.env["CYPRESS_VIDEO"] !== "false",
experimentalMemoryManagement: true,
numTestsKeptInMemory: 1,
specPattern: "e2e/test-component/scenarios/**/*.cy.spec.tsx",