The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
only re-run the failed specs for SDK component tests (#82220)
Chris Vandoodewaardfe4e2beffa32
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.ymlindex 37fac3fe45f..c84cd430759 100644--- a/.github/workflows/e2e-test.yml+++ b/.github/workflows/e2e-test.yml@@ -64,6 +64,7 @@ jobs:# workspace where the gate step runs; an absolute path anchored to the# workspace makes both agree regardless of cwd.QUARANTINE_FAILURES_FILE: ${{ github.workspace }}/target/quarantine-failures.jsonl+ RERUN_ARTIFACT_BASE: failed-tests-${{ inputs.name }}-${{ inputs.edition }}# Cypress records video for every spec but the passing specs' videos are# deleted in after:spec so recording is ~5-10% slowdown per shard and almost# never yields an artifact. Failure screenshots are unaffected, and the@@ -119,17 +120,28 @@ jobs:id: cypress-prepuses: ./.github/actions/prepare-cypress- - name: Download failed test artifacts- uses: actions/download-artifact@v8- if: github.run_attempt != '1'+ - name: Resolve previous attempt+ if: ${{ github.run_attempt != '1' }}+ run: echo "RERUN_PREV_ATTEMPT=$((GITHUB_RUN_ATTEMPT - 1))" >> "$GITHUB_ENV"++ - name: Restore previously-failed specs+ if: ${{ github.run_attempt != '1' }}continue-on-error: true- id: failed-specs+ uses: actions/download-artifact@v8with:- name: failed-tests-${{ inputs.name }}-${{ inputs.edition }}+ pattern: ${{ env.RERUN_ARTIFACT_BASE }}-attempt-${{ env.RERUN_PREV_ATTEMPT }}+ path: target/rerun- name: Get failed specs- if: github.run_attempt != '1'- run: echo "specs=$(cat failed-specs)" >> "$GITHUB_ENV"+ if: ${{ github.run_attempt != '1' }}+ run: |+ if [ -s target/rerun/failed-specs ]; then+ echo "Re-running only the specs that failed on attempt $RERUN_PREV_ATTEMPT:"+ cat target/rerun/failed-specs+ echo "specs=$(cat target/rerun/failed-specs)" >> "$GITHUB_ENV"+ else+ echo "No failed-spec list from attempt $RERUN_PREV_ATTEMPT; running the full suite."+ fi# run-tests.yml compiles this in parallel with the uberjar build. Downloading it is ~10x faster than compiling it here.# Callers without that job (stress tests, coverage shards, cross-run re-runs) fall through to compiling as before.@@ -232,7 +244,7 @@ jobs:if: |!cancelled() && (steps.split-e2e.outcome == 'failure' || steps.tagged-e2e.outcome == 'failure')with:- name: cypress-recording-${{ inputs.name }}-${{ inputs.edition }}+ name: cypress-recording-${{ inputs.name }}-${{ inputs.edition }}-attempt-${{ github.run_attempt }}path: |./cypress./logs/test.log@@ -242,8 +254,9 @@ jobs:uses: actions/upload-artifact@v7if: |!cancelled() && (steps.split-e2e.outcome == 'failure' || steps.tagged-e2e.outcome == 'failure')+ continue-on-error: truewith:- name: failed-tests-${{ inputs.name }}-${{ inputs.edition }}+ name: ${{ env.RERUN_ARTIFACT_BASE }}-attempt-${{ github.run_attempt }}path: ./cypress/test-resultsif-no-files-found: ignorediff --git a/.github/workflows/embedding-sdk-component-tests.yml b/.github/workflows/embedding-sdk-component-tests.ymlindex 0f795803ce7..f734a1d8540 100644--- a/.github/workflows/embedding-sdk-component-tests.yml+++ b/.github/workflows/embedding-sdk-component-tests.yml@@ -26,6 +26,9 @@ jobs:fail-fast: falseenv:CI_CONDUCTOR_TEST_SUITE: embedding-sdk-component-react-${{ matrix.react-version }}+ # Base name of this leg's granular-rerun artifact. Each matrix leg needs its own, or a+ # leg would restore the other leg's failed set. `-attempt-N` is appended below.+ RERUN_ARTIFACT_BASE: failed-specs-embedding-sdk-component-react-${{ matrix.react-version }}# Sensitive env vars (tokens, secrets) starting with `CYPRESS_` are accessed via async `cy.env()`# Example: `cy.env(["FOO"]).then(({ FOO }) => ...)`# For non-sensitive config, use `Cypress.expose()` in cypress config instead (synchronous)@@ -73,6 +76,30 @@ jobs:id: cypress-prepuses: ./.github/actions/prepare-cypress+ - name: Resolve previous attempt+ if: ${{ github.run_attempt != '1' }}+ run: echo "RERUN_PREV_ATTEMPT=$((GITHUB_RUN_ATTEMPT - 1))" >> "$GITHUB_ENV"++ - name: Restore previously-failed specs+ if: ${{ github.run_attempt != '1' }}+ continue-on-error: true+ uses: actions/download-artifact@v8+ with:+ # use pattern instead of name since this should still pass if the artifact doesn't exist+ pattern: ${{ env.RERUN_ARTIFACT_BASE }}-attempt-${{ env.RERUN_PREV_ATTEMPT }}+ path: target/rerun++ - name: Get failed specs+ if: ${{ github.run_attempt != '1' }}+ run: |+ if [ -s target/rerun/failed-specs ]; then+ echo "Re-running failed specs from attempt $RERUN_PREV_ATTEMPT:"+ cat target/rerun/failed-specs+ echo "specs=$(cat target/rerun/failed-specs)" >> "$GITHUB_ENV"+ else+ echo "No failed-spec list from attempt $RERUN_PREV_ATTEMPT; running the full suite."+ fi+# esbuild resolves every import in the spec bundle's graph, even the ones tree shaking then drops.# The cljs imports resolve into target/cljs_dev, so the compiled CLJS has to exist before Cypress runs.# run-tests.yml compiles it in parallel with the uberjar build, so downloading is ~10x faster than compiling here.@@ -107,9 +134,18 @@ jobs:- name: Run component tests for Embedding SDKid: run-tests+ shell: bashenv:CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}- run: node e2e/runner/run_cypress_ci.js component+ SPECS: ${{ env.specs }}+ # record video for re-runs of previously failed specs+ CYPRESS_VIDEO: ${{ env.specs && 'true' || 'false' }}+ run: |+ SPEC_ARGS=()+ if [ -n "$SPECS" ]; then+ SPEC_ARGS=(--spec "$SPECS")+ fi+ node e2e/runner/run_cypress_ci.js component "${SPEC_ARGS[@]}"# Keep this step continue-on-error because the quarantine logic controls whether it ultimately fails.continue-on-error: true@@ -140,12 +176,21 @@ jobs:uses: actions/upload-artifact@v7if: ${{ !cancelled() && steps.run-tests.outcome == 'failure' }}with:- name: cypress-recording-embedding-sdk-react-${{ matrix.react-version }}+ name: cypress-recording-embedding-sdk-react-${{ matrix.react-version }}-attempt-${{ github.run_attempt }}path: |./cypress./logs/test.logif-no-files-found: ignore+ - name: "SDK component tests > Upload failed specs for rerun"+ uses: actions/upload-artifact@v7+ if: ${{ !cancelled() && steps.run-tests.outcome == 'failure' }}+ continue-on-error: true+ with:+ name: ${{ env.RERUN_ARTIFACT_BASE }}-attempt-${{ github.run_attempt }}+ path: ./cypress/test-results+ if-no-files-found: ignore+- name: "SDK component tests > Publish Summary"if: ${{ !cancelled() && steps.run-tests.outcome == 'failure' }}uses: actions/github-script@v9@@ -195,6 +240,7 @@ jobs:MB_WAREHOUSE_ALLOWED_NETWORKS: "allow-all"METASTORE_DEV_SERVER_URL: ${{ secrets.METASTORE_DEV_SERVER_URL }}CI_CONDUCTOR_TEST_SUITE: embedding-sdk-component-backward-compatibility+ RERUN_ARTIFACT_BASE: failed-specs-embedding-sdk-component-backward-compatibility# Sensitive env vars (tokens, secrets) starting with `CYPRESS_` are accessed via async `cy.env()`# Example: `cy.env(["FOO"]).then(({ FOO }) => ...)`# For non-sensitive config, use `Cypress.expose()` in cypress config instead (synchronous)@@ -261,6 +307,30 @@ jobs:id: cypress-prepuses: ./.github/actions/prepare-cypress+ - name: Resolve previous attempt+ if: ${{ github.run_attempt != '1' }}+ run: echo "RERUN_PREV_ATTEMPT=$((GITHUB_RUN_ATTEMPT - 1))" >> "$GITHUB_ENV"++ - name: Restore previously-failed specs+ if: ${{ github.run_attempt != '1' }}+ continue-on-error: true+ uses: actions/download-artifact@v8+ with:+ # use pattern instead of name since this should still pass if the artifact doesn't exist+ pattern: ${{ env.RERUN_ARTIFACT_BASE }}-attempt-${{ env.RERUN_PREV_ATTEMPT }}+ path: target/rerun++ - name: Get failed specs+ if: ${{ github.run_attempt != '1' }}+ run: |+ if [ -s target/rerun/failed-specs ]; then+ echo "Re-running only the specs that failed on attempt $RERUN_PREV_ATTEMPT:"+ cat target/rerun/failed-specs+ echo "specs=$(cat target/rerun/failed-specs)" >> "$GITHUB_ENV"+ else+ echo "No failed-spec list from attempt $RERUN_PREV_ATTEMPT; running the full suite."+ fi+# esbuild resolves every import in the spec bundle's graph, even the ones tree shaking then drops.# The cljs imports resolve into target/cljs_dev, so the compiled CLJS has to exist before Cypress runs.# The checked-out tree is the previous commit, which the current run's CLJS artifact was not built from,@@ -283,11 +353,20 @@ jobs:- name: Run component tests for Embedding SDKid: run-tests+ shell: bashenv:CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}+ SPECS: ${{ env.specs }}+ # record video for re-runs of previously failed specs+ CYPRESS_VIDEO: ${{ env.specs && 'true' || 'false' }}run: |+ SPEC_ARGS=()+ if [ -n "$SPECS" ]; then+ SPEC_ARGS=(--spec "$SPECS")+ finode e2e/runner/run_cypress_ci.js component \- --expose grepTags="-@skip-backward-compatibility"+ --expose grepTags="-@skip-backward-compatibility" \+ "${SPEC_ARGS[@]}"# Keep this step continue-on-error because the quarantine logic controls whether it ultimately fails.continue-on-error: true@@ -318,12 +397,21 @@ jobs:uses: actions/upload-artifact@v7if: ${{ !cancelled() && steps.run-tests.outcome == 'failure' }}with:- name: cypress-recording-embedding-sdk-cross-version+ name: cypress-recording-embedding-sdk-cross-version-attempt-${{ github.run_attempt }}path: |./cypress./logs/test.logif-no-files-found: ignore+ - name: Upload failed specs for rerun+ uses: actions/upload-artifact@v7+ if: ${{ !cancelled() && steps.run-tests.outcome == 'failure' }}+ continue-on-error: true+ with:+ name: ${{ env.RERUN_ARTIFACT_BASE }}-attempt-${{ github.run_attempt }}+ path: ./cypress/test-results+ if-no-files-found: ignore+- name: Publish Summaryif: ${{ !cancelled() && steps.run-tests.outcome == 'failure' }}uses: actions/github-script@v9diff --git a/e2e/support/cypress-embedding-sdk-component-test.config.js b/e2e/support/cypress-embedding-sdk-component-test.config.jsindex edb09b7bf22..155beb72d54 100644--- a/e2e/support/cypress-embedding-sdk-component-test.config.js+++ b/e2e/support/cypress-embedding-sdk-component-test.config.js@@ -6,7 +6,7 @@ const embeddingSdkComponentTestConfig = {baseUrl: undefined, // baseUrl should not be set for component tests,defaultCommandTimeout: 10000,requestTimeout: 10000,- video: false,+ video: process.env["CYPRESS_VIDEO"] !== "false",experimentalMemoryManagement: true,numTestsKeptInMemory: 1,specPattern: "e2e/test-component/scenarios/**/*.cy.spec.tsx",