The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Gate cross-version tests with the ci-conductor quarantine (#85114)
Chris Vandoodewaarda70f6acbf8d6
diff --git a/.github/workflows/cross-version.yml b/.github/workflows/cross-version.ymlindex fa4de47eb4d..7936ff51014 100644--- a/.github/workflows/cross-version.yml+++ b/.github/workflows/cross-version.yml@@ -71,6 +71,7 @@ jobs:console.log("matrix:", matrix);test:+ name: cross-version test ${{ matrix.source }} → ${{ matrix.target }}needs: generate-test-matrixruns-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 30@@ -78,6 +79,14 @@ jobs:matrix: ${{ fromJson(needs.generate-test-matrix.outputs.matrix) }}fail-fast: falseenv:+ QUARANTINE_FAILURES_FILE: ${{ github.workspace }}/target/quarantine-failures.jsonl+ CROSS_VERSION_JUNIT_FILE: ${{ github.workspace }}/target/cross-version-junit/cross-version.xml+ CI_CONDUCTOR_BASE_URL: ${{ vars.CI_CONDUCTOR_BASE_URL }}+ CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}+ CI_CONDUCTOR_TEST_SUITE: cross-version-${{ matrix.source }}-${{ matrix.target }}+ REPO_ID: ${{ github.event.repository.id }}+ COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}+ TARGET_BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}# Number of days we don't fail a cross version test that is missing a migration that was recently merged to master# but targets a different version. For example: a migration for v63 is added to master but master is on v66. Cross# version tests will fail until that is backported and released on v63 but we should give people time to do that.@@ -111,6 +120,8 @@ jobs:if: steps.prebuilt-cljs.outcome != 'success'run: bun run build-pure:cljs+ - uses: ./.github/actions/resolve-job-id+- name: Run cross-version testid: run-testenv:@@ -120,6 +131,28 @@ jobs:run: |cd cross-version./test.sh --source "${{ matrix.source }}" --target "${{ matrix.target }}"+ continue-on-error: true++ - name: Report results to ci-conductor+ if: ${{ !cancelled() }}+ continue-on-error: true+ timeout-minutes: 3+ uses: ./.github/actions/report-failures-to-ci-conductor+ env:+ JEST_JUNIT_OUTPUT_DIR: ${{ github.workspace }}/target/cross-version-junit+ JEST_JUNIT_OUTPUT_NAME: cross-version.xml+ with:+ adapter: frontend+ test-suite: ${{ env.CI_CONDUCTOR_TEST_SUITE }}++ - name: Quarantine gate+ if: ${{ !cancelled() && steps.run-test.outcome == 'failure' }}+ timeout-minutes: 3+ uses: ./.github/actions/quarantine-gate+ with:+ adapter: e2e+ test-suite: ${{ env.CI_CONDUCTOR_TEST_SUITE }}+ enforce-quarantine: "true"- name: Notify Slack about migrations that need a backport or a releasecontinue-on-error: true@@ -162,7 +195,7 @@ jobs:if-no-files-found: ignore- name: Show logs on failure- if: failure()+ if: ${{ !cancelled() && steps.run-test.outcome == 'failure' }}run: |for f in cross-version/logs/*.log; do[ -e "$f" ] || continuediff --git a/cross-version/test.sh b/cross-version/test.shindex ff25de123a0..222f7a98628 100755--- a/cross-version/test.sh+++ b/cross-version/test.sh@@ -21,6 +21,7 @@ METABASE_PORT="${METABASE_PORT:-3000}"HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-120}"LOG_DIR="${LOG_DIR:-$SCRIPT_DIR/logs}"GRACE_PERIOD="${GRACE_PERIOD:-3}"+CYPRESS_FAILURES_RECORDED=0# Colors for outputRED='\033[0;31m'@@ -101,8 +102,23 @@ run_e2e() {filog "Running @${phase} e2e tests for ${version} from: e2e/cross-version/${specs_dir##*/}/"- CYPRESS_SPEC_PATTERN="${specs_dir}/**/*.cy.spec.ts" \- "$REPO_ROOT/e2e/cross-version/run.sh" --phase "$phase"+ local recorded_before+ recorded_before=$(count_recorded_failures)+ if ! CYPRESS_SPEC_PATTERN="${specs_dir}/**/*.cy.spec.ts" \+ "$REPO_ROOT/e2e/cross-version/run.sh" --phase "$phase"; then+ if (( $(count_recorded_failures) > recorded_before )); then+ CYPRESS_FAILURES_RECORDED=1+ fi+ return 1+ fi+}++count_recorded_failures() {+ if [[ -n "${QUARANTINE_FAILURES_FILE:-}" && -f "$QUARANTINE_FAILURES_FILE" ]]; then+ wc -l < "$QUARANTINE_FAILURES_FILE" | tr -d ' '+ else+ echo 0+ fi}@@ -420,6 +436,44 @@ cleanup() {teardown}+xml_escape() {+ local value="$1"+ value="${value//&/&}"+ value="${value//</<}"+ value="${value//>/>}"+ value="${value//\"/"}"+ printf '%s' "$value"+}++write_junit_result() {+ local code="$1"+ local name="$2"+ local result="/>"+ if (( code != 0 )); then+ result="><failure message=\"test.sh exited with code ${code}\">test.sh exited with code ${code}</failure></testcase>"+ fi+ mkdir -p "$(dirname "$CROSS_VERSION_JUNIT_FILE")"+ printf '<?xml version="1.0" encoding="UTF-8"?>\n<testsuite name="cross-version">\n <testcase name="%s" classname="cross-version" file="cross-version/test.sh"%s\n</testsuite>\n' \+ "$(xml_escape "$name")" "$result" > "$CROSS_VERSION_JUNIT_FILE"+}++# Writes a small xml file recording pass/fail that is uploaded to CI Conductor so we can quarantine these tests if required.+record_result() {+ local code="$1"+ local name="$SOURCE_VERSION -> $TARGET_VERSION"+ if (( CYPRESS_FAILURES_RECORDED == 1 )); then+ code=0+ fi+ if [[ -n "${CROSS_VERSION_JUNIT_FILE:-}" ]]; then+ write_junit_result "$code" "$name"+ fi+ (( code == 0 )) && return 0+ [[ -n "${QUARANTINE_FAILURES_FILE:-}" ]] || return 0+ mkdir -p "$(dirname "$QUARANTINE_FAILURES_FILE")"+ printf '{"test_name":"%s","test_path":"cross-version","file_path":"cross-version/test.sh"}\n' \+ "$name" >> "$QUARANTINE_FAILURES_FILE"+}+check_image_exists() {local image="$1"if ! docker manifest inspect "$image" > /dev/null 2>&1; then@@ -431,6 +485,8 @@ check_image_exists() {main() {local direction source_image target_image+ trap 'code=$?; record_result "$code"; exit "$code"' EXIT+direction=$(cli compare "$SOURCE_VERSION" "$TARGET_VERSION") || exit 1source_image=$(cli image "$SOURCE_VERSION") || exit 1target_image=$(cli image "$TARGET_VERSION") || exit 1@@ -456,7 +512,7 @@ main() {teardowninit_logs- trap cleanup EXIT+ trap 'code=$?; cleanup; record_result "$code"; exit "$code"' EXITlog ""log "Step 1: Starting SOURCE version ($SOURCE_VERSION)..."