Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Gate cross-version tests with the ci-conductor quarantine (#85114)

Chris Vandoodewaarda70f6acbf8d6
diff --git a/.github/workflows/cross-version.yml b/.github/workflows/cross-version.yml
index fa4de47eb4d..7936ff51014 100644
--- a/.github/workflows/cross-version.yml
+++ b/.github/workflows/cross-version.yml
@@ -71,6 +71,7 @@ jobs:
console.log("matrix:", matrix);
test:
+ name: cross-version test ${{ matrix.source }} → ${{ matrix.target }}
needs: generate-test-matrix
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 30
@@ -78,6 +79,14 @@ jobs:
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.matrix) }}
fail-fast: false
env:
+ QUARANTINE_FAILURES_FILE: ${{ github.workspace }}/target/quarantine-failures.jsonl
+ CROSS_VERSION_JUNIT_FILE: ${{ github.workspace }}/target/cross-version-junit/cross-version.xml
+ CI_CONDUCTOR_BASE_URL: ${{ vars.CI_CONDUCTOR_BASE_URL }}
+ CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}
+ CI_CONDUCTOR_TEST_SUITE: cross-version-${{ matrix.source }}-${{ matrix.target }}
+ REPO_ID: ${{ github.event.repository.id }}
+ COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+ TARGET_BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
# Number of days we don't fail a cross version test that is missing a migration that was recently merged to master
# but targets a different version. For example: a migration for v63 is added to master but master is on v66. Cross
# version tests will fail until that is backported and released on v63 but we should give people time to do that.
@@ -111,6 +120,8 @@ jobs:
if: steps.prebuilt-cljs.outcome != 'success'
run: bun run build-pure:cljs
+ - uses: ./.github/actions/resolve-job-id
+
- name: Run cross-version test
id: run-test
env:
@@ -120,6 +131,28 @@ jobs:
run: |
cd cross-version
./test.sh --source "${{ matrix.source }}" --target "${{ matrix.target }}"
+ continue-on-error: true
+
+ - name: Report results to ci-conductor
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ timeout-minutes: 3
+ uses: ./.github/actions/report-failures-to-ci-conductor
+ env:
+ JEST_JUNIT_OUTPUT_DIR: ${{ github.workspace }}/target/cross-version-junit
+ JEST_JUNIT_OUTPUT_NAME: cross-version.xml
+ with:
+ adapter: frontend
+ test-suite: ${{ env.CI_CONDUCTOR_TEST_SUITE }}
+
+ - name: Quarantine gate
+ if: ${{ !cancelled() && steps.run-test.outcome == 'failure' }}
+ timeout-minutes: 3
+ uses: ./.github/actions/quarantine-gate
+ with:
+ adapter: e2e
+ test-suite: ${{ env.CI_CONDUCTOR_TEST_SUITE }}
+ enforce-quarantine: "true"
- name: Notify Slack about migrations that need a backport or a release
continue-on-error: true
@@ -162,7 +195,7 @@ jobs:
if-no-files-found: ignore
- name: Show logs on failure
- if: failure()
+ if: ${{ !cancelled() && steps.run-test.outcome == 'failure' }}
run: |
for f in cross-version/logs/*.log; do
[ -e "$f" ] || continue
diff --git a/cross-version/test.sh b/cross-version/test.sh
index ff25de123a0..222f7a98628 100755
--- a/cross-version/test.sh
+++ b/cross-version/test.sh
@@ -21,6 +21,7 @@ METABASE_PORT="${METABASE_PORT:-3000}"
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-120}"
LOG_DIR="${LOG_DIR:-$SCRIPT_DIR/logs}"
GRACE_PERIOD="${GRACE_PERIOD:-3}"
+CYPRESS_FAILURES_RECORDED=0
# Colors for output
RED='\033[0;31m'
@@ -101,8 +102,23 @@ run_e2e() {
fi
log "Running @${phase} e2e tests for ${version} from: e2e/cross-version/${specs_dir##*/}/"
- CYPRESS_SPEC_PATTERN="${specs_dir}/**/*.cy.spec.ts" \
- "$REPO_ROOT/e2e/cross-version/run.sh" --phase "$phase"
+ local recorded_before
+ recorded_before=$(count_recorded_failures)
+ if ! CYPRESS_SPEC_PATTERN="${specs_dir}/**/*.cy.spec.ts" \
+ "$REPO_ROOT/e2e/cross-version/run.sh" --phase "$phase"; then
+ if (( $(count_recorded_failures) > recorded_before )); then
+ CYPRESS_FAILURES_RECORDED=1
+ fi
+ return 1
+ fi
+}
+
+count_recorded_failures() {
+ if [[ -n "${QUARANTINE_FAILURES_FILE:-}" && -f "$QUARANTINE_FAILURES_FILE" ]]; then
+ wc -l < "$QUARANTINE_FAILURES_FILE" | tr -d ' '
+ else
+ echo 0
+ fi
}
@@ -420,6 +436,44 @@ cleanup() {
teardown
}
+xml_escape() {
+ local value="$1"
+ value="${value//&/&amp;}"
+ value="${value//</&lt;}"
+ value="${value//>/&gt;}"
+ value="${value//\"/&quot;}"
+ printf '%s' "$value"
+}
+
+write_junit_result() {
+ local code="$1"
+ local name="$2"
+ local result="/>"
+ if (( code != 0 )); then
+ result="><failure message=\"test.sh exited with code ${code}\">test.sh exited with code ${code}</failure></testcase>"
+ fi
+ mkdir -p "$(dirname "$CROSS_VERSION_JUNIT_FILE")"
+ printf '<?xml version="1.0" encoding="UTF-8"?>\n<testsuite name="cross-version">\n <testcase name="%s" classname="cross-version" file="cross-version/test.sh"%s\n</testsuite>\n' \
+ "$(xml_escape "$name")" "$result" > "$CROSS_VERSION_JUNIT_FILE"
+}
+
+# Writes a small xml file recording pass/fail that is uploaded to CI Conductor so we can quarantine these tests if required.
+record_result() {
+ local code="$1"
+ local name="$SOURCE_VERSION -> $TARGET_VERSION"
+ if (( CYPRESS_FAILURES_RECORDED == 1 )); then
+ code=0
+ fi
+ if [[ -n "${CROSS_VERSION_JUNIT_FILE:-}" ]]; then
+ write_junit_result "$code" "$name"
+ fi
+ (( code == 0 )) && return 0
+ [[ -n "${QUARANTINE_FAILURES_FILE:-}" ]] || return 0
+ mkdir -p "$(dirname "$QUARANTINE_FAILURES_FILE")"
+ printf '{"test_name":"%s","test_path":"cross-version","file_path":"cross-version/test.sh"}\n' \
+ "$name" >> "$QUARANTINE_FAILURES_FILE"
+}
+
check_image_exists() {
local image="$1"
if ! docker manifest inspect "$image" > /dev/null 2>&1; then
@@ -431,6 +485,8 @@ check_image_exists() {
main() {
local direction source_image target_image
+ trap 'code=$?; record_result "$code"; exit "$code"' EXIT
+
direction=$(cli compare "$SOURCE_VERSION" "$TARGET_VERSION") || exit 1
source_image=$(cli image "$SOURCE_VERSION") || exit 1
target_image=$(cli image "$TARGET_VERSION") || exit 1
@@ -456,7 +512,7 @@ main() {
teardown
init_logs
- trap cleanup EXIT
+ trap 'code=$?; cleanup; record_result "$code"; exit "$code"' EXIT
log ""
log "Step 1: Starting SOURCE version ($SOURCE_VERSION)..."