Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

remove dead workflows and support code (#79624)

Ryan Laurie9e6e08bffbc5
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 505ec5a64d3..1d80345b160 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -523,7 +523,6 @@
metabase.test.data.interface/defdataset clojure.core/def
metabase.test.data.interface/defdataset-edn clojure.core/def
metabase.test.embedded-postgres.core/with-system clojure.core/let
- metabase.test.gentest/defgentest clojure.test/deftest
metabase.test/defdataset clojure.core/def
metabase.test/let-url clojure.core/let
metabase.test/with-actions clojure.core/let
@@ -682,7 +681,6 @@
metabase.test.data/dataset hooks.metabase.test.data/dataset
metabase.test.data/mbql-query hooks.metabase.test.data/mbql-query
metabase.test.data/run-mbql-query hooks.metabase.test.data/mbql-query
- metabase.test.gentest/iterate hooks.metabase.test.gentest/iterate*
metabase.test.util.async/with-open-channels hooks.common/let-with-optional-value-for-last-binding
metabase.test.util.generators.jvm/with-random-cards hooks.metabase.test.util.generators.jvm/with-random-cards
metabase.test.util.log/with-log-level hooks.common/with-ignored-first-arg
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index edcf6f3a9cb..607b14a0d1e 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -5,7 +5,7 @@
;; to defend in your PR.
;; :all is the vector-less ignore form, which suppresses every linter on the next form.
{:ignore-counts {:aliased-namespace-symbol 3
- :all 50
+ :all 49
:case-symbol-test 1
:clojure-lsp/unused-public-var 9
:def-fn 1
diff --git a/.clj-kondo/src/hooks/metabase/test/gentest.clj b/.clj-kondo/src/hooks/metabase/test/gentest.clj
deleted file mode 100644
index 2e6c31d4c86..00000000000
--- a/.clj-kondo/src/hooks/metabase/test/gentest.clj
+++ /dev/null
@@ -1,14 +0,0 @@
-(ns hooks.metabase.test.gentest
- (:require
- [clj-kondo.hooks-api :as hooks]))
-
-(defn iterate*
- "(iterate {:as limit-spec} [s1 val1 & more] body-form body-form...)"
- [{{:keys [children] :as node} :node :as orig}]
- (let [[token limit-spec bindings & body-nodes] children
- node* (-> (hooks/list-node
- (list token limit-spec
- (hooks/list-node (list* (hooks/token-node 'let) bindings
- body-nodes))))
- (with-meta (meta node)))]
- (assoc orig :node node*)))
diff --git a/.github/actions/verify-membership/action.yml b/.github/actions/verify-membership/action.yml
deleted file mode 100644
index b3f1961cd44..00000000000
--- a/.github/actions/verify-membership/action.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-name: 'Verify Organization Membership'
-description: 'Check if a user is a member of a GitHub organization'
-inputs:
- github_token:
- description: 'Token with access to the organization'
- required: true
- organization_name:
- description: 'Organization name'
- required: true
- username:
- description: 'Username'
- required: true
-outputs:
- is_member:
- value: ${{ steps.verify-membership.outputs.is_member }}
- description: "Is the user a member of the organization"
-runs:
- using: "composite"
- steps:
- - run: bash ${{ github.action_path }}/verify_membership.sh ${{ inputs.github_token }} ${{ inputs.organization_name }} ${{ inputs.username }}
- id: verify-membership
- shell: bash
diff --git a/.github/actions/verify-membership/verify_membership.sh b/.github/actions/verify-membership/verify_membership.sh
deleted file mode 100755
index 2870eb21f3d..00000000000
--- a/.github/actions/verify-membership/verify_membership.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-TOKEN=$1
-ORG_NAME=$2
-USERNAME=$3
-
-response=$(curl -L -o /dev/null -w "%{http_code}" \
- -H "Accept: application/vnd.github+json" \
- -H "Authorization: Bearer $TOKEN" \
- -H "X-GitHub-Api-Version: 2022-11-28" \
- "https://api.github.com/orgs/$ORG_NAME/members/$USERNAME")
-
-# Github returns an empty response with 204 status if user is a member and 404 status otherwise
-# See: https://docs.github.com/en/rest/orgs/members?apiVersion=2022-11-28#check-organization-membership-for-a-user
-if [ "$response" == "204" ]; then
- echo "The user $USERNAME is a member of $ORG_NAME"
- echo "is_member=true" >> $GITHUB_OUTPUT
-elif [ "$response" == "404" ]; then
- echo "The user $USERNAME is not a member of $ORG_NAME"
- echo "is_member=false" >> $GITHUB_OUTPUT
-else
- echo "Failed to determine the membership status for $USERNAME in $ORG_NAME"
- echo "is_member=false" >> $GITHUB_OUTPUT
- exit 1
-fi
diff --git a/.github/file-paths.yaml b/.github/file-paths.yaml
index 72e9c2b0b03..68a6990cd7e 100644
--- a/.github/file-paths.yaml
+++ b/.github/file-paths.yaml
@@ -106,13 +106,6 @@ backend_ci: &backend_ci
- ".github/workflows/backend.yml"
- ".github/workflows/drivers.yml"
-backend_presto_kerberos:
- - "**/presto_jdbc/**"
- - "**/presto-jdbc/**" # the driver module has a hyphen. underscores are classpaths
- - "**/presto_jdbc.clj"
- - "**/presto_jdbc_test.clj"
- - ".github/workflows/presto-kerberos-integration-test.yml"
-
backend_sources: &backend_sources
- *shared_sources
- "**deps.edn"
@@ -223,11 +216,6 @@ i18n:
- ".github/workflows/i18n.yml"
- *sources
-visualizations:
- - "frontend/src/metabase/visualizations/**"
- - "frontend/src/metabase/static-viz/**"
- - "src/metabase/pulse/**"
-
cljs:
- *shared_sources
- *shared_specs
diff --git a/.github/scripts/get-issue-data.js b/.github/scripts/get-issue-data.js
deleted file mode 100644
index 015f29ebb7d..00000000000
--- a/.github/scripts/get-issue-data.js
+++ /dev/null
@@ -1,84 +0,0 @@
-const githubSlackMap = require("../../release/github-slack-map.json");
-
-async function getIssueData({ context, github, manualIssueNumber }) {
- let issueNumber;
- let issueData;
- let prReviews = [];
- const triggerType = context.eventName;
-
- if (context.eventName === "issues") {
- issueNumber = context.payload.issue.number;
- issueData = context.payload.issue;
- } else if (context.eventName === "workflow_dispatch") {
- issueNumber = parseInt(manualIssueNumber);
- const { data } = await github.rest.issues.get({
- owner: context.repo.owner,
- repo: context.repo.repo,
- issue_number: issueNumber,
- });
- issueData = data;
- } else if (context.eventName === "pull_request_review") {
- const prNumber = context.payload.pull_request.number;
- const { data: prData } = await github.rest.pulls.get({
- owner: context.repo.owner,
- repo: context.repo.repo,
- pull_number: prNumber,
- });
-
- // Extract issue number from branch name (e.g., "claude-fix/issue-12345-some-title")
- const branchMatch = prData.head.ref.match(/claude-fix\/issue-(\d+)/);
- if (!branchMatch) {
- throw new Error(
- `Could not extract issue number from branch name: ${prData.head.ref}`,
- );
- }
-
- issueNumber = parseInt(branchMatch[1]);
-
- const { data } = await github.rest.issues.get({
- owner: context.repo.owner,
- repo: context.repo.repo,
- issue_number: issueNumber,
- });
- issueData = data;
-
- const { data: reviews } = await github.rest.pulls.listReviews({
- owner: context.repo.owner,
- repo: context.repo.repo,
- pull_number: prNumber,
- });
-
- prReviews = reviews.filter(
- (review) =>
- review.state === "CHANGES_REQUESTED" &&
- isMetabaseMember(review.user.login),
- );
- }
-
- const { data: comments } = await github.rest.issues.listComments({
- owner: context.repo.owner,
- repo: context.repo.repo,
- issue_number: issueNumber,
- });
-
- const employeeComments = comments.filter((comment) =>
- isMetabaseMember(comment.user.login),
- );
-
- return {
- issueNumber,
- issueData,
- employeeComments,
- prReviews,
- triggerType,
- isIssueAuthorEmployee: isMetabaseMember(issueData.user.login),
- };
-}
-
-function isMetabaseMember(username) {
- return username in githubSlackMap;
-}
-
-module.exports = {
- getIssueData,
-};
diff --git a/.github/scripts/run-presto-kerberos-integration-test.sh b/.github/scripts/run-presto-kerberos-integration-test.sh
deleted file mode 100755
index e51b4546eb5..00000000000
--- a/.github/scripts/run-presto-kerberos-integration-test.sh
+++ /dev/null
@@ -1,65 +0,0 @@
-#! /usr/bin/env bash
-# runs one or more Metabase test(s) against a Kerberized Presto instance
-set -eo pipefail
-
-# Need Java commands on $PATH, which apparently is not yet the case
-export JAVA_HOME="/usr/lib/jvm/msopenjdk-current"
-export PATH="$PATH:$JAVA_HOME/bin"
-
-# ensure java commmand is available
-which java
-
-# install clojure version needed for Metabase
-curl -O https://download.clojure.org/install/linux-install-1.12.0.1488.sh
-chmod +x linux-install-1.12.0.1488.sh
-./linux-install-1.12.0.1488.sh
-
-RESOURCES_DIR=/app/source/resources
-
-# ensure the expected files are in place, in the resources dir
-if [ ! -f "$RESOURCES_DIR/ssl_keystore.jks" ]; then
- echo "$RESOURCES_DIR/ssl_keystore.jks does not exist; cannot run test" >&2
- exit 11
-fi
-
-if [ ! -f "$RESOURCES_DIR/krb5.conf" ]; then
- echo "$RESOURCES_DIR/krb5.conf does not exist; cannot run test" >&2
- exit 12
-fi
-
-if [ ! -f "$RESOURCES_DIR/client.keytab" ]; then
- echo "$RESOURCES_DIR/client.keytab does not exist; cannot run test" >&2
- exit 13
-fi
-
-# Copy the JDK cacerts file to our resources
-cp $JAVA_HOME/lib/security/cacerts $RESOURCES_DIR/cacerts-with-presto-ca.jks
-
-# Capture the Presto server self signed CA in PEM format
-openssl s_client -showcerts -connect presto-kerberos:7778 </dev/null \
- | openssl x509 -outform PEM >$RESOURCES_DIR/presto-ssl-root-ca.pem
-
-# Convert the Presto server self signed CA to DER format
-openssl x509 -outform der -in $RESOURCES_DIR/presto-ssl-root-ca.pem -out $RESOURCES_DIR/presto-ssl-root-ca.der
-
-# Add Presto's self signed CA to the truststore
-keytool -noprompt -import -alias presto-kerberos -keystore $RESOURCES_DIR/cacerts-with-presto-ca.jks \
- -storepass changeit -file $RESOURCES_DIR/presto-ssl-root-ca.der -trustcacerts
-
-# Set up the environment variables pointing to all of this, and run some tests
-DRIVERS=presto-jdbc \
-MB_ENABLE_PRESTO_JDBC_DRIVER=true \
-MB_PRESTO_JDBC_TEST_HOST=presto-kerberos \
-MB_PRESTO_JDBC_TEST_PORT=7778 \
-MB_PRESTO_JDBC_TEST_SSL=true \
-MB_PRESTO_JDBC_TEST_SSL_KEYSTORE_PATH=$RESOURCES_DIR/ssl_keystore.jks \
-MB_PRESTO_JDBC_TEST_SSL_KEYSTORE_PASSWORD=presto \
-MB_PRESTO_JDBC_TEST_SSL_TRUSTSTORE_PATH=$RESOURCES_DIR/cacerts-with-presto-ca.jks \
-MB_PRESTO_JDBC_TEST_SSL_TRUSTSTORE_PASSWORD=changeit \
-MB_PRESTO_JDBC_TEST_KERBEROS=true \
-MB_PRESTO_JDBC_TEST_KERBEROS_REMOTE_SERVICE_NAME=HTTP \
-MB_PRESTO_JDBC_TEST_KERBEROS_KEYTAB_PATH=$RESOURCES_DIR/client.keytab \
-MB_PRESTO_JDBC_TEST_KERBEROS_CONFIG_PATH=$RESOURCES_DIR/krb5.conf \
-clojure -X:dev:test:drivers:drivers-dev :only metabase.driver.presto-jdbc-test
diff --git a/.github/workflows/build-for-release.yml b/.github/workflows/build-for-release.yml
deleted file mode 100644
index 3f69dd0eb97..00000000000
--- a/.github/workflows/build-for-release.yml
+++ /dev/null
@@ -1,273 +0,0 @@
-# Builds a fresh release JAR from scratch at a given commit.
-#
-# You should almost never have to use this. The normal release flow relies on a
-# commit that has already been built and tested on its branch — see
-# `tag-for-release.yml`, which tags that existing JAR. Building from scratch
-# here is only for exceptional cases.
-name: Release 0 - Build Release Artifact
-run-name: Build ${{ inputs.version }}
-
-on:
- workflow_dispatch:
- inputs:
- version:
- description: 'Metabase version (e.g. v0.46.3)'
- type: string
- required: true
- commit:
- description: 'A full-length commit SHA-1 hash'
- required: true
-
-jobs:
- start-message:
- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 5
- steps:
- - uses: actions/checkout@v6
- with:
- sparse-checkout: |
- release
- .github
- - name: Prepare release scripts
- uses: ./.github/actions/prepare-release-scripts
- - name: Send build start message
- uses: actions/github-script@v9
- env:
- SLACK_RELEASE_CHANNEL: ${{ vars.SLACK_RELEASE_CHANNEL }}
- SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
- with:
- script: | # js
- const { sendPreReleaseMessage } = require('${{ github.workspace }}/release/dist/index.cjs');
-
- await sendPreReleaseMessage({
- github,
- owner: context.repo.owner,
- repo: context.repo.repo,
- version: '${{ inputs.version }}',
- runId: '${{ github.run_id }}',
- releaseSha: '${{ inputs.commit }}',
- userName: '${{ github.actor }}',
- channelName: '${{ vars.SLACK_RELEASE_CHANNEL }}',
- }).catch(console.error);
-
- trigger-milestone-check:
- # Posts a Slack FYI about issues that still need review for this release.
- # Informational only — nothing depends on it, so it never blocks the release.
- needs: [check-version, check-commit]
- permissions:
- contents: read
- issues: read
- uses: ./.github/workflows/release-milestone-check.yml
- secrets: inherit
- with:
- version: ${{ inputs.version }}
- commit: ${{ inputs.commit }}
-
- check-version:
- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 10
- outputs:
- ee: ${{ fromJson(steps.canonical_version.outputs.result).ee }}
- oss: ${{ fromJson(steps.canonical_version.outputs.result).oss }}
- steps:
- - name: Fail early on the incorrect version format
- if: ${{ !(startsWith(inputs.version,'v0.') || startsWith(inputs.version,'v1.')) }}
- run: |
- echo "The version format is invalid!"
- echo "It must start with either 'v0.' or 'v1.'."
- echo "Please, try again."
- exit 1
- - uses: actions/checkout@v6
- with:
- sparse-checkout: |
- release
- .github
- - name: Prepare release scripts
- uses: ./.github/actions/prepare-release-scripts
- - name: Get Release Version
- uses: actions/github-script@v9
- id: canonical_version
- with:
- script: | # js
- const { isValidVersionString, getCanonicalVersion, hasBeenReleased } = require('${{ github.workspace }}/release/dist/index.cjs');
-
- const version = '${{ inputs.version }}';
-
- if (!isValidVersionString(version)) {
- throw new Error("The version format is invalid! It must start with either 'v0.' or 'v1.'.");
- }
-
- const versions = {
- ee: getCanonicalVersion(version, 'ee'),
- oss: getCanonicalVersion(version, 'oss'),
- };
-
- const released = await hasBeenReleased({
- github,
- owner: context.repo.owner,
- repo: context.repo.repo,
- version: versions.oss,
- });
-
- if (released) {
- throw new Error("This version has already been released!", version);
- }
-
- return versions;
-
- check-commit:
- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 10
- steps:
- - name: Check out the code to verify the release branch
- uses: actions/checkout@v6
- with:
- fetch-depth: 0 # IMPORTANT! to get all the branches
- sparse-checkout: |
- release
- .github
- - name: Prepare release scripts
- uses: ./.github/actions/prepare-release-scripts
- - name: Get Release Branch
- uses: actions/github-script@v9
- id: release_branch
- with:
- result-encoding: string
- script: | # js
- const { getReleaseBranch } = require('${{ github.workspace }}/release/dist/index.cjs');
- const version = '${{ inputs.version }}';
- const releaseBranch = getReleaseBranch(version);
- return releaseBranch;
- - name: Ensure that the specified commit exists in the ${{ steps.release_branch.outputs.result }} release branch
- run: | # bash
- RELEASE_BRANCH=${{ steps.release_branch.outputs.result }}
- git checkout "$RELEASE_BRANCH"
- if git branch --contains ${{ inputs.commit }} | grep -q "$RELEASE_BRANCH"; then
- echo "Commit found in correct release branch"
- else
- echo "Commit not found in correct release branch"
- exit 1
- fi
- git checkout -
-
- get-build-requirements:
- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 10
- outputs:
- java_version: ${{ fromJson(steps.dependencies.outputs.result).java_version }}
- node_version: ${{ fromJson(steps.dependencies.outputs.result).node_version }}
- build_process: ${{ fromJson(steps.dependencies.outputs.result).build_process }}
- steps:
- - uses: actions/checkout@v6
- with:
- sparse-checkout: |
- release
- .github
- - name: Prepare release scripts
- uses: ./.github/actions/prepare-release-scripts
- - name: Get build dependencies
- uses: actions/github-script@v9
- id: dependencies
- with:
- script: | # js
- const { getBuildRequirements, getMajorVersion } = require('${{ github.workspace }}/release/dist/index.cjs');
-
- const version = '${{ inputs.version }}';
-
- const requirements = getBuildRequirements(version);
- const build_process = getMajorVersion(version) < 46 ? 'legacy' : 'new';
- console.log('Build process:', build_process);
-
- return {
- java_version: requirements.java,
- node_version: requirements.node,
- build_process,
- };
-
- manage-milestones:
- # Close the milestone we're about to ship and open the next one BEFORE building,
- # so any PRs that merge while the build is in flight are tagged with the next
- # milestone (not the one about to be released).
- #
- # Sequenced after trigger-milestone-check so that the check sees the milestone
- # in its open state. milestone-check is informational and must never block, so
- # we run regardless of its outcome; only preflight failures suppress this.
- needs: [check-version, check-commit, trigger-milestone-check]
- if: ${{ !cancelled() && needs.check-version.result == 'success' && needs.check-commit.result == 'success' }}
- permissions:
- contents: read
- issues: write
- uses: ./.github/workflows/release-milestone-manage.yml
- with:
- version: ${{ inputs.version }}
-
- build-uberjar-for-release:
- needs: [check-version, check-commit, get-build-requirements]
- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 50
- strategy:
- matrix:
- edition: [oss, ee]
- env:
- INTERACTIVE: false
- steps:
- - name: Get correct version number
- run: | # bash
- if [ '${{matrix.edition}}' == 'ee' ]; then
- version='${{ needs.check-version.outputs.ee }}'
- else
- version='${{ needs.check-version.outputs.oss }}'
- fi
-
- echo "VERSION=$version" >> "$GITHUB_ENV"
- - name: Setup Java ${{ needs.get-build-requirements.outputs.java_version }}
- uses: actions/[email protected]
- with:
- distribution: 'temurin'
- java-version: ${{ needs.get-build-requirements.outputs.java_version }}
- - name: Setup Node.js ${{ needs.get-build-requirements.outputs.node_version }}
- uses: actions/[email protected]
- with:
- node-version: ${{ needs.get-build-requirements.outputs.node_version }}.x
- - name: Check out the code
- uses: actions/checkout@v6
- with:
- ref: ${{ github.event.inputs.commit }}
- - name: Prepare front-end environment
- uses: ./.github/actions/prepare-frontend
-
- - name: Prepare back-end environment
- uses: ./.github/actions/prepare-backend
-
- - name: Build Metabase
- if: ${{ needs.get-build-requirements.outputs.build_process != 'legacy' }}
- run: ./bin/build.sh :edition :${{ matrix.edition }} :version "$VERSION"
-
- - name: Build Metabase (legacy)
- if: ${{ needs.get-build-requirements.outputs.build_process == 'legacy' }}
- run: cd ./bin/build-mb && clojure -X build/build! :edition :${{ matrix.edition }} :version "$VERSION"
-
- - name: Store commit's SHA-1 hash
- run: echo ${{ inputs.commit }} > COMMIT-ID
- shell: bash
- - run: mv ./target/uberjar/metabase.jar ./metabase.jar
- - name: Calculate SHA256 checksum
- run: sha256sum ./metabase.jar > SHA256.sum
- shell: bash
- - name: Upload JARs as artifact
- uses: actions/upload-artifact@v7
- with:
- name: metabase-release-${{ matrix.edition }}-${{ inputs.commit }}-uberjar
- path: |
- ./metabase.jar
- ./COMMIT-ID
- ./SHA256.sum
-
- run-pre-release-tests:
- name: Run pre-release tests for ${{ inputs.version }}
- needs: build-uberjar-for-release
- uses: ./.github/workflows/pre-release.yml
- secrets: inherit
- with:
- commit: ${{ inputs.commit }}
- version: ${{ inputs.version }}
diff --git a/.github/workflows/cache-generator.yml b/.github/workflows/cache-generator.yml
index 89bf78ba5ca..7a98bbf2208 100644
--- a/.github/workflows/cache-generator.yml
+++ b/.github/workflows/cache-generator.yml
@@ -93,10 +93,10 @@ jobs:
":dev:ee:ee-dev:drivers:drivers-dev:test:eastwood" # backend.yml (eastwood + eastwood/test)
":dev:ci:ee:ee-dev:drivers:drivers-dev:test" # test-driver.yml (ee) -> covers data.json:2.4.0
":dev:ci:oss:oss-dev:drivers:drivers-dev:test" # test-driver.yml (oss)
- ":dev:ci:test:ee:ee-dev" # backend.yml matrix (ee) + generative-query-test.yml
+ ":dev:ci:test:ee:ee-dev" # backend.yml matrix (ee)
":dev:ci:test:oss:oss-dev" # backend.yml matrix (oss)
":dev:ci:ee:ee-dev:test:cloverage" # backend-cloverage.yml
- ":dev:drivers:drivers-dev:test" # reset-cloud-test-dataset.yml + presto script
+ ":dev:drivers:drivers-dev:test" # reset-cloud-test-dataset.yml
":dev:drivers:build:build-dev:build-test:ci" # build-scripts.yml
":ee:drivers:check" # backend.yml (check; git dep athos/clj-check)
":ee:doc" # docs-generate-base.yml
diff --git a/.github/workflows/generative-query-test.yml b/.github/workflows/generative-query-test.yml
deleted file mode 100644
index 6cc9d228ea6..00000000000
--- a/.github/workflows/generative-query-test.yml
+++ /dev/null
@@ -1,58 +0,0 @@
-name: Generative Query Test
-
-on:
- workflow_dispatch:
- inputs:
- context-seed:
- description: 'Context seed (empty = random)'
- type: number
- iteration-seed:
- description: 'Iteration seed (for iteration replay, empty = dependent on Context seed)'
- type: number
- iterations:
- description: 'Iterations (number of test iterations, overrides seconds)'
- type: number
- seconds:
- description: 'Seconds (Number of seconds after which next iteration will not start)'
- test:
- description: 'Test or namespace to run'
- default: metabase.query-processor.generative-test/execution-with-cards-test
- ref:
- description: 'Ref to run generative tests on'
- default: qgen-ms2
-
-# TODO: Consider renaming the job in next milestones
-jobs:
- generative-query-test:
- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 90
- name: Generative Query Test
- env:
- CI: 'true'
- MB_GENTEST_CONTEXT_SEED: ${{ inputs.context-seed }}
- MB_GENTEST_ITERATION_SEED: ${{ inputs.iteration-seed }}
- MB_GENTEST_LIMIT_ITERATIONS: ${{ inputs.iterations }}
- MB_GENTEST_LIMIT_SECONDS: ${{ inputs.seconds }}
- MB_GENTEST_RUN: true
- steps:
- - uses: actions/checkout@v6
- with:
- ref: ${{ inputs.ref }}
- - name: Prepare back-end environment
- uses: ./.github/actions/prepare-backend
- - name: Prepare front-end environment
- uses: ./.github/actions/prepare-frontend
-
- - name: Run test
- run: clojure -X:dev:ci:test:ee:ee-dev :only '${{ inputs.test }}'
-
- - name: Publish Test Report (JUnit)
- uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0
- if: failure()
- with:
- path: "target/junit/**/*_test.xml"
- name: JUnit Test Report (Generative Query test)
- reporter: java-junit
- list-suites: failed
- list-tests: failed
- fail-on-error: false
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
deleted file mode 100644
index 1b653028d56..00000000000
--- a/.github/workflows/main.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-name: Slack Notification for New Feature Requests
-
-on:
- issues:
- types:
- - labeled
-jobs:
- on-issue-labeled:
- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 5
- if: |
- contains(github.event.issue.labels.*.name, 'Type:New Feature')
- steps:
- - name: Setting title
- uses: actions/github-script@v9
- id: vars
- env:
- ISSUE_TITLE: ${{ github.event.issue.title }}
- with:
- script: |
- core.setOutput('issue_title', process.env.ISSUE_TITLE.replaceAll(/"/g, '\\"'));
- - name: Send the issue ${{ github.event.issue.number }} to Slack
- id: slack
- uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3
- with:
- webhook: ${{ secrets.SLACK_FEATURE_REQUEST_TRIAGE_WEBHOOK_URL }}
- webhook-type: incoming-webhook
- payload: |
- {
- "blocks": [
- {
- "type": "section",
- "text": {
- "type": "mrkdwn",
- "text": "New incoming feature request: ${{ steps.vars.outputs.issue_title }}\n${{ github.event.issue.html_url }}"
- }
- }
- ]
- }
diff --git a/.github/workflows/presto-kerberos-integration-test.yml b/.github/workflows/presto-kerberos-integration-test.yml
deleted file mode 100644
index 6bc8eaae482..00000000000
--- a/.github/workflows/presto-kerberos-integration-test.yml
+++ /dev/null
@@ -1,109 +0,0 @@
-# this file uses MBA https://github.com/metabase/mba to spin up a Metabase
-# instance then runs the test script in the MBA instance sharing the same
-# network to interact with presto container
-# TODO: get rid of MBA and use actions/prepare-backend instead
-
-name: Kerberized Presto Integration Test
-
-on:
- pull_request:
- push:
- branches:
- - master
- - "release**"
- - "feature**"
- tags:
- - "**"
-
-jobs:
- files-changed:
- name: Check which files changed
- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 3
- outputs:
- backend_presto_kerberos: ${{ steps.changes.outputs.backend_presto_kerberos }}
- steps:
- - uses: actions/checkout@v6
- - name: Test which files changed
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
- id: changes
- with:
- token: ${{ github.token }}
- filters: .github/file-paths.yaml
-
- run-presto-kerberos-test:
- needs: files-changed
- if: needs.files-changed.outputs.backend_presto_kerberos == 'true'
- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 40
- steps:
- - name: Install babashka
- run: >
- mkdir -p /tmp/babashka-install \
- && cd /tmp/babashka-install \
- && curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install \
- && chmod +x install \
- && sudo ./install \
- && cd -
- - name: Checkout Metabase repository
- uses: actions/checkout@v6
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- - name: Check out Presto Kerberos Docker Compose
- uses: actions/checkout@v6
- with:
- repository: metabase/presto-kerberos-docker
- ref: master
- token: ${{ secrets.GITHUB_TOKEN }}
- path: presto-kerberos-docker
- - name: Bring up Presto+Kerberos cluster
- run: cd presto-kerberos-docker && docker compose up -d && cd ..
- - name: Run Presto test query from command line (sanity check)
- run: cd presto-kerberos-docker && ./test.sh && cd ..
- # Since we are managing the Docker containers from the GitHub action container, we need to copy all the
- # relevant resources now, into the resources dir for later consumption by the app
- - name: Copy Presto SSL keystore to resources
- run: docker cp presto-kerberos:/tmp/ssl_keystore.jks resources
- - name: Copy krb5.conf file to resources
- run: docker cp presto-kerberos:/etc/krb5.conf resources
- - name: Copy client.keytab file to resources
- run: docker cp presto-kerberos:/home/presto/client.keytab resources
-
- - name: Checkout mba
- uses: actions/checkout@v6
- with:
- repository: metabase/mba
- ref: master
- token: ${{ secrets.GITHUB_TOKEN }}
- path: mba-src
- - name: ls mba
- run: ls -latr mba-src
- - name: Symlink mba
- run: cd mba-src && sudo ln -s "$(pwd)/src/main.clj" /usr/local/bin/mba && chmod +x /usr/local/bin/mba && cd ..
-
- - name: Ensure mba
- run: which mba
-
- - name: Run Metabase via MBA
- run: mba --mb . --data-db postgres-data -n example.com compose config > mba-compose.yml
-
- - name: Adjust generated docker-compose config
- uses: actions/github-script@v9
- with:
- script: | #js
- const fs = require("fs");
- const config = fs.readFileSync("mba-compose.yml", "utf8");
- const newConfig = config.replace(
- /name:\s+example.com/,
- "name: example.com\n external: true",
- );
- fs.writeFileSync("mba-compose.yml", newConfig);
-
- - name: Print compose config
- run: cat mba-compose.yml
-
- - name: Run mba
- run: docker compose -f mba-compose.yml up -d
-
- - name: Run test script in MBA instance
- run: docker compose -f mba-compose.yml exec metabase .github/scripts/run-presto-kerberos-integration-test.sh
diff --git a/release/README.md b/release/README.md
deleted file mode 100644
index 708a0522002..00000000000
--- a/release/README.md
+++ /dev/null
@@ -1,62 +0,0 @@
-# Metabase Release
-
-There are 3 ways to release
-
-- Automated Release in CI
-- Release from a local environment
-- Release from a local environment without github
-
-## Automated Release
-
-1. Find the commit hash that you want to release
-2. Run the "build-for-release" action on github actions, inputting the version and commit hash (always select the master branch)
-3. Wait for the build and test jobs to finish
-4. If everything is successful, run the "release" action on github actions, inputting the version and commit hash again (always select the master branch)
-
-## GitHub action-less release
-
-> **Note**
-> Use this only if for some reason the automated release in CI is not available.
-
-1. Copy the `.env-template` file to `.env` and fill in all environment variables
-2. Run `cd release && bun install && bun run release-offline v0.77.77 1234567890abcdef1234567890abcdef12345678 --build`
-3. Wait for the build and test jobs to finish
-4. Test the built jar locally: `MB_JETTY_PORT=3033 java --add-opens java.base/java.nio=ALL-UNNAMED -jar target/uberjar/metabase.jar`
-5. If everything is successful, run `bun run release-offline v0.77.77 1234567890abcdef1234567890abcdef12345678 --publish`
-
-You can also run publish steps in isolation using these flags instead of `--publish`
-
-- `--check-jar`
-- `--s3`
-- `--docker`
-- `--version-info`
-- `--tag`
-- `--release-notes`
-
-Alternatively, you can put any properly built jar in `target/uberjar` folder and run the publish script from there.
-
-See also: [Tutorial Video](https://www.loom.com/share/a56f5a6904ff4f48acaa021846c90aeb)
-
-## Totally GitHub-less release
-
-If GitHub goes down completely :fire:, you can still build and publish a release to s3 and dockerhub from a local machine. Because we cannot use github to determine whether this release is a "latest" release, we must manually set that option on the command line.
-
-The build step will be exactly the same as the with-github github-action-less release, but the publish step will be different:
-
-```
-bun run release-offline v0.77.77 1234567890abcdef1234567890abcdef12345678 --publish --without-github --not-latest
-```
-or
-```
-bun run release-offline v0.77.77 1234567890abcdef1234567890abcdef12345678 --publish --without-github --latest
-```
-
-The order of the arguments matters, the script will yell at you if you put the flags in the wrong order. We could make this more robust, but it's probably not worth the effort since hopefully we won't ever have to use this :crossed_fingers:.
-
-## Utilities
-
-In case you want to preview release notes generation, or re-generate them after a release has been built, you can use this command.
-
-```sh
-bun run generate-changelog v0.77.0 > changelog.log
-```
diff --git a/test/metabase/query_processor/generative_test.clj b/test/metabase/query_processor/generative_test.clj
deleted file mode 100644
index 0bec3db426d..00000000000
--- a/test/metabase/query_processor/generative_test.clj
+++ /dev/null
@@ -1,72 +0,0 @@
-(ns metabase.query-processor.generative-test
- "Query processor generative tests."
- (:require
- [clojure.test :refer [is testing]]
- [metabase.lib.convert :as lib.convert]
- [metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
- [metabase.lib.test-util :as lib.tu]
- [metabase.lib.test-util.generators :as lib.tu.gen]
- [metabase.query-processor.test :as qp]
- [metabase.test :as mt]
- [metabase.test.gentest :as gt]
- [metabase.test.util.generators.jvm :as tu.gen.jvm]
- [metabase.test.util.random :as tu.rng]))
-
-(set! *warn-on-reflection* true)
-
-(comment
- #_:clj-kondo/ignore
- (alter-var-root #'environ.core/env assoc
- :mb-gentest-run "true"
- :mb-gentest-context-seed "1"))
-
-(defn- valid-col?
- [maybe-col]
- (and (map? maybe-col)
- ((every-pred :name :base_type :display_name) maybe-col)))
-
-(gt/defgentest basic-query-execution-test
- (let [mp (mt/metadata-provider)]
- (gt/iterate
- #_{:gentest.default-limit/seconds 5}
- {:gentest.default-limit/iterations 1}
- [base-query (lib/query mp (tu.rng/rand-nth (lib.metadata/tables mp)))
- limit (inc (tu.rng/rand-int lib.tu.gen/sane-iterations-limit))]
- (doseq [query (lib.tu.gen/random-queries-from base-query limit)]
- (gt/testing ["query" query]
- (let [mp (lib.tu/mock-metadata-provider
- mp
- {:cards [{:id 1
- :dataset-query query}]})
- result (qp/process-query (lib/query mp (lib.metadata/card mp 1)))]
- (testing "Successful query execution"
- (is (= :completed
- (:status result))))
- (testing "At least one column returned"
- (is (<= 1 (count (mt/cols result))))
- (is (every? valid-col? (mt/cols result))))))))))
-
-(gt/defgentest execution-with-cards-test
- (let [mp (mt/metadata-provider)]
- ;; TODO: What is the reasonable amount of cards to use in context? Hardcoding 8 for now as 8 tables are avail in
- ;; mp's test-data.
- (tu.gen.jvm/with-random-cards mp 8
- (gt/iterate
- #_{:gentest.default-limit/seconds 5}
- {:gentest.default-limit/iterations 1}
- [base-query (lib/query mp (tu.rng/rand-nth (concat (lib.metadata/tables mp) &cards)))
- limit (inc (tu.rng/rand-int lib.tu.gen/sane-iterations-limit))]
- (doseq [query (lib.tu.gen/random-queries-from base-query limit)]
- (gt/testing ["query" query]
- (let [mp (lib.tu/mock-metadata-provider
- mp
- {:cards [{:id 1
- :dataset-query (lib.convert/->legacy-MBQL query)}]})
- result (qp/process-query (lib/query mp (lib.metadata/card mp 1)))]
- (testing "Successful query execution"
- (is (= :completed
- (:status result))))
- (testing "At least one column returned"
- (is (<= 1 (count (mt/cols result))))
- (is (every? valid-col? (mt/cols result)))))))))))
diff --git a/test/metabase/test/gentest.clj b/test/metabase/test/gentest.clj
deleted file mode 100644
index 12391578407..00000000000
--- a/test/metabase/test/gentest.clj
+++ /dev/null
@@ -1,123 +0,0 @@
-(ns metabase.test.gentest
- (:refer-clojure :exclude [iterate])
- (:require
- [clojure.test]
- [java-time.api :as t]
- [metabase.config.core :as config]
- [metabase.test.util.random :as tu.rng]
- [metabase.util :as u]
- [metabase.util.log :as log])
- (:import
- [java.util Random]))
-
-(set! *warn-on-reflection* true)
-
-;; TODO: In later ms add option to override via spec!!!
-(defn limit-spec->limit-fn
- [limit-spec]
- (let [env-iterations (config/config-int :mb-gentest-limit-iterations)
- env-seconds (config/config-int :mb-gentest-limit-seconds)
- limit-spec (cond env-iterations {:gentest.default-limit/iterations env-iterations}
- env-seconds {:gentest.default-limit/seconds env-seconds}
- :else limit-spec)]
- (log/debugf "limit-spec->limit-fn limit spec: %s" (u/pprint-to-str limit-spec))
- (or (and (map? limit-spec)
- (condp #(get %2 %1) limit-spec
- :gentest.default-limit/iterations
- :>> (fn [iterations]
- (let [counter (atom -1)]
- (fn []
- (< (swap! counter inc) iterations))))
-
- :gentest.default-limit/seconds
- :>> (fn [seconds]
- (let [stop-time (t/+ (t/local-date-time) (t/seconds seconds))]
- (fn []
- (let [now (t/local-date-time)]
- (t/< now stop-time)))))
-
- nil))
- (let [counter (atom -1)]
- (fn []
- (< (swap! counter inc) 1))))))
-
-;; It is written this way in case we decide for multiple parameters in fmtstr.
-(defmacro testing
- [[fmtstr & args] & body]
- `(clojure.test/testing (format ~(str "\nwith " fmtstr "\n%s\n")
- ~@(map (fn [form]
- `(u/pprint-to-str ~form))
- args))
- ~@body))
-
-(defonce startup-context-seed (doto (or (config/config-long :mb-gentest-context-seed)
- (.nextLong ^Random (Random.)))
- (as-> $ (log/infof "STARTUP context-seed %d" $))))
-(defn initial-context-seed
- "Context seed is either (1) from environment or (2) the one set during the first load of this namespace."
- []
- (or (config/config-long :mb-gentest-context-seed)
- startup-context-seed))
-
-(defn initial-iteration-seed
- "Iteration seed is either (1) from environment or (2) the [[initial-context-seed]]."
- []
- (or (config/config-long :mb-gentest-iteration-seed)
- (initial-context-seed)))
-
-(defn do-iterate
- [limit-spec thunk]
- (let [limit-fn (limit-spec->limit-fn limit-spec)
- ;; Atom is used to overcome recur over binding.
- iteration-seed (atom (initial-iteration-seed))]
- (loop [iteration-index 0]
- (when (limit-fn)
- (testing ["iteration index" iteration-index]
- (testing ["iteration seed" @iteration-seed]
- (log/tracef "do-iterate: iteration-index: %s" iteration-index)
- (log/tracef "do-iterate: iteration-seed: %s" @iteration-seed)
- (binding [tu.rng/*generator* (Random. @iteration-seed)]
- (try
- (thunk)
- (catch Throwable t
- ;; TODO: Consider setting actual to cause instead.
- (clojure.test/do-report {:type :error, :message nil, :expected nil, :actual t}))
- (finally
- (reset! iteration-seed (.nextLong ^Random tu.rng/*generator*)))))))
- (recur (inc iteration-index))))))
-
-(defmacro iterate
- [limit-spec bindings & body]
- (let [safer-bindings (mapcat (fn [[sym form]]
- [sym `(try
- ~form
- (catch Exception e#
- (throw (ex-info "Binding error"
- {:type :error
- ;; form is unused atm, maybe will be in do-with-
- :form (quote ~form)}
- e#))))])
- (partition 2 bindings))]
- `(do-iterate ~limit-spec
- (fn []
- (let [~@safer-bindings]
- (testing ["iteration bindings" ~(mapv (fn [sym#]
- [`'~sym# sym#])
- (take-nth 2 safer-bindings))]
- (try
- ~@body
- (catch Exception e#
- (throw (ex-info "Execution error"
- {:type :error}
- e#))))))))))
-
-;; TODO: Later milestones: Because of debug log it makes sense to do-defgentest.
-(defmacro defgentest
- [test-sym & body]
- `(clojure.test/deftest ~test-sym
- (when (config/config-bool :mb-gentest-run)
- (let [seed# (initial-context-seed)]
- (binding [tu.rng/*generator* (Random. seed#)]
- (log/debugf "defgentest: seed: %s" (pr-str seed#))
- (testing ["context-seed" seed#]
- ~@body))))))
diff --git a/test_config/log4j2-test.xml b/test_config/log4j2-test.xml
index 6b7cb4a00c1..731f9bb7367 100644
--- a/test_config/log4j2-test.xml
+++ b/test_config/log4j2-test.xml
@@ -7,12 +7,6 @@
<ThresholdFilter level="FATAL" onMatch="ACCEPT" onMismatch="DENY"/>
</Filters>
</Console>
- <Console name="ConsoleInfoLevel" target="SYSTEM_OUT" follow="true">
- <PatternLayout pattern="[%t] %-5level %logger{36} - %msg %notEmpty{%X}%n"/>
- <Filters>
- <ThresholdFilter level="FATAL" onMatch="ACCEPT" onMismatch="DENY"/>
- </Filters>
- </Console>
<File name="File" fileName="logs/test-log.json" append="false">
<JsonLayout locationInfo="false" properties="true" compact="true" eventEol="true"/>
<Filters>
@@ -31,12 +25,6 @@
resources/log4j2.xml, minus the per-session RoutingAppender (tests don't read the files). -->
<Logger name="metabase.ai-tracing.log" level="info" additivity="false"/>
- <Logger name="metabase.test.gentest" level="INFO" additivity="false">
- <AppenderRef ref="ConsoleInfoLevel"/>
- <AppenderRef ref="Console"/>
- <AppenderRef ref="File"/>
- </Logger>
-
<Root level="INFO">
<AppenderRef ref="Console"/>
<AppenderRef ref="File"/>