Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add workflow_dispatch action to reset cloud datasets (#70744)

Ngoc Khuatf7fd7fe5c5f2
diff --git a/.github/workflows/reset-cloud-test-dataset.yml b/.github/workflows/reset-cloud-test-dataset.yml
new file mode 100644
index 00000000000..48911ce707a
--- /dev/null
+++ b/.github/workflows/reset-cloud-test-dataset.yml
@@ -0,0 +1,63 @@
+name: Reset Cloud Test Dataset
+
+on:
+ workflow_dispatch:
+ inputs:
+ driver:
+ description: 'Driver to reset dataset for'
+ required: true
+ default: 'snowflake'
+ type: choice
+ options:
+ - snowflake
+ - bigquery-cloud-sdk
+ - redshift
+ - databricks
+ dataset-name:
+ description: 'Dataset name to drop (e.g., test-data)'
+ required: true
+ default: 'test-data'
+ type: string
+
+jobs:
+ reset-dataset:
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ env:
+ CI: 'true'
+ DRIVERS: ${{ inputs.driver }}
+ # Snowflake
+ MB_SNOWFLAKE_TEST_USER: METABASE CI
+ MB_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.MB_SNOWFLAKE_TEST_ACCOUNT }}
+ MB_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.MB_SNOWFLAKE_TEST_PASSWORD }}
+ MB_SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PRIVATE_KEY }}
+ MB_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.MB_SNOWFLAKE_TEST_WAREHOUSE }}
+ # BigQuery
+ MB_BIGQUERY_TEST_PROJECT_ID: ${{ secrets.BIGQUERY_TEST_PROJECT_ID }}
+ MB_BIGQUERY_TEST_CLIENT_ID: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_ID }}
+ MB_BIGQUERY_TEST_CLIENT_SECRET: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_SECRET }}
+ MB_BIGQUERY_TEST_ACCESS_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_ACCESS_TOKEN }}
+ MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}
+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}
+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}
+ # Redshift
+ MB_REDSHIFT_TEST_USER: metabase_ci
+ MB_REDSHIFT_TEST_DB: testdb
+ MB_REDSHIFT_TEST_HOST: ${{ secrets.MB_REDSHIFT_TEST_HOST }}
+ MB_REDSHIFT_TEST_PASSWORD: ${{ secrets.MB_REDSHIFT_TEST_PASSWORD }}
+ # Databricks
+ MB_DATABRICKS_TEST_HOST: ${{ secrets.MB_DATABRICKS_JDBC_TEST_HOST }}
+ MB_DATABRICKS_TEST_HTTP_PATH: ${{ secrets.MB_DATABRICKS_JDBC_TEST_HTTP_PATH }}
+ MB_DATABRICKS_TEST_TOKEN: ${{ secrets.MB_DATABRICKS_JDBC_TEST_TOKEN }}
+ MB_DATABRICKS_TEST_CATALOG: metabase_ci
+ MB_DATABRICKS_TEST_CLIENT_ID: ${{ secrets.MB_DATABRICKS_JDBC_TEST_CLIENT_ID }}
+ MB_DATABRICKS_TEST_OAUTH_SECRET: ${{ secrets.MB_DATABRICKS_JDBC_TEST_OAUTH_SECRET }}
+ steps:
+ - uses: actions/checkout@v4
+ - uses: ./.github/actions/prepare-backend
+ - name: Drop dataset
+ run: >-
+ clojure -X:dev:drivers:drivers-dev:test
+ metabase.test.data.impl/drop-dataset!
+ :driver '"${{ inputs.driver }}"'
+ :dataset-name '"${{ inputs.dataset-name }}"'
diff --git a/test/metabase/test/data/impl.clj b/test/metabase/test/data/impl.clj
index 236aedc37b9..ee83c3c456a 100644
--- a/test/metabase/test/data/impl.clj
+++ b/test/metabase/test/data/impl.clj
@@ -383,3 +383,50 @@
*db-id-fn* #(u/the-id (db-fn))
*dbdef-used-to-create-db* dbdef]
(f))))
+
+(defn- log! [fmt & args]
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (println (apply format fmt args)))
+
+(defn drop-dataset!
+ "Drop a test dataset by driver and name. Resolves the dataset name to its definition
+ and calls [[metabase.test.data.interface/destroy-db!]].
+
+ Can be called from the REPL or via clojure -X:
+
+ clojure -X:dev:drivers:drivers-dev:test metabase.test.data.impl/drop-dataset! :driver '\"snowflake\"' :dataset-name '\"test-data\"'"
+ [{:keys [driver dataset-name]}]
+ (let [driver (keyword driver)
+ _ (classloader/require 'metabase.test.data.dataset-definitions)
+ dataset-def (resolve-dataset-definition
+ 'metabase.test.data.dataset-definitions
+ (symbol dataset-name))
+ dbdef (tx/get-dataset-definition dataset-def)]
+ (log! "[%s] Dropping dataset '%s'..." (name driver) dataset-name)
+ (tx/destroy-db! driver dbdef)
+ (log! "[%s] Done." (name driver))))
+
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
+(defn test-drop-dataset
+ "Like [[drop-dataset!]] but checks existence before and after, verifying deletion.
+ Name lacks `!` because clojure -X cannot resolve function names ending in `!`.
+
+ clojure -X:dev:drivers:drivers-dev:test metabase.test.data.impl/test-drop-dataset :driver '\"snowflake\"' :dataset-name '\"test-data\"'"
+ [{:keys [driver dataset-name] :as opts}]
+ (let [driver (keyword driver)
+ _ (classloader/require 'metabase.test.data.dataset-definitions)
+ dataset-def (resolve-dataset-definition
+ 'metabase.test.data.dataset-definitions
+ (symbol dataset-name))
+ dbdef (tx/get-dataset-definition dataset-def)]
+ (log! "[%s] Checking if dataset '%s' exists..." (name driver) dataset-name)
+ (if-not (tx/dataset-already-loaded? driver dbdef)
+ (log! "[%s] Dataset '%s' does not exist, nothing to drop." (name driver) dataset-name)
+ (do
+ (log! "[%s] Dataset '%s' exists, proceeding with drop." (name driver) dataset-name)
+ (drop-dataset! opts)
+ (log! "[%s] Verifying dataset '%s' was deleted..." (name driver) dataset-name)
+ (if (tx/dataset-already-loaded? driver dbdef)
+ (do (log! "[%s] FAIL: dataset '%s' still exists after drop!" (name driver) dataset-name)
+ (System/exit 1))
+ (log! "[%s] PASS: dataset '%s' has been deleted." (name driver) dataset-name))))))