Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Remove be-tests-databricks-multi-catalog-ee job. (#81127)

technomancy9d216767d063
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index 239bd76fc3f..b7c783b8d33 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -338,15 +338,9 @@ jobs:
:fail-fast? true
:partition/total 2
:partition/index 1
- - name: Driver Tests (Multi-Catalog)
- multi-level-schema: true
- test-args: >-
- :only metabase.driver.databricks-test
- :fail-fast? true
env:
CI: 'true'
DRIVERS: databricks
- MB_DATABRICKS_TEST_MULTI_LEVEL_SCHEMA: ${{ matrix.job.multi-level-schema }}
MB_DATABRICKS_TEST_CATALOG: 'metabase_ci'
MB_DATABRICKS_TEST_CATALOG_ROUTING: 'metabase_routing_catalog'
MB_DATABRICKS_TEST_CLIENT_ID: ${{ secrets.MB_DATABRICKS_JDBC_TEST_CLIENT_ID }}
diff --git a/modules/drivers/databricks/test/metabase/driver/databricks_test.clj b/modules/drivers/databricks/test/metabase/driver/databricks_test.clj
index 697ba4d614f..0cbd24f5441 100644
--- a/modules/drivers/databricks/test/metabase/driver/databricks_test.clj
+++ b/modules/drivers/databricks/test/metabase/driver/databricks_test.clj
@@ -22,158 +22,175 @@
[metabase.util :as u]
[toucan2.core :as t2]))
-(defn- maybe-qualify-schema
- [schema]
- (let [multi-level? (tx/db-test-env-var :databricks :multi-level-schema false)
- catalog (get-in (mt/db) [:details :catalog])]
- (cond->> schema multi-level? (str catalog "."))))
+(def ^:dynamic *multi-level?* false)
+
+;; certain tests need to be run twice; once with the multi-level schema and once without.
+(defmacro with-and-without-multi-level [& body]
+ ;; run it once without multi-level and once with multi-level
+ `(do ~@body
+ (binding [*multi-level?* true]
+ (testing "with multi-level schema"
+ ~@body))))
+
+(defn- maybe-qualify-schema [schema]
+ (if *multi-level?*
+ (str (get-in (mt/db) [:details :catalog]) "." schema)
+ schema))
+
+(defn db []
+ (if *multi-level?*
+ (-> (mt/db)
+ (update-in [:details :schema-filters-patterns]
+ #(str "metabase_ci." %))
+ (assoc-in [:details :multi-level-schema] true))
+ (mt/db)))
;; Because the datasets that are tested are preloaded, it is fine just to modify the database details to sync other schemas.
(deftest ^:parallel sync-test
- (mt/test-driver
- :databricks
- (testing "`driver/describe-database` implementation returns expected results for inclusion of test-data schema."
- (is (= #{{:name "venues", :schema (maybe-qualify-schema "test-data"), :description nil}
- {:name "checkins", :schema (maybe-qualify-schema "test-data"), :description nil}
- {:name "users", :schema (maybe-qualify-schema "test-data"), :description nil}
- {:name "people", :schema (maybe-qualify-schema "test-data"), :description nil}
- {:name "categories", :schema (maybe-qualify-schema "test-data"), :description nil}
- {:name "reviews", :schema (maybe-qualify-schema "test-data"), :description nil}
- {:name "orders", :schema (maybe-qualify-schema "test-data"), :description nil}
- {:name "products", :schema (maybe-qualify-schema "test-data"), :description nil}}
- (into #{} (:tables (driver/describe-database :databricks (mt/db)))))))
- (testing "`driver/describe-database` returns expected results for `all` schema filters."
- (let [actual-tables (update (driver/describe-database :databricks (-> (mt/db)
- (update :details dissoc :schema-filters-patterns)
- (update :details assoc :schema-filters-type "all")))
- ;; `:tables` is a reducible (streamed); realize it for the set ops below
- :tables #(into #{} %))]
- (testing "tables from multiple schemas were found"
- (are [name schema] (contains? (:tables actual-tables)
- {:name name, :schema schema, :description nil})
- "venues" (maybe-qualify-schema "test-data")
- "checkins" (maybe-qualify-schema "test-data")
- "airport" (maybe-qualify-schema "airports")
- "bird" (maybe-qualify-schema "bird-flocks")))
- (testing "information_schema is excluded"
- (is (empty? (filter #(str/includes? "information_schema" (:schema %)) (:tables actual-tables)))))))
- (testing "`driver/describe-database` returns expected results for `exclusion` schema filters."
- (let [actual-tables (update (driver/describe-database :databricks (update (mt/db) :details assoc
- :schema-filters-patterns (maybe-qualify-schema "test-data")
- :schema-filters-type "exclusion"))
- ;; `:tables` is a reducible (streamed); realize it for the set ops below
- :tables #(into #{} %))]
- (testing "tables from multiple schemas were found"
- (is (not (contains? (set (map :schema (:tables actual-tables))) (maybe-qualify-schema "test-data"))))
- (is (contains? (:tables actual-tables) {:name "airport", :schema (maybe-qualify-schema "airports"), :description nil}))
- (is (contains? (:tables actual-tables) {:name "bird", :schema (maybe-qualify-schema "bird-flocks"), :description nil})))))))
+ (with-and-without-multi-level
+ (mt/test-driver :databricks
+ (testing "`driver/describe-database` implementation returns expected results for inclusion of test-data schema."
+ (is (= #{{:name "venues", :schema (maybe-qualify-schema "test-data"), :description nil}
+ {:name "checkins", :schema (maybe-qualify-schema "test-data"), :description nil}
+ {:name "users", :schema (maybe-qualify-schema "test-data"), :description nil}
+ {:name "people", :schema (maybe-qualify-schema "test-data"), :description nil}
+ {:name "categories", :schema (maybe-qualify-schema "test-data"), :description nil}
+ {:name "reviews", :schema (maybe-qualify-schema "test-data"), :description nil}
+ {:name "orders", :schema (maybe-qualify-schema "test-data"), :description nil}
+ {:name "products", :schema (maybe-qualify-schema "test-data"), :description nil}}
+ (into #{} (:tables (driver/describe-database :databricks (db)))))))
+ (testing "`driver/describe-database` returns expected results for `all` schema filters."
+ (let [actual-tables (update (driver/describe-database :databricks (-> (db)
+ (update :details dissoc :schema-filters-patterns)
+ (update :details assoc :schema-filters-type "all")))
+ ;; `:tables` is a reducible (streamed); realize it for the set ops below
+ :tables #(into #{} %))]
+ (testing "tables from multiple schemas were found"
+ (are [name schema] (contains? (:tables actual-tables)
+ {:name name, :schema schema, :description nil})
+ "venues" (maybe-qualify-schema "test-data")
+ "checkins" (maybe-qualify-schema "test-data")
+ "airport" (maybe-qualify-schema "airports")
+ "bird" (maybe-qualify-schema "bird-flocks")))
+ (testing "information_schema is excluded"
+ (is (empty? (filter #(str/includes? "information_schema" (:schema %)) (:tables actual-tables)))))))
+ (testing "`driver/describe-database` returns expected results for `exclusion` schema filters."
+ (let [actual-tables (update (driver/describe-database :databricks (update (db) :details assoc
+ :schema-filters-patterns (maybe-qualify-schema "test-data")
+ :schema-filters-type "exclusion"))
+ ;; `:tables` is a reducible (streamed); realize it for the set ops below
+ :tables #(into #{} %))]
+ (testing "tables from multiple schemas were found"
+ (is (not (contains? (set (map :schema (:tables actual-tables))) (maybe-qualify-schema "test-data"))))
+ (is (contains? (:tables actual-tables) {:name "airport", :schema (maybe-qualify-schema "airports"), :description nil}))
+ (is (contains? (:tables actual-tables) {:name "bird", :schema (maybe-qualify-schema "bird-flocks"), :description nil}))))))))
(deftest ^:parallel describe-fields-test
- (mt/test-driver
- :databricks
- (let [fields (vec (driver/describe-fields :databricks (mt/db) {:schema-names [(maybe-qualify-schema "test-data")]
- :table-names ["orders"]}))]
- (testing "Underlying query returns only fields from selected catalog"
- (is (= 9 (count fields))))
- (testing "Expected fields are returned"
- (is (= (into #{}
- (map #(update % :table-schema maybe-qualify-schema))
- #{{:table-schema "test-data"
- :table-name "orders"
- :pk? true
- :name "id"
- :database-type "int"
- :database-position 0
- :base-type :type/Integer
- :json-unfolding false}
- {:table-schema "test-data"
- :table-name "orders"
- :pk? false
- :name "user_id"
- :database-type "int"
- :database-position 1
- :base-type :type/Integer
- :json-unfolding false}
- {:table-schema "test-data"
- :table-name "orders"
- :pk? false
- :name "product_id"
- :database-type "int"
- :database-position 2
- :base-type :type/Integer
- :json-unfolding false}
- {:table-schema "test-data"
- :table-name "orders"
- :pk? false
- :name "subtotal"
- :database-type "double"
- :database-position 3
- :base-type :type/Float
- :json-unfolding false}
- {:table-schema "test-data"
- :table-name "orders"
- :pk? false
- :name "tax"
- :database-type "double"
- :database-position 4
- :base-type :type/Float
- :json-unfolding false}
- {:table-schema "test-data"
- :table-name "orders"
- :pk? false
- :name "total"
- :database-type "double"
- :database-position 5
- :base-type :type/Float
- :json-unfolding false}
- {:table-schema "test-data"
- :table-name "orders"
- :pk? false
- :name "discount"
- :database-type "double"
- :database-position 6
- :base-type :type/Float
- :json-unfolding false}
- {:table-schema "test-data"
- :table-name "orders"
- :pk? false
- :name "created_at"
- :database-type "timestamp"
- :database-position 7
- :base-type :type/DateTimeWithLocalTZ
- :json-unfolding false}
- {:table-schema "test-data"
- :table-name "orders"
- :pk? false
- :name "quantity"
- :database-type "int"
- :database-position 8
- :base-type :type/Integer
- :json-unfolding false}})
- (set fields)))))))
+ (with-and-without-multi-level
+ (mt/test-driver :databricks
+ (let [fields (vec (driver/describe-fields :databricks (db) {:schema-names [(maybe-qualify-schema "test-data")]
+ :table-names ["orders"]}))]
+ (testing "Underlying query returns only fields from selected catalog"
+ (is (= 9 (count fields))))
+ (testing "Expected fields are returned"
+ (is (= (into #{}
+ (map #(update % :table-schema maybe-qualify-schema))
+ #{{:table-schema "test-data"
+ :table-name "orders"
+ :pk? true
+ :name "id"
+ :database-type "int"
+ :database-position 0
+ :base-type :type/Integer
+ :json-unfolding false}
+ {:table-schema "test-data"
+ :table-name "orders"
+ :pk? false
+ :name "user_id"
+ :database-type "int"
+ :database-position 1
+ :base-type :type/Integer
+ :json-unfolding false}
+ {:table-schema "test-data"
+ :table-name "orders"
+ :pk? false
+ :name "product_id"
+ :database-type "int"
+ :database-position 2
+ :base-type :type/Integer
+ :json-unfolding false}
+ {:table-schema "test-data"
+ :table-name "orders"
+ :pk? false
+ :name "subtotal"
+ :database-type "double"
+ :database-position 3
+ :base-type :type/Float
+ :json-unfolding false}
+ {:table-schema "test-data"
+ :table-name "orders"
+ :pk? false
+ :name "tax"
+ :database-type "double"
+ :database-position 4
+ :base-type :type/Float
+ :json-unfolding false}
+ {:table-schema "test-data"
+ :table-name "orders"
+ :pk? false
+ :name "total"
+ :database-type "double"
+ :database-position 5
+ :base-type :type/Float
+ :json-unfolding false}
+ {:table-schema "test-data"
+ :table-name "orders"
+ :pk? false
+ :name "discount"
+ :database-type "double"
+ :database-position 6
+ :base-type :type/Float
+ :json-unfolding false}
+ {:table-schema "test-data"
+ :table-name "orders"
+ :pk? false
+ :name "created_at"
+ :database-type "timestamp"
+ :database-position 7
+ :base-type :type/DateTimeWithLocalTZ
+ :json-unfolding false}
+ {:table-schema "test-data"
+ :table-name "orders"
+ :pk? false
+ :name "quantity"
+ :database-type "int"
+ :database-position 8
+ :base-type :type/Integer
+ :json-unfolding false}})
+ (set fields))))))))
(deftest ^:parallel describe-fks-test
- (mt/test-driver :databricks
- (let [fks (vec (driver/describe-fks :databricks
- (lib.metadata/database (mt/metadata-provider))
- {:schema-names [(maybe-qualify-schema "test-data")]
- :table-names ["orders"]}))]
- (testing "Only fks from current catalog are registered"
- (is (= 2 (count fks))))
- (testing "Expected fks are returned"
- (is (= #{{:fk-table-schema (maybe-qualify-schema "test-data")
- :fk-table-name "orders"
- :fk-column-name "product_id"
- :pk-table-schema (maybe-qualify-schema "test-data")
- :pk-table-name "products"
- :pk-column-name "id"}
- {:fk-table-schema (maybe-qualify-schema "test-data")
- :fk-table-name "orders"
- :fk-column-name "user_id"
- :pk-table-schema (maybe-qualify-schema "test-data")
- :pk-table-name "people"
- :pk-column-name "id"}}
- (set fks)))))))
+ (with-and-without-multi-level
+ (mt/test-driver :databricks
+ (let [fks (driver/describe-fks :databricks
+ (assoc (lib.metadata/database (mt/metadata-provider))
+ :details (:details (db)))
+ {:schema-names [(maybe-qualify-schema "test-data")]
+ :table-names ["orders"]})]
+ (testing "Expected fks are returned"
+ (is (= #{{:fk-table-schema (maybe-qualify-schema "test-data")
+ :fk-table-name "orders"
+ :fk-column-name "product_id"
+ :pk-table-schema (maybe-qualify-schema "test-data")
+ :pk-table-name "products"
+ :pk-column-name "id"}
+ {:fk-table-schema (maybe-qualify-schema "test-data")
+ :fk-table-name "orders"
+ :fk-column-name "user_id"
+ :pk-table-schema (maybe-qualify-schema "test-data")
+ :pk-table-name "people"
+ :pk-column-name "id"}}
+ (set fks))))))))
(deftest ^:parallel multi-level-schema-filter-sql-test
(testing "multi-catalog sync filters on catalog and schema with equalities, not a row-constructor IN (GHY-4263)"
@@ -345,163 +362,149 @@
:subname))))))
(deftest multi-level-schema-test
- (mt/test-driver
- :databricks
- ;; skip if running in multi-level since we are manipulating it in this test
- (when-not (tx/db-test-env-var :databricks :multi-level-schema false)
- (let [details (get (mt/db) :details)
- ;; metabase_ci_multicatalog.test_schema.test (id, name)
- multicatalog (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")
- multicatalog-schema (tx/db-test-env-var :databricks :multicatalog-schema "test_schema")
- multi-pattern (format "%s.%s,%s.%s"
- (:catalog details)
- (:schema-filters-patterns details)
- multicatalog
- multicatalog-schema)]
- (mt/with-temp [:model/Database {db-id :id :as db} {:engine :databricks :details details}]
+ (mt/test-driver :databricks
+ (let [details (get (mt/db) :details)
+ ;; metabase_ci_multicatalog.test_schema.test (id, name)
+ multicatalog (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")
+ multicatalog-schema (tx/db-test-env-var :databricks :multicatalog-schema "test_schema")
+ multi-pattern (format "%s.%s,%s.%s"
+ (:catalog details)
+ (:schema-filters-patterns details)
+ multicatalog
+ multicatalog-schema)]
+ (mt/with-temp [:model/Database {db-id :id :as db} {:engine :databricks :details details}]
+ (mt/with-db db
+ (testing "With multi-level-schema default (off)"
+ (sync/sync-database! (mt/db))
+ (is (= #{"test-data"} (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)))
+ (is (= 52 (count (t2/select
+ :model/Field
+ :table_id [:in (t2/select-fn-set :id :model/Table :db_id (mt/id) :active true)]
+ :active true))))
+ (is (= 1 (count (mt/rows (mt/run-mbql-query venues {:limit 1})))))))
+ (testing "With multi-level-schema on"
+ (t2/update! :model/Database db-id {:details (assoc details
+ :multi-level-schema true
+ :schema-filters-patterns multi-pattern)})
+ (mt/with-db
+ (t2/select-one :model/Database db-id)
+ (sync/sync-database! (mt/db))
+ (is (= #{(format "%s.%s" (:catalog details) "test-data")
+ (format "%s.%s" multicatalog multicatalog-schema)}
+ (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)))
+ ;; Adds four fields for metabase_ci_multicatalog.test_schema.test id,name,ci_venue_id,drivers_venue_id
+ (is (= 56 (count (t2/select
+ :model/Field
+ :table_id [:in (t2/select-fn-set :id :model/Table :db_id (mt/id) :active true)]
+ :active true))))
+ (is (= 1 (count (mt/rows (mt/run-mbql-query venues {:limit 1})))))))
+ (testing "With multi-level-schema off"
+ (t2/update! :model/Database db-id {:details (assoc details :multi-level-schema false)})
(mt/with-db
- db
- (testing "With multi-level-schema default (off)"
- (sync/sync-database! (mt/db))
- (is (= #{"test-data"} (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)))
- (is (= 52 (count (t2/select
- :model/Field
- :table_id [:in (t2/select-fn-set :id :model/Table :db_id (mt/id) :active true)]
- :active true))))
- (is (= 1 (count (mt/rows (mt/run-mbql-query venues {:limit 1})))))))
- (testing "With multi-level-schema on"
- (t2/update! :model/Database db-id {:details (assoc details
- :multi-level-schema true
- :schema-filters-patterns multi-pattern)})
- (mt/with-db
- (t2/select-one :model/Database db-id)
- (sync/sync-database! (mt/db))
- (is (= #{(format "%s.%s" (:catalog details) "test-data")
- (format "%s.%s" multicatalog multicatalog-schema)}
- (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)))
- ;; Adds four fields for metabase_ci_multicatalog.test_schema.test id,name,ci_venue_id,drivers_venue_id
- (is (= 56 (count (t2/select
- :model/Field
- :table_id [:in (t2/select-fn-set :id :model/Table :db_id (mt/id) :active true)]
- :active true))))
- (is (= 1 (count (mt/rows (mt/run-mbql-query venues {:limit 1})))))))
- (testing "With multi-level-schema off"
- (t2/update! :model/Database db-id {:details (assoc details :multi-level-schema false)})
- (mt/with-db
- (t2/select-one :model/Database db-id)
- (sync/sync-database! (mt/db))
- (is (= #{"test-data"} (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)))
- (is (= 52 (count (t2/select
- :model/Field
- :table_id [:in (t2/select-fn-set :id :model/Table :db_id (mt/id) :active true)]
- :active true))))
- (is (= 1 (count (mt/rows (mt/run-mbql-query venues {:limit 1}))))))))))))
+ (t2/select-one :model/Database db-id)
+ (sync/sync-database! (mt/db))
+ (is (= #{"test-data"} (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)))
+ (is (= 52 (count (t2/select
+ :model/Field
+ :table_id [:in (t2/select-fn-set :id :model/Table :db_id (mt/id) :active true)]
+ :active true))))
+ (is (= 1 (count (mt/rows (mt/run-mbql-query venues {:limit 1})))))))))))
(deftest multi-level-changes-inactive-table-schemas-too
- (mt/test-driver
- :databricks
- ;; skip if running in multi-level since we are manipulating it in this test
- (when-not (tx/db-test-env-var :databricks :multi-level-schema false)
- (let [details (get (mt/db) :details)
- ;; metabase_ci_multicatalog.test_schema.test (id, name)
- multicatalog (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")
- multicatalog-schema (tx/db-test-env-var :databricks :multicatalog-schema "test_schema")
- multi-pattern (format "%s.%s,%s.%s"
- (:catalog details)
- (:schema-filters-patterns details)
- multicatalog
- multicatalog-schema)]
- (mt/with-temp [:model/Database {db-id :id :as db} {:engine :databricks :details details}]
- ;; First sync with multi-level-schema off
+ (mt/test-driver :databricks
+ (let [details (get (mt/db) :details)
+ ;; metabase_ci_multicatalog.test_schema.test (id, name)
+ multicatalog (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")
+ multicatalog-schema (tx/db-test-env-var :databricks :multicatalog-schema "test_schema")
+ multi-pattern (format "%s.%s,%s.%s"
+ (:catalog details)
+ (:schema-filters-patterns details)
+ multicatalog
+ multicatalog-schema)]
+ (mt/with-temp [:model/Database {db-id :id :as db} {:engine :databricks :details details}]
+ ;; First sync with multi-level-schema off
+ (mt/with-db db
+ (testing "With multi-level-schema default (off)"
+ (sync/sync-database! (mt/db))
+ ;; originally we have unqualified schemas, only in one catalog
+ (is (= #{"test-data"}
+ (t2/select-fn-set :schema :model/Table :db_id (mt/id))))))
+ (testing "With multi-level-schema on, schemas are qualified"
+ (t2/update! :model/Database db-id {:details (assoc details
+ :multi-level-schema true
+ :schema-filters-patterns multi-pattern)})
+ ;; Deactivate its tables for testing
+ (t2/update! :model/Table {:db_id db-id} {:active false})
(mt/with-db
- db
- (testing "With multi-level-schema default (off)"
- (sync/sync-database! (mt/db))
- ;; originally we have unqualified schemas, only in one catalog
- (is (= #{"test-data"}
- (t2/select-fn-set :schema :model/Table :db_id (mt/id))))))
- (testing "With multi-level-schema on, schemas are qualified"
- (t2/update! :model/Database db-id {:details (assoc details
- :multi-level-schema true
- :schema-filters-patterns multi-pattern)})
- ;; Deactivate its tables for testing
- (t2/update! :model/Table {:db_id db-id} {:active false})
- (mt/with-db
- (t2/select-one :model/Database db-id)
- (sync/sync-database! (mt/db))
- (is (= #{(format "%s.%s" (:catalog details) "test-data")
- (format "%s.%s" multicatalog multicatalog-schema)}
- ;; active` *and* inactive tables both have their schemas changed.
- (t2/select-fn-set :schema :model/Table :db_id (mt/id)))))))))))
+ (t2/select-one :model/Database db-id)
+ (sync/sync-database! (mt/db))
+ (is (= #{(format "%s.%s" (:catalog details) "test-data")
+ (format "%s.%s" multicatalog multicatalog-schema)}
+ ;; active` *and* inactive tables both have their schemas changed.
+ (t2/select-fn-set :schema :model/Table :db_id (mt/id))))))))))
(deftest multi-level-schema-wanted-catalogs-test
- (mt/test-driver
- :databricks
- ;; skip if running in multi-level since we are manipulating it in this test
- (when-not (tx/db-test-env-var :databricks :multi-level-schema false)
- (let [details (get (mt/db) :details)
- ;; metabase_ci_multicatalog.test_schema.test (id, name)
- multicatalog (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")
- multicatalog-schema (tx/db-test-env-var :databricks :multicatalog-schema "test_schema")
- schema-filters (set [(format "%s.%s" (:catalog details) "test-data")
- (format "%s.%s" multicatalog multicatalog-schema)
- "system.query"])]
- (mt/with-temp [:model/Database {db-id :id :as _db} {:engine :databricks
- :details (-> details
- (assoc :multi-level-schema true
- :schema-filters-type "inclusion"
- :schema-filters-patterns (str/join ", " schema-filters)))}]
- (mt/with-db
- (t2/select-one :model/Database db-id)
- (sync/sync-database! (mt/db) {:scan :schema})
- (let [table-schemas (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)]
- (is (= schema-filters table-schemas))
- (is (nil? (some #(str/starts-with? % "__databricks") table-schemas))))))))))
+ (mt/test-driver :databricks
+ (let [details (get (mt/db) :details)
+ ;; metabase_ci_multicatalog.test_schema.test (id, name)
+ multicatalog (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")
+ multicatalog-schema (tx/db-test-env-var :databricks :multicatalog-schema "test_schema")
+ schema-filters (set [(format "%s.%s" (:catalog details) "test-data")
+ (format "%s.%s" multicatalog multicatalog-schema)
+ "system.query"])]
+ (mt/with-temp [:model/Database {db-id :id :as _db} {:engine :databricks
+ :details (-> details
+ (assoc :multi-level-schema true
+ :schema-filters-type "inclusion"
+ :schema-filters-patterns (str/join ", " schema-filters)))}]
+ (mt/with-db
+ (t2/select-one :model/Database db-id)
+ (sync/sync-database! (mt/db) {:scan :schema})
+ (let [table-schemas (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)]
+ (is (= schema-filters table-schemas))
+ (is (nil? (some #(str/starts-with? % "__databricks") table-schemas)))))))))
(deftest multi-catalog-joins
- (mt/test-driver
- :databricks
- ;; skip if running in multi-level since we are manipulating it in this test
- (when-not (tx/db-test-env-var :databricks :multi-level-schema false)
- (let [details (get (mt/db) :details)
- catalog+schema (format "%s.%s"
- (:catalog details)
- (:schema-filters-patterns details))
- multi-catalog+schema (format "%s.%s"
- (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")
- (tx/db-test-env-var :databricks :multicatalog-schema "test_schema"))
- ;; metabase_ci_multicatalog.test_schema.test (id, name,ci_venue_id,drivers_venue_id)
- multi-pattern (format "%s,%s" catalog+schema multi-catalog+schema)
- details (assoc details
- :multi-level-schema true
- :schema-filters-patterns multi-pattern)]
- (mt/with-temp [:model/Database {db-id :id :as db} {:engine :databricks :details details}]
- (mt/with-db db
- (sync/sync-database! (mt/db) {:scan :schema})
- (let [mp (lib-be/application-database-metadata-provider db-id)
- [t1-id t2-id] (t2/select-fn-vec
- :id
- :model/Table
- :db_id db-id
- [:composite :schema :name]
- [:in [[:composite catalog+schema "venues"]
- [:composite multi-catalog+schema "test"]]]
- {:order-by [:schema]})
- t1-id-field (m/find-first (comp #(= % "id") :name) (lib.metadata/fields mp t1-id))
- t2-id-field (m/find-first (comp #(= % "id") :name) (lib.metadata/fields mp t2-id))
- fk-query (-> (lib/query mp (lib.metadata/table mp t1-id))
- (lib/join (lib.metadata/table mp t2-id))
- (lib/filter (lib/= t2-id-field 1))
- (lib/limit 1))
- manual-query (-> (lib/query mp (lib.metadata/table mp t1-id))
- (lib/join (lib/join-clause (lib.metadata/table mp t2-id)
- [(lib/= t1-id-field (lib/+ t2-id-field 1))]))
- (lib/filter (lib/= t2-id-field 1))
- (lib/limit 1))]
- (is (= [[1 "Red Medicine" 4 10.0646 -165.374 3 1 "toucan" 1 1]]
- (mt/rows (qp/process-query fk-query))))
- (is (= [[2 "Stout Burgers & Beers" 11 34.0996 -118.329 2 1 "toucan" 1 1]]
- (mt/rows (qp/process-query manual-query)))))))))))
+ (mt/test-driver :databricks
+ (let [details (get (mt/db) :details)
+ catalog+schema (format "%s.%s"
+ (:catalog details)
+ (:schema-filters-patterns details))
+ multi-catalog+schema (format "%s.%s"
+ (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")
+ (tx/db-test-env-var :databricks :multicatalog-schema "test_schema"))
+ ;; metabase_ci_multicatalog.test_schema.test (id, name,ci_venue_id,drivers_venue_id)
+ multi-pattern (format "%s,%s" catalog+schema multi-catalog+schema)
+ details (assoc details
+ :multi-level-schema true
+ :schema-filters-patterns multi-pattern)]
+ (mt/with-temp [:model/Database {db-id :id :as db} {:engine :databricks :details details}]
+ (mt/with-db db
+ (sync/sync-database! (mt/db) {:scan :schema})
+ (let [mp (lib-be/application-database-metadata-provider db-id)
+ [t1-id t2-id] (t2/select-fn-vec
+ :id
+ :model/Table
+ :db_id db-id
+ [:composite :schema :name]
+ [:in [[:composite catalog+schema "venues"]
+ [:composite multi-catalog+schema "test"]]]
+ {:order-by [:schema]})
+ t1-id-field (m/find-first (comp #(= % "id") :name) (lib.metadata/fields mp t1-id))
+ t2-id-field (m/find-first (comp #(= % "id") :name) (lib.metadata/fields mp t2-id))
+ fk-query (-> (lib/query mp (lib.metadata/table mp t1-id))
+ (lib/join (lib.metadata/table mp t2-id))
+ (lib/filter (lib/= t2-id-field 1))
+ (lib/limit 1))
+ manual-query (-> (lib/query mp (lib.metadata/table mp t1-id))
+ (lib/join (lib/join-clause (lib.metadata/table mp t2-id)
+ [(lib/= t1-id-field (lib/+ t2-id-field 1))]))
+ (lib/filter (lib/= t2-id-field 1))
+ (lib/limit 1))]
+ (is (= [[1 "Red Medicine" 4 10.0646 -165.374 3 1 "toucan" 1 1]]
+ (mt/rows (qp/process-query fk-query))))
+ (is (= [[2 "Stout Burgers & Beers" 11 34.0996 -118.329 2 1 "toucan" 1 1]]
+ (mt/rows (qp/process-query manual-query))))))))))
(deftest ^:parallel array-test
(mt/test-driver :databricks
diff --git a/modules/drivers/databricks/test/metabase/test/data/databricks.clj b/modules/drivers/databricks/test/metabase/test/data/databricks.clj
index 6135eaa8dba..ea9b8da3be0 100644
--- a/modules/drivers/databricks/test/metabase/test/data/databricks.clj
+++ b/modules/drivers/databricks/test/metabase/test/data/databricks.clj
@@ -52,16 +52,12 @@
(defmethod tx/dbdef->connection-details :databricks
[_driver _connection-type {:keys [database-name] :as _dbdef}]
(let [catalog (tx/db-test-env-var-or-throw :databricks :catalog)
- multi-level? (tx/db-test-env-var :databricks :multi-level-schema)
;; Databricks' namespace model: catalog, schema, table. With current implementation user can add all schemas
;; in catalog or all catalogs on one Metabase database connection. Following expression generates schema
;; filters so only one schema is treated as a Metabase database, for compatibility with existing tests.
- schema-filters (when (or (string? (not-empty database-name))
- multi-level?)
+ schema-filters (when (string? (not-empty database-name))
{:schema-filters-type "inclusion"
- :schema-filters-patterns (str
- (when multi-level? (str catalog "."))
- (if database-name database-name "*"))})]
+ :schema-filters-patterns (if database-name database-name "*")})]
(merge
{:host (tx/db-test-env-var-or-throw :databricks :host)
:token (tx/db-test-env-var-or-throw :databricks :token)
@@ -72,8 +68,7 @@
;; keep going for 120 seconds (basically 120 retries since each takes one
;; second) essentially DOSing the database.
;; https://github.com/databricks/databricks-jdbc/blob/v3.4.2/src/main/java/com/databricks/jdbc/dbclient/impl/http/DatabricksHttpRetryHandler.java#L187
- :rate-limit-retry 0
- :multi-level-schema multi-level?}
+ :rate-limit-retry 0}
schema-filters)))
(defn- existing-databases