The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Remove be-tests-databricks-multi-catalog-ee job. (#81127)
technomancy9d216767d063
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex 239bd76fc3f..b7c783b8d33 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -338,15 +338,9 @@ jobs::fail-fast? true:partition/total 2:partition/index 1- - name: Driver Tests (Multi-Catalog)- multi-level-schema: true- test-args: >-- :only metabase.driver.databricks-test- :fail-fast? trueenv:CI: 'true'DRIVERS: databricks- MB_DATABRICKS_TEST_MULTI_LEVEL_SCHEMA: ${{ matrix.job.multi-level-schema }}MB_DATABRICKS_TEST_CATALOG: 'metabase_ci'MB_DATABRICKS_TEST_CATALOG_ROUTING: 'metabase_routing_catalog'MB_DATABRICKS_TEST_CLIENT_ID: ${{ secrets.MB_DATABRICKS_JDBC_TEST_CLIENT_ID }}diff --git a/modules/drivers/databricks/test/metabase/driver/databricks_test.clj b/modules/drivers/databricks/test/metabase/driver/databricks_test.cljindex 697ba4d614f..0cbd24f5441 100644--- a/modules/drivers/databricks/test/metabase/driver/databricks_test.clj+++ b/modules/drivers/databricks/test/metabase/driver/databricks_test.clj@@ -22,158 +22,175 @@[metabase.util :as u][toucan2.core :as t2]))-(defn- maybe-qualify-schema- [schema]- (let [multi-level? (tx/db-test-env-var :databricks :multi-level-schema false)- catalog (get-in (mt/db) [:details :catalog])]- (cond->> schema multi-level? (str catalog "."))))+(def ^:dynamic *multi-level?* false)++;; certain tests need to be run twice; once with the multi-level schema and once without.+(defmacro with-and-without-multi-level [& body]+ ;; run it once without multi-level and once with multi-level+ `(do ~@body+ (binding [*multi-level?* true]+ (testing "with multi-level schema"+ ~@body))))++(defn- maybe-qualify-schema [schema]+ (if *multi-level?*+ (str (get-in (mt/db) [:details :catalog]) "." schema)+ schema))++(defn db []+ (if *multi-level?*+ (-> (mt/db)+ (update-in [:details :schema-filters-patterns]+ #(str "metabase_ci." %))+ (assoc-in [:details :multi-level-schema] true))+ (mt/db)));; Because the datasets that are tested are preloaded, it is fine just to modify the database details to sync other schemas.(deftest ^:parallel sync-test- (mt/test-driver- :databricks- (testing "`driver/describe-database` implementation returns expected results for inclusion of test-data schema."- (is (= #{{:name "venues", :schema (maybe-qualify-schema "test-data"), :description nil}- {:name "checkins", :schema (maybe-qualify-schema "test-data"), :description nil}- {:name "users", :schema (maybe-qualify-schema "test-data"), :description nil}- {:name "people", :schema (maybe-qualify-schema "test-data"), :description nil}- {:name "categories", :schema (maybe-qualify-schema "test-data"), :description nil}- {:name "reviews", :schema (maybe-qualify-schema "test-data"), :description nil}- {:name "orders", :schema (maybe-qualify-schema "test-data"), :description nil}- {:name "products", :schema (maybe-qualify-schema "test-data"), :description nil}}- (into #{} (:tables (driver/describe-database :databricks (mt/db)))))))- (testing "`driver/describe-database` returns expected results for `all` schema filters."- (let [actual-tables (update (driver/describe-database :databricks (-> (mt/db)- (update :details dissoc :schema-filters-patterns)- (update :details assoc :schema-filters-type "all")))- ;; `:tables` is a reducible (streamed); realize it for the set ops below- :tables #(into #{} %))]- (testing "tables from multiple schemas were found"- (are [name schema] (contains? (:tables actual-tables)- {:name name, :schema schema, :description nil})- "venues" (maybe-qualify-schema "test-data")- "checkins" (maybe-qualify-schema "test-data")- "airport" (maybe-qualify-schema "airports")- "bird" (maybe-qualify-schema "bird-flocks")))- (testing "information_schema is excluded"- (is (empty? (filter #(str/includes? "information_schema" (:schema %)) (:tables actual-tables)))))))- (testing "`driver/describe-database` returns expected results for `exclusion` schema filters."- (let [actual-tables (update (driver/describe-database :databricks (update (mt/db) :details assoc- :schema-filters-patterns (maybe-qualify-schema "test-data")- :schema-filters-type "exclusion"))- ;; `:tables` is a reducible (streamed); realize it for the set ops below- :tables #(into #{} %))]- (testing "tables from multiple schemas were found"- (is (not (contains? (set (map :schema (:tables actual-tables))) (maybe-qualify-schema "test-data"))))- (is (contains? (:tables actual-tables) {:name "airport", :schema (maybe-qualify-schema "airports"), :description nil}))- (is (contains? (:tables actual-tables) {:name "bird", :schema (maybe-qualify-schema "bird-flocks"), :description nil})))))))+ (with-and-without-multi-level+ (mt/test-driver :databricks+ (testing "`driver/describe-database` implementation returns expected results for inclusion of test-data schema."+ (is (= #{{:name "venues", :schema (maybe-qualify-schema "test-data"), :description nil}+ {:name "checkins", :schema (maybe-qualify-schema "test-data"), :description nil}+ {:name "users", :schema (maybe-qualify-schema "test-data"), :description nil}+ {:name "people", :schema (maybe-qualify-schema "test-data"), :description nil}+ {:name "categories", :schema (maybe-qualify-schema "test-data"), :description nil}+ {:name "reviews", :schema (maybe-qualify-schema "test-data"), :description nil}+ {:name "orders", :schema (maybe-qualify-schema "test-data"), :description nil}+ {:name "products", :schema (maybe-qualify-schema "test-data"), :description nil}}+ (into #{} (:tables (driver/describe-database :databricks (db)))))))+ (testing "`driver/describe-database` returns expected results for `all` schema filters."+ (let [actual-tables (update (driver/describe-database :databricks (-> (db)+ (update :details dissoc :schema-filters-patterns)+ (update :details assoc :schema-filters-type "all")))+ ;; `:tables` is a reducible (streamed); realize it for the set ops below+ :tables #(into #{} %))]+ (testing "tables from multiple schemas were found"+ (are [name schema] (contains? (:tables actual-tables)+ {:name name, :schema schema, :description nil})+ "venues" (maybe-qualify-schema "test-data")+ "checkins" (maybe-qualify-schema "test-data")+ "airport" (maybe-qualify-schema "airports")+ "bird" (maybe-qualify-schema "bird-flocks")))+ (testing "information_schema is excluded"+ (is (empty? (filter #(str/includes? "information_schema" (:schema %)) (:tables actual-tables)))))))+ (testing "`driver/describe-database` returns expected results for `exclusion` schema filters."+ (let [actual-tables (update (driver/describe-database :databricks (update (db) :details assoc+ :schema-filters-patterns (maybe-qualify-schema "test-data")+ :schema-filters-type "exclusion"))+ ;; `:tables` is a reducible (streamed); realize it for the set ops below+ :tables #(into #{} %))]+ (testing "tables from multiple schemas were found"+ (is (not (contains? (set (map :schema (:tables actual-tables))) (maybe-qualify-schema "test-data"))))+ (is (contains? (:tables actual-tables) {:name "airport", :schema (maybe-qualify-schema "airports"), :description nil}))+ (is (contains? (:tables actual-tables) {:name "bird", :schema (maybe-qualify-schema "bird-flocks"), :description nil}))))))))(deftest ^:parallel describe-fields-test- (mt/test-driver- :databricks- (let [fields (vec (driver/describe-fields :databricks (mt/db) {:schema-names [(maybe-qualify-schema "test-data")]- :table-names ["orders"]}))]- (testing "Underlying query returns only fields from selected catalog"- (is (= 9 (count fields))))- (testing "Expected fields are returned"- (is (= (into #{}- (map #(update % :table-schema maybe-qualify-schema))- #{{:table-schema "test-data"- :table-name "orders"- :pk? true- :name "id"- :database-type "int"- :database-position 0- :base-type :type/Integer- :json-unfolding false}- {:table-schema "test-data"- :table-name "orders"- :pk? false- :name "user_id"- :database-type "int"- :database-position 1- :base-type :type/Integer- :json-unfolding false}- {:table-schema "test-data"- :table-name "orders"- :pk? false- :name "product_id"- :database-type "int"- :database-position 2- :base-type :type/Integer- :json-unfolding false}- {:table-schema "test-data"- :table-name "orders"- :pk? false- :name "subtotal"- :database-type "double"- :database-position 3- :base-type :type/Float- :json-unfolding false}- {:table-schema "test-data"- :table-name "orders"- :pk? false- :name "tax"- :database-type "double"- :database-position 4- :base-type :type/Float- :json-unfolding false}- {:table-schema "test-data"- :table-name "orders"- :pk? false- :name "total"- :database-type "double"- :database-position 5- :base-type :type/Float- :json-unfolding false}- {:table-schema "test-data"- :table-name "orders"- :pk? false- :name "discount"- :database-type "double"- :database-position 6- :base-type :type/Float- :json-unfolding false}- {:table-schema "test-data"- :table-name "orders"- :pk? false- :name "created_at"- :database-type "timestamp"- :database-position 7- :base-type :type/DateTimeWithLocalTZ- :json-unfolding false}- {:table-schema "test-data"- :table-name "orders"- :pk? false- :name "quantity"- :database-type "int"- :database-position 8- :base-type :type/Integer- :json-unfolding false}})- (set fields)))))))+ (with-and-without-multi-level+ (mt/test-driver :databricks+ (let [fields (vec (driver/describe-fields :databricks (db) {:schema-names [(maybe-qualify-schema "test-data")]+ :table-names ["orders"]}))]+ (testing "Underlying query returns only fields from selected catalog"+ (is (= 9 (count fields))))+ (testing "Expected fields are returned"+ (is (= (into #{}+ (map #(update % :table-schema maybe-qualify-schema))+ #{{:table-schema "test-data"+ :table-name "orders"+ :pk? true+ :name "id"+ :database-type "int"+ :database-position 0+ :base-type :type/Integer+ :json-unfolding false}+ {:table-schema "test-data"+ :table-name "orders"+ :pk? false+ :name "user_id"+ :database-type "int"+ :database-position 1+ :base-type :type/Integer+ :json-unfolding false}+ {:table-schema "test-data"+ :table-name "orders"+ :pk? false+ :name "product_id"+ :database-type "int"+ :database-position 2+ :base-type :type/Integer+ :json-unfolding false}+ {:table-schema "test-data"+ :table-name "orders"+ :pk? false+ :name "subtotal"+ :database-type "double"+ :database-position 3+ :base-type :type/Float+ :json-unfolding false}+ {:table-schema "test-data"+ :table-name "orders"+ :pk? false+ :name "tax"+ :database-type "double"+ :database-position 4+ :base-type :type/Float+ :json-unfolding false}+ {:table-schema "test-data"+ :table-name "orders"+ :pk? false+ :name "total"+ :database-type "double"+ :database-position 5+ :base-type :type/Float+ :json-unfolding false}+ {:table-schema "test-data"+ :table-name "orders"+ :pk? false+ :name "discount"+ :database-type "double"+ :database-position 6+ :base-type :type/Float+ :json-unfolding false}+ {:table-schema "test-data"+ :table-name "orders"+ :pk? false+ :name "created_at"+ :database-type "timestamp"+ :database-position 7+ :base-type :type/DateTimeWithLocalTZ+ :json-unfolding false}+ {:table-schema "test-data"+ :table-name "orders"+ :pk? false+ :name "quantity"+ :database-type "int"+ :database-position 8+ :base-type :type/Integer+ :json-unfolding false}})+ (set fields))))))))(deftest ^:parallel describe-fks-test- (mt/test-driver :databricks- (let [fks (vec (driver/describe-fks :databricks- (lib.metadata/database (mt/metadata-provider))- {:schema-names [(maybe-qualify-schema "test-data")]- :table-names ["orders"]}))]- (testing "Only fks from current catalog are registered"- (is (= 2 (count fks))))- (testing "Expected fks are returned"- (is (= #{{:fk-table-schema (maybe-qualify-schema "test-data")- :fk-table-name "orders"- :fk-column-name "product_id"- :pk-table-schema (maybe-qualify-schema "test-data")- :pk-table-name "products"- :pk-column-name "id"}- {:fk-table-schema (maybe-qualify-schema "test-data")- :fk-table-name "orders"- :fk-column-name "user_id"- :pk-table-schema (maybe-qualify-schema "test-data")- :pk-table-name "people"- :pk-column-name "id"}}- (set fks)))))))+ (with-and-without-multi-level+ (mt/test-driver :databricks+ (let [fks (driver/describe-fks :databricks+ (assoc (lib.metadata/database (mt/metadata-provider))+ :details (:details (db)))+ {:schema-names [(maybe-qualify-schema "test-data")]+ :table-names ["orders"]})]+ (testing "Expected fks are returned"+ (is (= #{{:fk-table-schema (maybe-qualify-schema "test-data")+ :fk-table-name "orders"+ :fk-column-name "product_id"+ :pk-table-schema (maybe-qualify-schema "test-data")+ :pk-table-name "products"+ :pk-column-name "id"}+ {:fk-table-schema (maybe-qualify-schema "test-data")+ :fk-table-name "orders"+ :fk-column-name "user_id"+ :pk-table-schema (maybe-qualify-schema "test-data")+ :pk-table-name "people"+ :pk-column-name "id"}}+ (set fks))))))))(deftest ^:parallel multi-level-schema-filter-sql-test(testing "multi-catalog sync filters on catalog and schema with equalities, not a row-constructor IN (GHY-4263)"@@ -345,163 +362,149 @@:subname))))))(deftest multi-level-schema-test- (mt/test-driver- :databricks- ;; skip if running in multi-level since we are manipulating it in this test- (when-not (tx/db-test-env-var :databricks :multi-level-schema false)- (let [details (get (mt/db) :details)- ;; metabase_ci_multicatalog.test_schema.test (id, name)- multicatalog (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")- multicatalog-schema (tx/db-test-env-var :databricks :multicatalog-schema "test_schema")- multi-pattern (format "%s.%s,%s.%s"- (:catalog details)- (:schema-filters-patterns details)- multicatalog- multicatalog-schema)]- (mt/with-temp [:model/Database {db-id :id :as db} {:engine :databricks :details details}]+ (mt/test-driver :databricks+ (let [details (get (mt/db) :details)+ ;; metabase_ci_multicatalog.test_schema.test (id, name)+ multicatalog (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")+ multicatalog-schema (tx/db-test-env-var :databricks :multicatalog-schema "test_schema")+ multi-pattern (format "%s.%s,%s.%s"+ (:catalog details)+ (:schema-filters-patterns details)+ multicatalog+ multicatalog-schema)]+ (mt/with-temp [:model/Database {db-id :id :as db} {:engine :databricks :details details}]+ (mt/with-db db+ (testing "With multi-level-schema default (off)"+ (sync/sync-database! (mt/db))+ (is (= #{"test-data"} (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)))+ (is (= 52 (count (t2/select+ :model/Field+ :table_id [:in (t2/select-fn-set :id :model/Table :db_id (mt/id) :active true)]+ :active true))))+ (is (= 1 (count (mt/rows (mt/run-mbql-query venues {:limit 1})))))))+ (testing "With multi-level-schema on"+ (t2/update! :model/Database db-id {:details (assoc details+ :multi-level-schema true+ :schema-filters-patterns multi-pattern)})+ (mt/with-db+ (t2/select-one :model/Database db-id)+ (sync/sync-database! (mt/db))+ (is (= #{(format "%s.%s" (:catalog details) "test-data")+ (format "%s.%s" multicatalog multicatalog-schema)}+ (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)))+ ;; Adds four fields for metabase_ci_multicatalog.test_schema.test id,name,ci_venue_id,drivers_venue_id+ (is (= 56 (count (t2/select+ :model/Field+ :table_id [:in (t2/select-fn-set :id :model/Table :db_id (mt/id) :active true)]+ :active true))))+ (is (= 1 (count (mt/rows (mt/run-mbql-query venues {:limit 1})))))))+ (testing "With multi-level-schema off"+ (t2/update! :model/Database db-id {:details (assoc details :multi-level-schema false)})(mt/with-db- db- (testing "With multi-level-schema default (off)"- (sync/sync-database! (mt/db))- (is (= #{"test-data"} (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)))- (is (= 52 (count (t2/select- :model/Field- :table_id [:in (t2/select-fn-set :id :model/Table :db_id (mt/id) :active true)]- :active true))))- (is (= 1 (count (mt/rows (mt/run-mbql-query venues {:limit 1})))))))- (testing "With multi-level-schema on"- (t2/update! :model/Database db-id {:details (assoc details- :multi-level-schema true- :schema-filters-patterns multi-pattern)})- (mt/with-db- (t2/select-one :model/Database db-id)- (sync/sync-database! (mt/db))- (is (= #{(format "%s.%s" (:catalog details) "test-data")- (format "%s.%s" multicatalog multicatalog-schema)}- (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)))- ;; Adds four fields for metabase_ci_multicatalog.test_schema.test id,name,ci_venue_id,drivers_venue_id- (is (= 56 (count (t2/select- :model/Field- :table_id [:in (t2/select-fn-set :id :model/Table :db_id (mt/id) :active true)]- :active true))))- (is (= 1 (count (mt/rows (mt/run-mbql-query venues {:limit 1})))))))- (testing "With multi-level-schema off"- (t2/update! :model/Database db-id {:details (assoc details :multi-level-schema false)})- (mt/with-db- (t2/select-one :model/Database db-id)- (sync/sync-database! (mt/db))- (is (= #{"test-data"} (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)))- (is (= 52 (count (t2/select- :model/Field- :table_id [:in (t2/select-fn-set :id :model/Table :db_id (mt/id) :active true)]- :active true))))- (is (= 1 (count (mt/rows (mt/run-mbql-query venues {:limit 1}))))))))))))+ (t2/select-one :model/Database db-id)+ (sync/sync-database! (mt/db))+ (is (= #{"test-data"} (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)))+ (is (= 52 (count (t2/select+ :model/Field+ :table_id [:in (t2/select-fn-set :id :model/Table :db_id (mt/id) :active true)]+ :active true))))+ (is (= 1 (count (mt/rows (mt/run-mbql-query venues {:limit 1})))))))))))(deftest multi-level-changes-inactive-table-schemas-too- (mt/test-driver- :databricks- ;; skip if running in multi-level since we are manipulating it in this test- (when-not (tx/db-test-env-var :databricks :multi-level-schema false)- (let [details (get (mt/db) :details)- ;; metabase_ci_multicatalog.test_schema.test (id, name)- multicatalog (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")- multicatalog-schema (tx/db-test-env-var :databricks :multicatalog-schema "test_schema")- multi-pattern (format "%s.%s,%s.%s"- (:catalog details)- (:schema-filters-patterns details)- multicatalog- multicatalog-schema)]- (mt/with-temp [:model/Database {db-id :id :as db} {:engine :databricks :details details}]- ;; First sync with multi-level-schema off+ (mt/test-driver :databricks+ (let [details (get (mt/db) :details)+ ;; metabase_ci_multicatalog.test_schema.test (id, name)+ multicatalog (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")+ multicatalog-schema (tx/db-test-env-var :databricks :multicatalog-schema "test_schema")+ multi-pattern (format "%s.%s,%s.%s"+ (:catalog details)+ (:schema-filters-patterns details)+ multicatalog+ multicatalog-schema)]+ (mt/with-temp [:model/Database {db-id :id :as db} {:engine :databricks :details details}]+ ;; First sync with multi-level-schema off+ (mt/with-db db+ (testing "With multi-level-schema default (off)"+ (sync/sync-database! (mt/db))+ ;; originally we have unqualified schemas, only in one catalog+ (is (= #{"test-data"}+ (t2/select-fn-set :schema :model/Table :db_id (mt/id))))))+ (testing "With multi-level-schema on, schemas are qualified"+ (t2/update! :model/Database db-id {:details (assoc details+ :multi-level-schema true+ :schema-filters-patterns multi-pattern)})+ ;; Deactivate its tables for testing+ (t2/update! :model/Table {:db_id db-id} {:active false})(mt/with-db- db- (testing "With multi-level-schema default (off)"- (sync/sync-database! (mt/db))- ;; originally we have unqualified schemas, only in one catalog- (is (= #{"test-data"}- (t2/select-fn-set :schema :model/Table :db_id (mt/id))))))- (testing "With multi-level-schema on, schemas are qualified"- (t2/update! :model/Database db-id {:details (assoc details- :multi-level-schema true- :schema-filters-patterns multi-pattern)})- ;; Deactivate its tables for testing- (t2/update! :model/Table {:db_id db-id} {:active false})- (mt/with-db- (t2/select-one :model/Database db-id)- (sync/sync-database! (mt/db))- (is (= #{(format "%s.%s" (:catalog details) "test-data")- (format "%s.%s" multicatalog multicatalog-schema)}- ;; active` *and* inactive tables both have their schemas changed.- (t2/select-fn-set :schema :model/Table :db_id (mt/id)))))))))))+ (t2/select-one :model/Database db-id)+ (sync/sync-database! (mt/db))+ (is (= #{(format "%s.%s" (:catalog details) "test-data")+ (format "%s.%s" multicatalog multicatalog-schema)}+ ;; active` *and* inactive tables both have their schemas changed.+ (t2/select-fn-set :schema :model/Table :db_id (mt/id))))))))))(deftest multi-level-schema-wanted-catalogs-test- (mt/test-driver- :databricks- ;; skip if running in multi-level since we are manipulating it in this test- (when-not (tx/db-test-env-var :databricks :multi-level-schema false)- (let [details (get (mt/db) :details)- ;; metabase_ci_multicatalog.test_schema.test (id, name)- multicatalog (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")- multicatalog-schema (tx/db-test-env-var :databricks :multicatalog-schema "test_schema")- schema-filters (set [(format "%s.%s" (:catalog details) "test-data")- (format "%s.%s" multicatalog multicatalog-schema)- "system.query"])]- (mt/with-temp [:model/Database {db-id :id :as _db} {:engine :databricks- :details (-> details- (assoc :multi-level-schema true- :schema-filters-type "inclusion"- :schema-filters-patterns (str/join ", " schema-filters)))}]- (mt/with-db- (t2/select-one :model/Database db-id)- (sync/sync-database! (mt/db) {:scan :schema})- (let [table-schemas (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)]- (is (= schema-filters table-schemas))- (is (nil? (some #(str/starts-with? % "__databricks") table-schemas))))))))))+ (mt/test-driver :databricks+ (let [details (get (mt/db) :details)+ ;; metabase_ci_multicatalog.test_schema.test (id, name)+ multicatalog (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")+ multicatalog-schema (tx/db-test-env-var :databricks :multicatalog-schema "test_schema")+ schema-filters (set [(format "%s.%s" (:catalog details) "test-data")+ (format "%s.%s" multicatalog multicatalog-schema)+ "system.query"])]+ (mt/with-temp [:model/Database {db-id :id :as _db} {:engine :databricks+ :details (-> details+ (assoc :multi-level-schema true+ :schema-filters-type "inclusion"+ :schema-filters-patterns (str/join ", " schema-filters)))}]+ (mt/with-db+ (t2/select-one :model/Database db-id)+ (sync/sync-database! (mt/db) {:scan :schema})+ (let [table-schemas (t2/select-fn-set :schema :model/Table :db_id (mt/id) :active true)]+ (is (= schema-filters table-schemas))+ (is (nil? (some #(str/starts-with? % "__databricks") table-schemas)))))))))(deftest multi-catalog-joins- (mt/test-driver- :databricks- ;; skip if running in multi-level since we are manipulating it in this test- (when-not (tx/db-test-env-var :databricks :multi-level-schema false)- (let [details (get (mt/db) :details)- catalog+schema (format "%s.%s"- (:catalog details)- (:schema-filters-patterns details))- multi-catalog+schema (format "%s.%s"- (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")- (tx/db-test-env-var :databricks :multicatalog-schema "test_schema"))- ;; metabase_ci_multicatalog.test_schema.test (id, name,ci_venue_id,drivers_venue_id)- multi-pattern (format "%s,%s" catalog+schema multi-catalog+schema)- details (assoc details- :multi-level-schema true- :schema-filters-patterns multi-pattern)]- (mt/with-temp [:model/Database {db-id :id :as db} {:engine :databricks :details details}]- (mt/with-db db- (sync/sync-database! (mt/db) {:scan :schema})- (let [mp (lib-be/application-database-metadata-provider db-id)- [t1-id t2-id] (t2/select-fn-vec- :id- :model/Table- :db_id db-id- [:composite :schema :name]- [:in [[:composite catalog+schema "venues"]- [:composite multi-catalog+schema "test"]]]- {:order-by [:schema]})- t1-id-field (m/find-first (comp #(= % "id") :name) (lib.metadata/fields mp t1-id))- t2-id-field (m/find-first (comp #(= % "id") :name) (lib.metadata/fields mp t2-id))- fk-query (-> (lib/query mp (lib.metadata/table mp t1-id))- (lib/join (lib.metadata/table mp t2-id))- (lib/filter (lib/= t2-id-field 1))- (lib/limit 1))- manual-query (-> (lib/query mp (lib.metadata/table mp t1-id))- (lib/join (lib/join-clause (lib.metadata/table mp t2-id)- [(lib/= t1-id-field (lib/+ t2-id-field 1))]))- (lib/filter (lib/= t2-id-field 1))- (lib/limit 1))]- (is (= [[1 "Red Medicine" 4 10.0646 -165.374 3 1 "toucan" 1 1]]- (mt/rows (qp/process-query fk-query))))- (is (= [[2 "Stout Burgers & Beers" 11 34.0996 -118.329 2 1 "toucan" 1 1]]- (mt/rows (qp/process-query manual-query)))))))))))+ (mt/test-driver :databricks+ (let [details (get (mt/db) :details)+ catalog+schema (format "%s.%s"+ (:catalog details)+ (:schema-filters-patterns details))+ multi-catalog+schema (format "%s.%s"+ (tx/db-test-env-var :databricks :multicatalog-catalog "metabase_ci_multicatalog")+ (tx/db-test-env-var :databricks :multicatalog-schema "test_schema"))+ ;; metabase_ci_multicatalog.test_schema.test (id, name,ci_venue_id,drivers_venue_id)+ multi-pattern (format "%s,%s" catalog+schema multi-catalog+schema)+ details (assoc details+ :multi-level-schema true+ :schema-filters-patterns multi-pattern)]+ (mt/with-temp [:model/Database {db-id :id :as db} {:engine :databricks :details details}]+ (mt/with-db db+ (sync/sync-database! (mt/db) {:scan :schema})+ (let [mp (lib-be/application-database-metadata-provider db-id)+ [t1-id t2-id] (t2/select-fn-vec+ :id+ :model/Table+ :db_id db-id+ [:composite :schema :name]+ [:in [[:composite catalog+schema "venues"]+ [:composite multi-catalog+schema "test"]]]+ {:order-by [:schema]})+ t1-id-field (m/find-first (comp #(= % "id") :name) (lib.metadata/fields mp t1-id))+ t2-id-field (m/find-first (comp #(= % "id") :name) (lib.metadata/fields mp t2-id))+ fk-query (-> (lib/query mp (lib.metadata/table mp t1-id))+ (lib/join (lib.metadata/table mp t2-id))+ (lib/filter (lib/= t2-id-field 1))+ (lib/limit 1))+ manual-query (-> (lib/query mp (lib.metadata/table mp t1-id))+ (lib/join (lib/join-clause (lib.metadata/table mp t2-id)+ [(lib/= t1-id-field (lib/+ t2-id-field 1))]))+ (lib/filter (lib/= t2-id-field 1))+ (lib/limit 1))]+ (is (= [[1 "Red Medicine" 4 10.0646 -165.374 3 1 "toucan" 1 1]]+ (mt/rows (qp/process-query fk-query))))+ (is (= [[2 "Stout Burgers & Beers" 11 34.0996 -118.329 2 1 "toucan" 1 1]]+ (mt/rows (qp/process-query manual-query))))))))))(deftest ^:parallel array-test(mt/test-driver :databricksdiff --git a/modules/drivers/databricks/test/metabase/test/data/databricks.clj b/modules/drivers/databricks/test/metabase/test/data/databricks.cljindex 6135eaa8dba..ea9b8da3be0 100644--- a/modules/drivers/databricks/test/metabase/test/data/databricks.clj+++ b/modules/drivers/databricks/test/metabase/test/data/databricks.clj@@ -52,16 +52,12 @@(defmethod tx/dbdef->connection-details :databricks[_driver _connection-type {:keys [database-name] :as _dbdef}](let [catalog (tx/db-test-env-var-or-throw :databricks :catalog)- multi-level? (tx/db-test-env-var :databricks :multi-level-schema);; Databricks' namespace model: catalog, schema, table. With current implementation user can add all schemas;; in catalog or all catalogs on one Metabase database connection. Following expression generates schema;; filters so only one schema is treated as a Metabase database, for compatibility with existing tests.- schema-filters (when (or (string? (not-empty database-name))- multi-level?)+ schema-filters (when (string? (not-empty database-name)){:schema-filters-type "inclusion"- :schema-filters-patterns (str- (when multi-level? (str catalog "."))- (if database-name database-name "*"))})]+ :schema-filters-patterns (if database-name database-name "*")})](merge{:host (tx/db-test-env-var-or-throw :databricks :host):token (tx/db-test-env-var-or-throw :databricks :token)@@ -72,8 +68,7 @@;; keep going for 120 seconds (basically 120 retries since each takes one;; second) essentially DOSing the database.;; https://github.com/databricks/databricks-jdbc/blob/v3.4.2/src/main/java/com/databricks/jdbc/dbclient/impl/http/DatabricksHttpRetryHandler.java#L187- :rate-limit-retry 0- :multi-level-schema multi-level?}+ :rate-limit-retry 0}schema-filters)))(defn- existing-databases