The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Avoid row-constructor IN in multi-catalog Databricks sync (GHY-4263) (#79533)
Eric Normand62f8cd51eec1
diff --git a/modules/drivers/databricks/src/metabase/driver/databricks.clj b/modules/drivers/databricks/src/metabase/driver/databricks.cljindex 67a937f69f8..df5cd8603e7 100644--- a/modules/drivers/databricks/src/metabase/driver/databricks.clj+++ b/modules/drivers/databricks/src/metabase/driver/databricks.clj@@ -153,11 +153,12 @@(defn- schema-names-filter [schema-names multi-level-schema catalog-column schema-column](when schema-names(if multi-level-schema- [:in [:composite catalog-column schema-column]- (map (comp (fn [catalog+schema]- (into [:composite] catalog+schema))- split-catalog+schema)- schema-names)]+ ;; Use OR / AND because they are faster than IN for tuples on Databricks+ (into [:or]+ (map (fn [schema-name]+ (let [[catalog schema] (split-catalog+schema schema-name)]+ [:and [:= catalog-column catalog] [:= schema-column schema]])))+ schema-names)[:in schema-column schema-names])))(defmethod sql-jdbc.sync/describe-fields-sql :databricksdiff --git a/modules/drivers/databricks/test/metabase/driver/databricks_test.clj b/modules/drivers/databricks/test/metabase/driver/databricks_test.cljindex f1455e53ac3..697ba4d614f 100644--- a/modules/drivers/databricks/test/metabase/driver/databricks_test.clj+++ b/modules/drivers/databricks/test/metabase/driver/databricks_test.clj@@ -11,6 +11,7 @@[metabase.driver :as driver][metabase.driver.databricks :as databricks][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]+ [metabase.driver.sql-jdbc.sync :as sql-jdbc.sync][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -174,6 +175,41 @@:pk-column-name "id"}}(set fks)))))))+(deftest ^:parallel multi-level-schema-filter-sql-test+ (testing "multi-catalog sync filters on catalog and schema with equalities, not a row-constructor IN (GHY-4263)"+ (testing "Databricks' planner is ~200x slower on `(catalog, schema) IN ((?, ?))` than on AND-ed equalities"+ (doseq [[label sql+params] {"describe-fields"+ (sql-jdbc.sync/describe-fields-sql+ :databricks+ :schema-names ["mycatalog.myschema"]+ :details {:catalog "mycatalog" :multi-level-schema true})++ "describe-fks"+ (sql-jdbc.sync/describe-fks-sql+ :databricks+ :schema-names ["mycatalog.myschema"]+ :details {:catalog "mycatalog" :multi-level-schema true})}]+ (testing label+ (let [[sql & params] sql+params]+ (is (not (str/includes? sql "IN ((?, ?)")))+ (is (str/includes? sql "`table_catalog` = ?"))+ (is (str/includes? sql "`table_schema` = ?"))+ (is (= ["mycatalog" "myschema"] (vec (take-last 2 params)))))))))+ (testing "several catalog.schema pairs are OR-ed pairwise"+ (let [[sql & params] (sql-jdbc.sync/describe-fields-sql+ :databricks+ :schema-names ["c1.s1" "c2.s2"]+ :details {:catalog "c1" :multi-level-schema true})]+ (is (not (str/includes? sql "IN ((?, ?)")))+ (is (= ["c1" "s1" "c2" "s2"] params))))+ (testing "single-catalog connections keep the plain schema IN"+ (let [[sql & params] (sql-jdbc.sync/describe-fields-sql+ :databricks+ :schema-names ["s1" "s2"]+ :details {:catalog "c1" :multi-level-schema false})]+ (is (str/includes? sql "`table_schema` IN (?, ?)"))+ (is (= ["c1" "s1" "s2"] params)))))+(mt/defdataset dataset-with-ntz[["table_with_ntz" [{:field-name "timestamp":base-type {:native "timestamp_ntz"}}]