Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Avoid row-constructor IN in multi-catalog Databricks sync (GHY-4263) (#79533)

Eric Normand62f8cd51eec1
diff --git a/modules/drivers/databricks/src/metabase/driver/databricks.clj b/modules/drivers/databricks/src/metabase/driver/databricks.clj
index 67a937f69f8..df5cd8603e7 100644
--- a/modules/drivers/databricks/src/metabase/driver/databricks.clj
+++ b/modules/drivers/databricks/src/metabase/driver/databricks.clj
@@ -153,11 +153,12 @@
(defn- schema-names-filter [schema-names multi-level-schema catalog-column schema-column]
(when schema-names
(if multi-level-schema
- [:in [:composite catalog-column schema-column]
- (map (comp (fn [catalog+schema]
- (into [:composite] catalog+schema))
- split-catalog+schema)
- schema-names)]
+ ;; Use OR / AND because they are faster than IN for tuples on Databricks
+ (into [:or]
+ (map (fn [schema-name]
+ (let [[catalog schema] (split-catalog+schema schema-name)]
+ [:and [:= catalog-column catalog] [:= schema-column schema]])))
+ schema-names)
[:in schema-column schema-names])))
(defmethod sql-jdbc.sync/describe-fields-sql :databricks
diff --git a/modules/drivers/databricks/test/metabase/driver/databricks_test.clj b/modules/drivers/databricks/test/metabase/driver/databricks_test.clj
index f1455e53ac3..697ba4d614f 100644
--- a/modules/drivers/databricks/test/metabase/driver/databricks_test.clj
+++ b/modules/drivers/databricks/test/metabase/driver/databricks_test.clj
@@ -11,6 +11,7 @@
[metabase.driver :as driver]
[metabase.driver.databricks :as databricks]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+ [metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
@@ -174,6 +175,41 @@
:pk-column-name "id"}}
(set fks)))))))
+(deftest ^:parallel multi-level-schema-filter-sql-test
+ (testing "multi-catalog sync filters on catalog and schema with equalities, not a row-constructor IN (GHY-4263)"
+ (testing "Databricks' planner is ~200x slower on `(catalog, schema) IN ((?, ?))` than on AND-ed equalities"
+ (doseq [[label sql+params] {"describe-fields"
+ (sql-jdbc.sync/describe-fields-sql
+ :databricks
+ :schema-names ["mycatalog.myschema"]
+ :details {:catalog "mycatalog" :multi-level-schema true})
+
+ "describe-fks"
+ (sql-jdbc.sync/describe-fks-sql
+ :databricks
+ :schema-names ["mycatalog.myschema"]
+ :details {:catalog "mycatalog" :multi-level-schema true})}]
+ (testing label
+ (let [[sql & params] sql+params]
+ (is (not (str/includes? sql "IN ((?, ?)")))
+ (is (str/includes? sql "`table_catalog` = ?"))
+ (is (str/includes? sql "`table_schema` = ?"))
+ (is (= ["mycatalog" "myschema"] (vec (take-last 2 params)))))))))
+ (testing "several catalog.schema pairs are OR-ed pairwise"
+ (let [[sql & params] (sql-jdbc.sync/describe-fields-sql
+ :databricks
+ :schema-names ["c1.s1" "c2.s2"]
+ :details {:catalog "c1" :multi-level-schema true})]
+ (is (not (str/includes? sql "IN ((?, ?)")))
+ (is (= ["c1" "s1" "c2" "s2"] params))))
+ (testing "single-catalog connections keep the plain schema IN"
+ (let [[sql & params] (sql-jdbc.sync/describe-fields-sql
+ :databricks
+ :schema-names ["s1" "s2"]
+ :details {:catalog "c1" :multi-level-schema false})]
+ (is (str/includes? sql "`table_schema` IN (?, ?)"))
+ (is (= ["c1" "s1" "s2"] params)))))
+
(mt/defdataset dataset-with-ntz
[["table_with_ntz" [{:field-name "timestamp"
:base-type {:native "timestamp_ntz"}}]