Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix slow Athena syncs for database names with underscores (#84583)

John3fa8a743387a
diff --git a/modules/drivers/athena/src/metabase/driver/athena.clj b/modules/drivers/athena/src/metabase/driver/athena.clj
index ccafb80ce02..1cf20c102c1 100644
--- a/modules/drivers/athena/src/metabase/driver/athena.clj
+++ b/modules/drivers/athena/src/metabase/driver/athena.clj
@@ -462,10 +462,23 @@
(defn- table-has-nested-fields? [columns]
(some #(= "struct" (:type_name %)) columns))
+(defn- escape-metadata-pattern
+ "Escape a literal schema or table name for JDBC metadata pattern arguments."
+ [^DatabaseMetaData metadata entity-name]
+ (when entity-name
+ (let [escape (.getSearchStringEscape metadata)]
+ (-> entity-name
+ (str/replace escape (str escape escape))
+ (str/replace "_" (str escape "_"))
+ (str/replace "%" (str escape "%"))))))
+
(defn- get-columns
[^DatabaseMetaData metadata catalog schema table-name]
(try
- (with-open [rs (.getColumns metadata catalog schema table-name nil)]
+ (with-open [rs (.getColumns metadata catalog
+ (escape-metadata-pattern metadata schema)
+ (escape-metadata-pattern metadata table-name)
+ nil)]
(jdbc/metadata-result rs))
(catch Throwable e
(log/warnf "`.getColumns` failed for catalog `%s`, schema `%s`, table name `%s` with message: `%s`"
@@ -509,7 +522,10 @@
database
nil
(fn [^Connection conn]
- (let [metadata (.getMetaData conn)]
+ (let [metadata (.getMetaData conn)
+ catalog (if (and (str/blank? catalog) (not (str/blank? dbname)))
+ (.getCatalog conn)
+ catalog)]
(assoc (select-keys table [:name :schema])
:fields (try
(describe-table-fields metadata database driver table catalog dbname)
@@ -519,7 +535,8 @@
(defn- get-tables
[^DatabaseMetaData metadata, ^String schema-or-nil, ^String db-name-or-nil]
;; tablePattern "%" = match all tables
- (with-open [rs (.getTables metadata db-name-or-nil schema-or-nil "%"
+ (with-open [rs (.getTables metadata db-name-or-nil
+ (escape-metadata-pattern metadata schema-or-nil) "%"
(into-array String ["TABLE"
"VIEW"]))]
(vec (jdbc/metadata-result rs))))
diff --git a/modules/drivers/athena/test/metabase/driver/athena_test.clj b/modules/drivers/athena/test/metabase/driver/athena_test.clj
index 4cb1003c49f..7853c569159 100644
--- a/modules/drivers/athena/test/metabase/driver/athena_test.clj
+++ b/modules/drivers/athena/test/metabase/driver/athena_test.clj
@@ -5,6 +5,7 @@
[clojure.test :refer :all]
[metabase.driver :as driver]
[metabase.driver.athena :as athena]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.sql.query-processor :as sql.qp]
@@ -21,7 +22,9 @@
[metabase.test.data.interface :as tx]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (java.sql Connection DatabaseMetaData ResultSet ResultSetMetaData)))
(set! *warn-on-reflection* true)
@@ -37,6 +40,69 @@
[{:column_name "id", :type_name "string"}
{:column_name "ts", :type_name "STRING"}])
+(defn- empty-metadata-result-set []
+ (reify ResultSet
+ (getMetaData [_]
+ (reify ResultSetMetaData
+ (getColumnCount [_] 0)))
+ (next [_] false)
+ (close [_])))
+
+(deftest ^:parallel get-columns-literal-patterns-test
+ (doseq [[schema table expected-schema expected-table]
+ [[nil nil nil nil]
+ ["database" "birds" "database" "birds"]
+ ["database_name" "birds_seen" "database\\_name" "birds\\_seen"]
+ ["db%name" "birds\\seen" "db\\%name" "birds\\\\seen"]]]
+ (let [called (atom nil)
+ metadata (reify DatabaseMetaData
+ (getSearchStringEscape [_] "\\")
+ (getColumns [_ catalog schema-pattern table-pattern column-pattern]
+ (reset! called [catalog schema-pattern table-pattern column-pattern])
+ (empty-metadata-result-set)))]
+ (is (empty? (#'athena/get-columns metadata "AwsDataCatalog" schema table)))
+ (is (= ["AwsDataCatalog" expected-schema expected-table nil] @called)))))
+
+(deftest ^:parallel get-tables-literal-schema-test
+ (doseq [[schema expected] [[nil nil] ["database_name%" "database!_name!%"]]]
+ (let [called (atom nil)
+ metadata (reify DatabaseMetaData
+ (getSearchStringEscape [_] "!")
+ (getTables [_ catalog schema-pattern table-pattern types]
+ (reset! called [catalog schema-pattern table-pattern (vec types)])
+ (empty-metadata-result-set)))]
+ (is (empty? (#'athena/get-tables metadata schema "AwsDataCatalog")))
+ (is (= ["AwsDataCatalog" expected "%" ["TABLE" "VIEW"]] @called)))))
+
+(driver/register! ::metadata-test :parent :athena)
+
+(defmethod sql-jdbc.execute/do-with-connection-with-options ::metadata-test
+ [_driver {:keys [connection]} _options f]
+ (f connection))
+
+(deftest ^:parallel describe-table-catalog-test
+ (doseq [[catalog dbname connection-catalog expected]
+ [[nil "database_name" "AwsDataCatalog" "AwsDataCatalog"]
+ ["" "database_name" "AwsDataCatalog" "AwsDataCatalog"]
+ [" " "database_name" "AwsDataCatalog" "AwsDataCatalog"]
+ [nil "database_name" "CustomCatalog" "CustomCatalog"]
+ ["ConfiguredCatalog" "database_name" "CustomCatalog" "ConfiguredCatalog"]
+ [nil nil "AwsDataCatalog" nil]
+ [nil "" "AwsDataCatalog" nil]]]
+ (testing (str "catalog=" (pr-str catalog) ", dbname=" (pr-str dbname))
+ (let [called (atom nil)
+ conn (reify Connection
+ (getMetaData [_] nil)
+ (getCatalog [_] connection-catalog))]
+ (mt/with-dynamic-fn-redefs [driver.conn/effective-details
+ (constantly {:catalog catalog :dbname dbname})
+ athena/describe-table-fields
+ (fn [_metadata _database _driver _table catalog dbname]
+ (reset! called [catalog dbname])
+ #{})]
+ (driver/describe-table ::metadata-test {:connection conn} {:name "birds_seen" :schema nil})
+ (is (= [expected dbname] @called)))))))
+
(deftest sync-test
(testing "sync with nested fields"
(mt/with-dynamic-fn-redefs [athena/run-query (constantly nested-schema)]