The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix slow Athena syncs for database names with underscores (#84583)
John3fa8a743387a
diff --git a/modules/drivers/athena/src/metabase/driver/athena.clj b/modules/drivers/athena/src/metabase/driver/athena.cljindex ccafb80ce02..1cf20c102c1 100644--- a/modules/drivers/athena/src/metabase/driver/athena.clj+++ b/modules/drivers/athena/src/metabase/driver/athena.clj@@ -462,10 +462,23 @@(defn- table-has-nested-fields? [columns](some #(= "struct" (:type_name %)) columns))+(defn- escape-metadata-pattern+ "Escape a literal schema or table name for JDBC metadata pattern arguments."+ [^DatabaseMetaData metadata entity-name]+ (when entity-name+ (let [escape (.getSearchStringEscape metadata)]+ (-> entity-name+ (str/replace escape (str escape escape))+ (str/replace "_" (str escape "_"))+ (str/replace "%" (str escape "%"))))))+(defn- get-columns[^DatabaseMetaData metadata catalog schema table-name](try- (with-open [rs (.getColumns metadata catalog schema table-name nil)]+ (with-open [rs (.getColumns metadata catalog+ (escape-metadata-pattern metadata schema)+ (escape-metadata-pattern metadata table-name)+ nil)](jdbc/metadata-result rs))(catch Throwable e(log/warnf "`.getColumns` failed for catalog `%s`, schema `%s`, table name `%s` with message: `%s`"@@ -509,7 +522,10 @@databasenil(fn [^Connection conn]- (let [metadata (.getMetaData conn)]+ (let [metadata (.getMetaData conn)+ catalog (if (and (str/blank? catalog) (not (str/blank? dbname)))+ (.getCatalog conn)+ catalog)](assoc (select-keys table [:name :schema]):fields (try(describe-table-fields metadata database driver table catalog dbname)@@ -519,7 +535,8 @@(defn- get-tables[^DatabaseMetaData metadata, ^String schema-or-nil, ^String db-name-or-nil];; tablePattern "%" = match all tables- (with-open [rs (.getTables metadata db-name-or-nil schema-or-nil "%"+ (with-open [rs (.getTables metadata db-name-or-nil+ (escape-metadata-pattern metadata schema-or-nil) "%"(into-array String ["TABLE""VIEW"]))](vec (jdbc/metadata-result rs))))diff --git a/modules/drivers/athena/test/metabase/driver/athena_test.clj b/modules/drivers/athena/test/metabase/driver/athena_test.cljindex 4cb1003c49f..7853c569159 100644--- a/modules/drivers/athena/test/metabase/driver/athena_test.clj+++ b/modules/drivers/athena/test/metabase/driver/athena_test.clj@@ -5,6 +5,7 @@[clojure.test :refer :all][metabase.driver :as driver][metabase.driver.athena :as athena]+ [metabase.driver.connection :as driver.conn][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.sql.query-processor :as sql.qp]@@ -21,7 +22,9 @@[metabase.test.data.interface :as tx][metabase.util :as u][metabase.util.date-2 :as u.date]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (java.sql Connection DatabaseMetaData ResultSet ResultSetMetaData)))(set! *warn-on-reflection* true)@@ -37,6 +40,69 @@[{:column_name "id", :type_name "string"}{:column_name "ts", :type_name "STRING"}])+(defn- empty-metadata-result-set []+ (reify ResultSet+ (getMetaData [_]+ (reify ResultSetMetaData+ (getColumnCount [_] 0)))+ (next [_] false)+ (close [_])))++(deftest ^:parallel get-columns-literal-patterns-test+ (doseq [[schema table expected-schema expected-table]+ [[nil nil nil nil]+ ["database" "birds" "database" "birds"]+ ["database_name" "birds_seen" "database\\_name" "birds\\_seen"]+ ["db%name" "birds\\seen" "db\\%name" "birds\\\\seen"]]]+ (let [called (atom nil)+ metadata (reify DatabaseMetaData+ (getSearchStringEscape [_] "\\")+ (getColumns [_ catalog schema-pattern table-pattern column-pattern]+ (reset! called [catalog schema-pattern table-pattern column-pattern])+ (empty-metadata-result-set)))]+ (is (empty? (#'athena/get-columns metadata "AwsDataCatalog" schema table)))+ (is (= ["AwsDataCatalog" expected-schema expected-table nil] @called)))))++(deftest ^:parallel get-tables-literal-schema-test+ (doseq [[schema expected] [[nil nil] ["database_name%" "database!_name!%"]]]+ (let [called (atom nil)+ metadata (reify DatabaseMetaData+ (getSearchStringEscape [_] "!")+ (getTables [_ catalog schema-pattern table-pattern types]+ (reset! called [catalog schema-pattern table-pattern (vec types)])+ (empty-metadata-result-set)))]+ (is (empty? (#'athena/get-tables metadata schema "AwsDataCatalog")))+ (is (= ["AwsDataCatalog" expected "%" ["TABLE" "VIEW"]] @called)))))++(driver/register! ::metadata-test :parent :athena)++(defmethod sql-jdbc.execute/do-with-connection-with-options ::metadata-test+ [_driver {:keys [connection]} _options f]+ (f connection))++(deftest ^:parallel describe-table-catalog-test+ (doseq [[catalog dbname connection-catalog expected]+ [[nil "database_name" "AwsDataCatalog" "AwsDataCatalog"]+ ["" "database_name" "AwsDataCatalog" "AwsDataCatalog"]+ [" " "database_name" "AwsDataCatalog" "AwsDataCatalog"]+ [nil "database_name" "CustomCatalog" "CustomCatalog"]+ ["ConfiguredCatalog" "database_name" "CustomCatalog" "ConfiguredCatalog"]+ [nil nil "AwsDataCatalog" nil]+ [nil "" "AwsDataCatalog" nil]]]+ (testing (str "catalog=" (pr-str catalog) ", dbname=" (pr-str dbname))+ (let [called (atom nil)+ conn (reify Connection+ (getMetaData [_] nil)+ (getCatalog [_] connection-catalog))]+ (mt/with-dynamic-fn-redefs [driver.conn/effective-details+ (constantly {:catalog catalog :dbname dbname})+ athena/describe-table-fields+ (fn [_metadata _database _driver _table catalog dbname]+ (reset! called [catalog dbname])+ #{})]+ (driver/describe-table ::metadata-test {:connection conn} {:name "birds_seen" :schema nil})+ (is (= [expected dbname] @called)))))))+(deftest sync-test(testing "sync with nested fields"(mt/with-dynamic-fn-redefs [athena/run-query (constantly nested-schema)]