The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix Athena sync failing on partitioned Iceberg tables (#75666)
Timothy S. Dean80646be59970
diff --git a/modules/drivers/athena/src/metabase/driver/athena.clj b/modules/drivers/athena/src/metabase/driver/athena.cljindex 41680017350..3a42766bb48 100644--- a/modules/drivers/athena/src/metabase/driver/athena.clj+++ b/modules/drivers/athena/src/metabase/driver/athena.clj@@ -370,14 +370,32 @@{:name (str/trim (:col_name column-metadata)):type (str/trim (:data_type column-metadata))})+;; the partition section markers and row shapes come from Hive's DESCRIBE formatter, which this+;; output format derives from:+;; https://github.com/apache/hive/blob/1cb455bbb6753e548e1435f4345b4e0d4644bf9f/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java#L121-L185+(defn- partition-section-start?+ "Whether this row of `DESCRIBE` output starts the partition section: `# Partition Information`+ for natively-partitioned tables, or `# Partition Transform Information` for Iceberg tables."+ [{:keys [col_name]}]+ (str/starts-with? (str col_name) "# Partition"))+(defn- remove-invalid-columns- "Returns a transducer."+ "Returns a transducer that converts rows of `DESCRIBE` output into column metadata, dropping the+ partition section and other non-column rows."[]- (comp (remove #(= (:col_name %) ""))+ ;; everything from the partition section marker on describes the partition spec rather than+ ;; additional columns. Hive tables repeat the partition columns there with their real types;+ ;; Iceberg tables repeat them with the partition transform (e.g. `identity`) as the type, which+ ;; would otherwise sync as duplicate columns with no base type (#75579)+ (comp (take-while (complement partition-section-start?))+ (remove #(= (:col_name %) ""))(remove #(= (:col_name %) nil))(remove #(= (:data_type %) nil))(remove #(str/starts-with? (:col_name %) "#")) ; remove comment- (distinct) ; driver can return twice the partitioning fields+ ;; the partition section only repeats columns that are already in the main column list, so+ ;; if the marker text above ever changes, keeping the first occurrence of each name still+ ;; keeps the partition rows out+ (m/distinct-by :col_name)(map describe-database->clj)))(defn- normalize-field-infodiff --git a/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj b/modules/drivers/athena/src/metabase/driver/athena/schema_parser.cljindex d38a812ea8a..5474c08320e 100644--- a/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj+++ b/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj@@ -6,8 +6,8 @@(set! *warn-on-reflection* true)-(defn- column->base-type [column-type]- (sql-jdbc.sync/database-type->base-type :athena (keyword (re-find #"\w+" column-type))))+(defn- column->base-type [column-name column-type]+ (sql-jdbc.sync/database-type->base-type-or-warn :athena [column-name] (re-find #"\w+" column-type)))#_(defn- create-nested-fields [schema database-position](set (map (fn [[k v]]@@ -55,6 +55,6 @@:else{:name (:name field-info)- :base-type (column->base-type (:type field-info))+ :base-type (column->base-type (:name field-info) (:type field-info)):database-type (:type field-info):database-position (:database-position field-info)}))diff --git a/modules/drivers/athena/test/metabase/driver/athena_test.clj b/modules/drivers/athena/test/metabase/driver/athena_test.cljindex 07860232d3b..3a43f487e51 100644--- a/modules/drivers/athena/test/metabase/driver/athena_test.clj+++ b/modules/drivers/athena/test/metabase/driver/athena_test.clj@@ -59,6 +59,70 @@{:name "ts", :base-type :type/Text, :database-type "STRING", :database-position 1}}(#'athena/describe-table-fields-without-nested-fields :athena "test" "test" upper-case-schema-columns)))))+;; the partition section format comes from Hive's DESCRIBE formatter; see+;; `PARTITION_TRANSFORM_SPEC_SCHEMA` in+;; https://github.com/apache/hive/blob/1cb455bbb6753e548e1435f4345b4e0d4644bf9f/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java#L43+(def ^:private iceberg-partitioned-schema+ "DESCRIBE output for an Iceberg table partitioned by `identity(weather_id)` and+ `identity(temporal_key)`. Unlike Hive tables, where the partition section repeats the columns+ with their real types, Iceberg lists the partition transform (e.g. `identity`) in the type+ column."+ [{:_col0 "weather_id\tstring\t"}+ {:_col0 "subcounty_id\tstring\t"}+ {:_col0 "weight\tfloat\t"}+ {:_col0 "unused_index\tint\t"}+ {:_col0 "temporal_key\tstring\t"}+ {:_col0 "# Partition Transform Information\t\t"}+ {:_col0 "# col_name\ttransform_type\t"}+ {:_col0 "weather_id\tidentity\t"}+ {:_col0 "temporal_key\tidentity\t"}])++(def ^:private hive-partitioned-schema+ "The same table as a Hive-style partitioned table: the partition section repeats the partition+ columns with their real types."+ [{:_col0 "weather_id\tstring\t"}+ {:_col0 "subcounty_id\tstring\t"}+ {:_col0 "weight\tfloat\t"}+ {:_col0 "unused_index\tint\t"}+ {:_col0 "temporal_key\tstring\t"}+ {:_col0 "# Partition Information\t\t"}+ {:_col0 "# col_name\tdata_type\tcomment"}+ {:_col0 "weather_id\tstring\t"}+ {:_col0 "temporal_key\tstring\t"}])++(deftest ^:parallel describe-iceberg-partitioned-table-test+ (let [expected-fields #{{:name "weather_id", :base-type :type/Text, :database-type "string", :database-position 0}+ {:name "subcounty_id", :base-type :type/Text, :database-type "string", :database-position 1}+ {:name "weight", :base-type :type/Float, :database-type "float", :database-position 2}+ {:name "unused_index", :base-type :type/Integer, :database-type "int", :database-position 3}+ {:name "temporal_key", :base-type :type/Text, :database-type "string", :database-position 4}}]+ (testing "Iceberg partition transform rows in DESCRIBE output shouldn't create duplicate Fields (#75579)"+ (mt/with-dynamic-fn-redefs [athena/run-query (constantly iceberg-partitioned-schema)]+ (is (= expected-fields+ (#'athena/describe-table-fields-with-nested-fields "test" "test" "test")))))+ (testing "partition columns repeated Hive-style with their real types are only synced once"+ (mt/with-dynamic-fn-redefs [athena/run-query (constantly hive-partitioned-schema)]+ (is (= expected-fields+ (#'athena/describe-table-fields-with-nested-fields "test" "test" "test")))))+ (testing "repeated partition rows are dropped even if the partition section marker is missing"+ (mt/with-dynamic-fn-redefs [athena/run-query (constantly [{:_col0 "weather_id\tstring\t"}+ {:_col0 "subcounty_id\tstring\t"}+ {:_col0 "weight\tfloat\t"}+ {:_col0 "unused_index\tint\t"}+ {:_col0 "temporal_key\tstring\t"}+ {:_col0 "weather_id\tidentity\t"}+ {:_col0 "temporal_key\tidentity\t"}])]+ (is (= expected-fields+ (#'athena/describe-table-fields-with-nested-fields "test" "test" "test")))))))++(deftest ^:parallel describe-unknown-database-type-test+ (testing "unknown database types should fall back to :type/* instead of a nil base type (#75579)"+ (mt/with-dynamic-fn-redefs [athena/run-query (constantly [{:_col0 "id\tint\t"}+ {:_col0 "hll\thyperloglog\t"}])]+ (is (= #{{:name "id", :base-type :type/Integer, :database-type "int", :database-position 0}+ {:name "hll", :base-type :type/*, :database-type "hyperloglog", :database-position 1}}+ (#'athena/describe-table-fields-with-nested-fields "test" "test" "test"))))))+(deftest ^:parallel describe-table-fields-with-nested-fields-test(driver/with-driver :athena(is (= #{{:name "id", :base-type :type/Integer, :database-type "int", :database-position 0}