Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix Athena sync failing on partitioned Iceberg tables (#75666)

Timothy S. Dean80646be59970
diff --git a/modules/drivers/athena/src/metabase/driver/athena.clj b/modules/drivers/athena/src/metabase/driver/athena.clj
index 41680017350..3a42766bb48 100644
--- a/modules/drivers/athena/src/metabase/driver/athena.clj
+++ b/modules/drivers/athena/src/metabase/driver/athena.clj
@@ -370,14 +370,32 @@
{:name (str/trim (:col_name column-metadata))
:type (str/trim (:data_type column-metadata))})
+;; the partition section markers and row shapes come from Hive's DESCRIBE formatter, which this
+;; output format derives from:
+;; https://github.com/apache/hive/blob/1cb455bbb6753e548e1435f4345b4e0d4644bf9f/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java#L121-L185
+(defn- partition-section-start?
+ "Whether this row of `DESCRIBE` output starts the partition section: `# Partition Information`
+ for natively-partitioned tables, or `# Partition Transform Information` for Iceberg tables."
+ [{:keys [col_name]}]
+ (str/starts-with? (str col_name) "# Partition"))
+
(defn- remove-invalid-columns
- "Returns a transducer."
+ "Returns a transducer that converts rows of `DESCRIBE` output into column metadata, dropping the
+ partition section and other non-column rows."
[]
- (comp (remove #(= (:col_name %) ""))
+ ;; everything from the partition section marker on describes the partition spec rather than
+ ;; additional columns. Hive tables repeat the partition columns there with their real types;
+ ;; Iceberg tables repeat them with the partition transform (e.g. `identity`) as the type, which
+ ;; would otherwise sync as duplicate columns with no base type (#75579)
+ (comp (take-while (complement partition-section-start?))
+ (remove #(= (:col_name %) ""))
(remove #(= (:col_name %) nil))
(remove #(= (:data_type %) nil))
(remove #(str/starts-with? (:col_name %) "#")) ; remove comment
- (distinct) ; driver can return twice the partitioning fields
+ ;; the partition section only repeats columns that are already in the main column list, so
+ ;; if the marker text above ever changes, keeping the first occurrence of each name still
+ ;; keeps the partition rows out
+ (m/distinct-by :col_name)
(map describe-database->clj)))
(defn- normalize-field-info
diff --git a/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj b/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj
index d38a812ea8a..5474c08320e 100644
--- a/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj
+++ b/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj
@@ -6,8 +6,8 @@
(set! *warn-on-reflection* true)
-(defn- column->base-type [column-type]
- (sql-jdbc.sync/database-type->base-type :athena (keyword (re-find #"\w+" column-type))))
+(defn- column->base-type [column-name column-type]
+ (sql-jdbc.sync/database-type->base-type-or-warn :athena [column-name] (re-find #"\w+" column-type)))
#_(defn- create-nested-fields [schema database-position]
(set (map (fn [[k v]]
@@ -55,6 +55,6 @@
:else
{:name (:name field-info)
- :base-type (column->base-type (:type field-info))
+ :base-type (column->base-type (:name field-info) (:type field-info))
:database-type (:type field-info)
:database-position (:database-position field-info)}))
diff --git a/modules/drivers/athena/test/metabase/driver/athena_test.clj b/modules/drivers/athena/test/metabase/driver/athena_test.clj
index 07860232d3b..3a43f487e51 100644
--- a/modules/drivers/athena/test/metabase/driver/athena_test.clj
+++ b/modules/drivers/athena/test/metabase/driver/athena_test.clj
@@ -59,6 +59,70 @@
{:name "ts", :base-type :type/Text, :database-type "STRING", :database-position 1}}
(#'athena/describe-table-fields-without-nested-fields :athena "test" "test" upper-case-schema-columns)))))
+;; the partition section format comes from Hive's DESCRIBE formatter; see
+;; `PARTITION_TRANSFORM_SPEC_SCHEMA` in
+;; https://github.com/apache/hive/blob/1cb455bbb6753e548e1435f4345b4e0d4644bf9f/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java#L43
+(def ^:private iceberg-partitioned-schema
+ "DESCRIBE output for an Iceberg table partitioned by `identity(weather_id)` and
+ `identity(temporal_key)`. Unlike Hive tables, where the partition section repeats the columns
+ with their real types, Iceberg lists the partition transform (e.g. `identity`) in the type
+ column."
+ [{:_col0 "weather_id\tstring\t"}
+ {:_col0 "subcounty_id\tstring\t"}
+ {:_col0 "weight\tfloat\t"}
+ {:_col0 "unused_index\tint\t"}
+ {:_col0 "temporal_key\tstring\t"}
+ {:_col0 "# Partition Transform Information\t\t"}
+ {:_col0 "# col_name\ttransform_type\t"}
+ {:_col0 "weather_id\tidentity\t"}
+ {:_col0 "temporal_key\tidentity\t"}])
+
+(def ^:private hive-partitioned-schema
+ "The same table as a Hive-style partitioned table: the partition section repeats the partition
+ columns with their real types."
+ [{:_col0 "weather_id\tstring\t"}
+ {:_col0 "subcounty_id\tstring\t"}
+ {:_col0 "weight\tfloat\t"}
+ {:_col0 "unused_index\tint\t"}
+ {:_col0 "temporal_key\tstring\t"}
+ {:_col0 "# Partition Information\t\t"}
+ {:_col0 "# col_name\tdata_type\tcomment"}
+ {:_col0 "weather_id\tstring\t"}
+ {:_col0 "temporal_key\tstring\t"}])
+
+(deftest ^:parallel describe-iceberg-partitioned-table-test
+ (let [expected-fields #{{:name "weather_id", :base-type :type/Text, :database-type "string", :database-position 0}
+ {:name "subcounty_id", :base-type :type/Text, :database-type "string", :database-position 1}
+ {:name "weight", :base-type :type/Float, :database-type "float", :database-position 2}
+ {:name "unused_index", :base-type :type/Integer, :database-type "int", :database-position 3}
+ {:name "temporal_key", :base-type :type/Text, :database-type "string", :database-position 4}}]
+ (testing "Iceberg partition transform rows in DESCRIBE output shouldn't create duplicate Fields (#75579)"
+ (mt/with-dynamic-fn-redefs [athena/run-query (constantly iceberg-partitioned-schema)]
+ (is (= expected-fields
+ (#'athena/describe-table-fields-with-nested-fields "test" "test" "test")))))
+ (testing "partition columns repeated Hive-style with their real types are only synced once"
+ (mt/with-dynamic-fn-redefs [athena/run-query (constantly hive-partitioned-schema)]
+ (is (= expected-fields
+ (#'athena/describe-table-fields-with-nested-fields "test" "test" "test")))))
+ (testing "repeated partition rows are dropped even if the partition section marker is missing"
+ (mt/with-dynamic-fn-redefs [athena/run-query (constantly [{:_col0 "weather_id\tstring\t"}
+ {:_col0 "subcounty_id\tstring\t"}
+ {:_col0 "weight\tfloat\t"}
+ {:_col0 "unused_index\tint\t"}
+ {:_col0 "temporal_key\tstring\t"}
+ {:_col0 "weather_id\tidentity\t"}
+ {:_col0 "temporal_key\tidentity\t"}])]
+ (is (= expected-fields
+ (#'athena/describe-table-fields-with-nested-fields "test" "test" "test")))))))
+
+(deftest ^:parallel describe-unknown-database-type-test
+ (testing "unknown database types should fall back to :type/* instead of a nil base type (#75579)"
+ (mt/with-dynamic-fn-redefs [athena/run-query (constantly [{:_col0 "id\tint\t"}
+ {:_col0 "hll\thyperloglog\t"}])]
+ (is (= #{{:name "id", :base-type :type/Integer, :database-type "int", :database-position 0}
+ {:name "hll", :base-type :type/*, :database-type "hyperloglog", :database-position 1}}
+ (#'athena/describe-table-fields-with-nested-fields "test" "test" "test"))))))
+
(deftest ^:parallel describe-table-fields-with-nested-fields-test
(driver/with-driver :athena
(is (= #{{:name "id", :base-type :type/Integer, :database-type "int", :database-position 0}