Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix athena sync 65728 (#65777)

Riley Thompson854650cd864a
diff --git a/modules/drivers/athena/src/metabase/driver/athena/hive_parser.clj b/modules/drivers/athena/src/metabase/driver/athena/hive_parser.clj
index 148f30b4c56..c50d3df2155 100644
--- a/modules/drivers/athena/src/metabase/driver/athena/hive_parser.clj
+++ b/modules/drivers/athena/src/metabase/driver/athena/hive_parser.clj
@@ -6,10 +6,14 @@
(set! *warn-on-reflection* true)
+(def map-close
+ "The closing brackets of a map type"
+ "}]")
+
(defn- parse-to-json-string [s]
- (loop [schema s
- closes []
- result ""]
+ (loop [schema s
+ closes []
+ result ""]
(cond
(not (nil? (re-find #"^array<[a-z0-9]*>" schema)))
(recur (str/replace-first schema #"^array<[a-z0-9]*>" "")
@@ -21,10 +25,10 @@
(conj closes "]")
(str result "["))
- (str/starts-with? schema "map<string,string>")
- (recur (str/replace-first schema #"map<string,string>" "")
- closes
- (str result "[{\"key\":\"string\",\"value\":\"string\"}]"))
+ (str/starts-with? schema "map<")
+ (recur (str/replace-first schema #"map<" "")
+ (conj closes map-close)
+ (str result "[{\"key\":"))
(str/starts-with? schema "struct<")
(recur (str/replace-first schema #"struct<" "")
@@ -39,12 +43,14 @@
(str/starts-with? schema ",")
(recur (str/replace-first schema #",\s*" "")
closes
- (str result ","))
+ (str result (if (= (peek closes) map-close)
+ ",\"value\":"
+ ",")))
(str/starts-with? schema ">")
(recur (str/replace-first schema #">" "")
(pop closes)
- (str result (peek closes))) ; Preciso saber se é } ou ]
+ (str result (peek closes)))
:else (let [name-or-type (re-find #"\w+" schema)]
(if (= name-or-type nil)
diff --git a/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj b/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj
index cf3fd1a36c5..d38a812ea8a 100644
--- a/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj
+++ b/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj
@@ -1,7 +1,7 @@
(ns metabase.driver.athena.schema-parser
(:require
+ #_[metabase.driver.athena.hive-parser :as athena.hive-parser]
[clojure.string :as str]
- [metabase.driver.athena.hive-parser :as athena.hive-parser]
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]))
(set! *warn-on-reflection* true)
@@ -9,29 +9,29 @@
(defn- column->base-type [column-type]
(sql-jdbc.sync/database-type->base-type :athena (keyword (re-find #"\w+" column-type))))
-(defn- create-nested-fields [schema database-position]
- (set (map (fn [[k v]]
- (let [root {:name (name k)
- :base-type (cond (map? v) :type/Dictionary
- (sequential? v) :type/Array
- :else (column->base-type v))
- :database-type (cond (map? v) "map"
- (sequential? v) "array"
- :else v)
- :database-position database-position}]
- (cond
- (map? v) (assoc root :nested-fields (create-nested-fields v database-position))
- :else root)))
- schema)))
+#_(defn- create-nested-fields [schema database-position]
+ (set (map (fn [[k v]]
+ (let [root {:name (name k)
+ :base-type (cond (map? v) :type/Dictionary
+ (sequential? v) :type/Array
+ :else (column->base-type v))
+ :database-type (cond (map? v) "map"
+ (sequential? v) "array"
+ :else v)
+ :database-position database-position}]
+ (cond
+ (map? v) (assoc root :nested-fields (create-nested-fields v database-position))
+ :else root)))
+ schema)))
(defn- parse-struct-type-field [field-info database-position]
(let [root-field-name (:name field-info)
- schema (athena.hive-parser/hive-schema->map (:type field-info))]
+ #_#_schema (athena.hive-parser/hive-schema->map (:type field-info))]
{:name root-field-name
:base-type :type/Dictionary
:database-type "struct"
:database-position database-position
- :nested-fields (create-nested-fields schema database-position)}))
+ #_#_:nested-fields (create-nested-fields schema database-position)}))
(defn- parse-array-type-field [field-info database-position]
{:name (:name field-info) :base-type :type/Array :database-type "array" :database-position database-position})
diff --git a/modules/drivers/athena/test/metabase/driver/athena/hive_parser_test.clj b/modules/drivers/athena/test/metabase/driver/athena/hive_parser_test.clj
index b1ad9d97d1b..97e23f3f5b2 100644
--- a/modules/drivers/athena/test/metabase/driver/athena/hive_parser_test.clj
+++ b/modules/drivers/athena/test/metabase/driver/athena/hive_parser_test.clj
@@ -16,4 +16,17 @@
(is (= {:accredited_buyer_representative_abr "boolean"}
(hive-schema->map "struct<accredited_buyer_representative_abr: boolean>")))
(is (= {:mediacategory "string" :mediakey "string" :mediaurl "string" :order "bigint"}
- (hive-schema->map "struct<mediacategory: string, mediakey: string, mediaurl: string, order: bigint>")))))
+ (hive-schema->map "struct<mediacategory: string, mediakey: string, mediaurl: string, order: bigint>")))
+ (is (= [{:key "string" :value "string"}]
+ (hive-schema->map "map<string, string>")))
+ (is (= [{:key "int" :value "int"}]
+ (hive-schema->map "map<int, int>")))
+ (is (= [{:key "int" :value [{:key "string" :value "boolean"}]}]
+ (hive-schema->map "map<int, map<string, boolean>>")))
+ (is (= {:field [{:key "string" :value {:x "string"}}]}
+ (hive-schema->map "struct<field:map<string,struct<x:string>>>")))
+ (is (= {:extendedfields [{:key "string"
+ :value {:note {:title "string" :description "string" :values []}
+ :channels {:terminal "string" :app "string" :web "string"}}}]}
+ (hive-schema->map "struct<extendedfields:map<string,struct<note:struct<title:string,description:string,values:array<string>>,channels:struct<terminal:string,app:string,web:string>>>>")))))
+
diff --git a/modules/drivers/athena/test/metabase/driver/athena_test.clj b/modules/drivers/athena/test/metabase/driver/athena_test.clj
index 81d310418d9..e3cc889d430 100644
--- a/modules/drivers/athena/test/metabase/driver/athena_test.clj
+++ b/modules/drivers/athena/test/metabase/driver/athena_test.clj
@@ -41,7 +41,7 @@
{:name "data"
:base-type :type/Dictionary
:database-type "struct"
- :nested-fields #{{:name "name", :base-type :type/Text, :database-type "string", :database-position 1}},
+ #_#_:nested-fields #{{:name "name", :base-type :type/Text, :database-type "string", :database-position 1}},
:database-position 1}}
(#'athena/describe-table-fields-with-nested-fields "test" "test" "test")))))
(testing "sync without nested fields"