The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix athena sync 65728 (#65777)
Riley Thompson854650cd864a
diff --git a/modules/drivers/athena/src/metabase/driver/athena/hive_parser.clj b/modules/drivers/athena/src/metabase/driver/athena/hive_parser.cljindex 148f30b4c56..c50d3df2155 100644--- a/modules/drivers/athena/src/metabase/driver/athena/hive_parser.clj+++ b/modules/drivers/athena/src/metabase/driver/athena/hive_parser.clj@@ -6,10 +6,14 @@(set! *warn-on-reflection* true)+(def map-close+ "The closing brackets of a map type"+ "}]")+(defn- parse-to-json-string [s]- (loop [schema s- closes []- result ""]+ (loop [schema s+ closes []+ result ""](cond(not (nil? (re-find #"^array<[a-z0-9]*>" schema)))(recur (str/replace-first schema #"^array<[a-z0-9]*>" "")@@ -21,10 +25,10 @@(conj closes "]")(str result "["))- (str/starts-with? schema "map<string,string>")- (recur (str/replace-first schema #"map<string,string>" "")- closes- (str result "[{\"key\":\"string\",\"value\":\"string\"}]"))+ (str/starts-with? schema "map<")+ (recur (str/replace-first schema #"map<" "")+ (conj closes map-close)+ (str result "[{\"key\":"))(str/starts-with? schema "struct<")(recur (str/replace-first schema #"struct<" "")@@ -39,12 +43,14 @@(str/starts-with? schema ",")(recur (str/replace-first schema #",\s*" "")closes- (str result ","))+ (str result (if (= (peek closes) map-close)+ ",\"value\":"+ ",")))(str/starts-with? schema ">")(recur (str/replace-first schema #">" "")(pop closes)- (str result (peek closes))) ; Preciso saber se é } ou ]+ (str result (peek closes))):else (let [name-or-type (re-find #"\w+" schema)](if (= name-or-type nil)diff --git a/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj b/modules/drivers/athena/src/metabase/driver/athena/schema_parser.cljindex cf3fd1a36c5..d38a812ea8a 100644--- a/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj+++ b/modules/drivers/athena/src/metabase/driver/athena/schema_parser.clj@@ -1,7 +1,7 @@(ns metabase.driver.athena.schema-parser(:require+ #_[metabase.driver.athena.hive-parser :as athena.hive-parser][clojure.string :as str]- [metabase.driver.athena.hive-parser :as athena.hive-parser][metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]))(set! *warn-on-reflection* true)@@ -9,29 +9,29 @@(defn- column->base-type [column-type](sql-jdbc.sync/database-type->base-type :athena (keyword (re-find #"\w+" column-type))))-(defn- create-nested-fields [schema database-position]- (set (map (fn [[k v]]- (let [root {:name (name k)- :base-type (cond (map? v) :type/Dictionary- (sequential? v) :type/Array- :else (column->base-type v))- :database-type (cond (map? v) "map"- (sequential? v) "array"- :else v)- :database-position database-position}]- (cond- (map? v) (assoc root :nested-fields (create-nested-fields v database-position))- :else root)))- schema)))+#_(defn- create-nested-fields [schema database-position]+ (set (map (fn [[k v]]+ (let [root {:name (name k)+ :base-type (cond (map? v) :type/Dictionary+ (sequential? v) :type/Array+ :else (column->base-type v))+ :database-type (cond (map? v) "map"+ (sequential? v) "array"+ :else v)+ :database-position database-position}]+ (cond+ (map? v) (assoc root :nested-fields (create-nested-fields v database-position))+ :else root)))+ schema)))(defn- parse-struct-type-field [field-info database-position](let [root-field-name (:name field-info)- schema (athena.hive-parser/hive-schema->map (:type field-info))]+ #_#_schema (athena.hive-parser/hive-schema->map (:type field-info))]{:name root-field-name:base-type :type/Dictionary:database-type "struct":database-position database-position- :nested-fields (create-nested-fields schema database-position)}))+ #_#_:nested-fields (create-nested-fields schema database-position)}))(defn- parse-array-type-field [field-info database-position]{:name (:name field-info) :base-type :type/Array :database-type "array" :database-position database-position})diff --git a/modules/drivers/athena/test/metabase/driver/athena/hive_parser_test.clj b/modules/drivers/athena/test/metabase/driver/athena/hive_parser_test.cljindex b1ad9d97d1b..97e23f3f5b2 100644--- a/modules/drivers/athena/test/metabase/driver/athena/hive_parser_test.clj+++ b/modules/drivers/athena/test/metabase/driver/athena/hive_parser_test.clj@@ -16,4 +16,17 @@(is (= {:accredited_buyer_representative_abr "boolean"}(hive-schema->map "struct<accredited_buyer_representative_abr: boolean>")))(is (= {:mediacategory "string" :mediakey "string" :mediaurl "string" :order "bigint"}- (hive-schema->map "struct<mediacategory: string, mediakey: string, mediaurl: string, order: bigint>")))))+ (hive-schema->map "struct<mediacategory: string, mediakey: string, mediaurl: string, order: bigint>")))+ (is (= [{:key "string" :value "string"}]+ (hive-schema->map "map<string, string>")))+ (is (= [{:key "int" :value "int"}]+ (hive-schema->map "map<int, int>")))+ (is (= [{:key "int" :value [{:key "string" :value "boolean"}]}]+ (hive-schema->map "map<int, map<string, boolean>>")))+ (is (= {:field [{:key "string" :value {:x "string"}}]}+ (hive-schema->map "struct<field:map<string,struct<x:string>>>")))+ (is (= {:extendedfields [{:key "string"+ :value {:note {:title "string" :description "string" :values []}+ :channels {:terminal "string" :app "string" :web "string"}}}]}+ (hive-schema->map "struct<extendedfields:map<string,struct<note:struct<title:string,description:string,values:array<string>>,channels:struct<terminal:string,app:string,web:string>>>>")))))+diff --git a/modules/drivers/athena/test/metabase/driver/athena_test.clj b/modules/drivers/athena/test/metabase/driver/athena_test.cljindex 81d310418d9..e3cc889d430 100644--- a/modules/drivers/athena/test/metabase/driver/athena_test.clj+++ b/modules/drivers/athena/test/metabase/driver/athena_test.clj@@ -41,7 +41,7 @@{:name "data":base-type :type/Dictionary:database-type "struct"- :nested-fields #{{:name "name", :base-type :type/Text, :database-type "string", :database-position 1}},+ #_#_:nested-fields #{{:name "name", :base-type :type/Text, :database-type "string", :database-position 1}},:database-position 1}}(#'athena/describe-table-fields-with-nested-fields "test" "test" "test")))))(testing "sync without nested fields"