The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix redshift sync failing with duplicate key error (#68097)
Riley Thompson2ef8d5038767
diff --git a/modules/drivers/redshift/src/metabase/driver/redshift.clj b/modules/drivers/redshift/src/metabase/driver/redshift.cljindex a3f618e2004..6175bc835fc 100644--- a/modules/drivers/redshift/src/metabase/driver/redshift.clj+++ b/modules/drivers/redshift/src/metabase/driver/redshift.clj@@ -20,7 +20,8 @@[metabase.util.date-2 :as u.date][metabase.util.honey-sql-2 :as h2x][metabase.util.json :as json]- [metabase.util.log :as log])+ [metabase.util.log :as log]+ [metabase.util.performance :as perf])(:import(com.amazon.redshift.util RedshiftInterval)(java.sql@@ -59,10 +60,30 @@;;; | metabase.driver impls |;;; +----------------------------------------------------------------------------------------------------------------+-;; Skip the postgres implementation of describe fields as it has to handle custom enums which redshift doesn't support.+(defn- remove-duplicate-fields+ "Redshift views can have duplicate column names, but when these columns appear in a query+ they produce an ambiguous column error. To avoid this remove all duplicate columns"+ [fields]+ (let [field-key (fn [f] (perf/select-keys f [:table-schema :table-name :name]))+ key-counts (frequencies (map field-key fields))+ duplicate-keys (into #{} (keep (fn [[k cnt]] (when (> cnt 1) k)) key-counts))]+ (doseq [{:keys [table-schema table-name name]} duplicate-keys]+ (log/warnf "Duplicate column '%s' in %s.%s - skipping all occurrences"+ name table-schema table-name))+ (remove #(contains? duplicate-keys (field-key %)) fields)))+(defmethod sql-jdbc.sync/describe-fields-pre-process-xf :redshift- [driver database & args]- (apply (get-method sql-jdbc.sync/describe-fields-pre-process-xf :sql-jdbc) driver database args))+ [_driver _db & _args]+ (fn [rf]+ (let [fields (volatile! (transient []))]+ (fn+ ([] (rf))+ ([result]+ (let [filtered (remove-duplicate-fields (persistent! @fields))]+ (rf (reduce rf result filtered))))+ ([result field]+ (vswap! fields conj! field)+ result)))));; Skip the postgres implementation as it has to handle custom enums which redshift doesn't support.(defmethod driver/dynamic-database-types-lookup :redshiftdiff --git a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj b/modules/drivers/redshift/test/metabase/driver/redshift_test.cljindex 66354199b63..c8433388193 100644--- a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj+++ b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj@@ -6,6 +6,7 @@[metabase.driver :as driver][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]+ [metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table][metabase.driver.sql-jdbc.sync.interface :as sql-jdbc.sync][metabase.driver.sql.query-processor :as sql.qp][metabase.lib.core :as lib]@@ -570,6 +571,19 @@:type/DateTimeWithTZ [:timestamp-with-time-zone]:type/Time [:time])))+(deftest ^:parallel describe-fields-pre-process-xf-removes-all-duplicates-test+ (testing "describe-fields-pre-process-xf removes ALL fields that have duplicate (table-schema, table-name, name)"+ (is (= [{:table-schema "public" :table-name "orders" :name "id" :database-type "integer"}+ {:table-schema "other" :table-name "users" :name "id" :database-type "integer"}]+ (into []+ (sql-jdbc.describe-table/describe-fields-pre-process-xf :redshift nil)+ [{:table-schema "public" :table-name "users" :name "id" :database-type "integer"}+ {:table-schema "public" :table-name "users" :name "id" :database-type "bigint"}+ {:table-schema "public" :table-name "orders" :name "id" :database-type "integer"}+ {:table-schema "other" :table-name "users" :name "id" :database-type "integer"}+ {:table-schema "public" :table-name "users" :name "name" :database-type "varchar"}+ {:table-schema "public" :table-name "users" :name "name" :database-type "text"}])))))+(deftest ^:parallel alternate-config-options-test(testing "Can configure with either db or dbname"(let [options {:host "test.example.com"}]