Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix redshift sync failing with duplicate key error (#68097)

Riley Thompson2ef8d5038767
diff --git a/modules/drivers/redshift/src/metabase/driver/redshift.clj b/modules/drivers/redshift/src/metabase/driver/redshift.clj
index a3f618e2004..6175bc835fc 100644
--- a/modules/drivers/redshift/src/metabase/driver/redshift.clj
+++ b/modules/drivers/redshift/src/metabase/driver/redshift.clj
@@ -20,7 +20,8 @@
[metabase.util.date-2 :as u.date]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.json :as json]
- [metabase.util.log :as log])
+ [metabase.util.log :as log]
+ [metabase.util.performance :as perf])
(:import
(com.amazon.redshift.util RedshiftInterval)
(java.sql
@@ -59,10 +60,30 @@
;;; | metabase.driver impls |
;;; +----------------------------------------------------------------------------------------------------------------+
-;; Skip the postgres implementation of describe fields as it has to handle custom enums which redshift doesn't support.
+(defn- remove-duplicate-fields
+ "Redshift views can have duplicate column names, but when these columns appear in a query
+ they produce an ambiguous column error. To avoid this remove all duplicate columns"
+ [fields]
+ (let [field-key (fn [f] (perf/select-keys f [:table-schema :table-name :name]))
+ key-counts (frequencies (map field-key fields))
+ duplicate-keys (into #{} (keep (fn [[k cnt]] (when (> cnt 1) k)) key-counts))]
+ (doseq [{:keys [table-schema table-name name]} duplicate-keys]
+ (log/warnf "Duplicate column '%s' in %s.%s - skipping all occurrences"
+ name table-schema table-name))
+ (remove #(contains? duplicate-keys (field-key %)) fields)))
+
(defmethod sql-jdbc.sync/describe-fields-pre-process-xf :redshift
- [driver database & args]
- (apply (get-method sql-jdbc.sync/describe-fields-pre-process-xf :sql-jdbc) driver database args))
+ [_driver _db & _args]
+ (fn [rf]
+ (let [fields (volatile! (transient []))]
+ (fn
+ ([] (rf))
+ ([result]
+ (let [filtered (remove-duplicate-fields (persistent! @fields))]
+ (rf (reduce rf result filtered))))
+ ([result field]
+ (vswap! fields conj! field)
+ result)))))
;; Skip the postgres implementation as it has to handle custom enums which redshift doesn't support.
(defmethod driver/dynamic-database-types-lookup :redshift
diff --git a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
index 66354199b63..c8433388193 100644
--- a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
+++ b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
@@ -6,6 +6,7 @@
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
+ [metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table]
[metabase.driver.sql-jdbc.sync.interface :as sql-jdbc.sync]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.lib.core :as lib]
@@ -570,6 +571,19 @@
:type/DateTimeWithTZ [:timestamp-with-time-zone]
:type/Time [:time])))
+(deftest ^:parallel describe-fields-pre-process-xf-removes-all-duplicates-test
+ (testing "describe-fields-pre-process-xf removes ALL fields that have duplicate (table-schema, table-name, name)"
+ (is (= [{:table-schema "public" :table-name "orders" :name "id" :database-type "integer"}
+ {:table-schema "other" :table-name "users" :name "id" :database-type "integer"}]
+ (into []
+ (sql-jdbc.describe-table/describe-fields-pre-process-xf :redshift nil)
+ [{:table-schema "public" :table-name "users" :name "id" :database-type "integer"}
+ {:table-schema "public" :table-name "users" :name "id" :database-type "bigint"}
+ {:table-schema "public" :table-name "orders" :name "id" :database-type "integer"}
+ {:table-schema "other" :table-name "users" :name "id" :database-type "integer"}
+ {:table-schema "public" :table-name "users" :name "name" :database-type "varchar"}
+ {:table-schema "public" :table-name "users" :name "name" :database-type "text"}])))))
+
(deftest ^:parallel alternate-config-options-test
(testing "Can configure with either db or dbname"
(let [options {:host "test.example.com"}]