Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix append for CSVs with really long column names with shared prefixes (#65793)

Chris Truter55deb9059835
diff --git a/src/metabase/upload/impl.clj b/src/metabase/upload/impl.clj
index 2b4bc94564d..1e797dafbd4 100644
--- a/src/metabase/upload/impl.clj
+++ b/src/metabase/upload/impl.clj
@@ -678,12 +678,10 @@
- the schema of the CSV file does not match the schema of the table
Note that we do not require the column ordering to be consistent between the header and the table schema."
- [fields-by-normed-name normalized-header]
- ;; Assumes table-cols are unique when normalized
- (let [normalized-field-names (keys fields-by-normed-name)
- [extra missing _both] (data/diff (set normalized-header) (set normalized-field-names))]
- ;; check for duplicates
- (when (some #(< 1 %) (vals (frequencies normalized-header)))
+ [existing-column-names upload-column-names raw-header]
+ (let [[extra missing _both] (data/diff (set upload-column-names) (set existing-column-names))]
+ ;; We use lossy normalization for column names then *force* uniqueness, so this must use the original header.
+ (when (some #(< 1 %) (vals (frequencies raw-header)))
(throw (ex-info (tru "The CSV file contains duplicate column names.")
{:status-code 422})))
(when-let [error-message (extra-and-missing-error-markdown extra missing)]
@@ -788,14 +786,20 @@
auto-pk?
without-auto-pk-columns)
name->field (m/index-by :name (t2/select :model/Field :table_id (:id table) :active true))
- column-names (for [h header] (normalize-column-name driver h))
+ ;; Gotcha: Long column names, which get sanitized and truncated to the same string, will be match to the
+ ;; database columns based on their order. If their order in the new upload differs from that in previous
+ ;; uploads, they will be matched incorrectly.
+ ;; We accept this edge case (customers can reorder CSV columns to fix) rather than rejecting uploads
+ ;; with ambiguous column names even when the order is consistent (see #44926/#issuecomment-3524373073).
+ ;; Future idea: match on display names for smart re-ordering.
+ column-names (map name (derive-column-names driver header))
display-names (for [h header] (normalize-display-name h))
create-auto-pk? (and
auto-pk?
(driver/create-auto-pk-with-append-csv? driver)
(not (contains? name->field auto-pk-column-name)))
name->field (cond-> name->field auto-pk? (dissoc auto-pk-column-name))
- _ (check-schema name->field column-names)
+ _ (check-schema (keys name->field) column-names header)
settings (upload-parsing/get-settings)
;; TODO: Add a method for drivers to override types here. See https://github.com/metabase/metabase/pull/55209.
old-types (map (comp upload-types/base-type->upload-type :base_type name->field) column-names)
diff --git a/test/metabase/upload/impl_test.clj b/test/metabase/upload/impl_test.clj
index 25f785fa93f..0cefe60af7d 100644
--- a/test/metabase/upload/impl_test.clj
+++ b/test/metabase/upload/impl_test.clj
@@ -2556,7 +2556,7 @@
(io/delete-file file))))))))
(deftest append-with-really-long-names-that-duplicate-test
- (testing "Upload a CSV file with unique column names that get sanitized to the same string"
+ (testing "Upload a CSV file with unique column names that get sanitized to the same string\n"
(mt/test-drivers (mt/normal-drivers-with-feature :uploads)
(with-mysql-local-infile-on-and-off
(let [long-string (str (str/join (repeat 1000 "really_")) "long")
@@ -2570,14 +2570,14 @@
:file (csv-file-with [header original-row]))]
(let [csv-rows [header appended-row]
file (csv-file-with csv-rows (mt/random-name))]
- ;; TODO: we should be able to make this work with smarter truncation
- (is (= {:message "The CSV file contains duplicate column names."
- :data {:status-code 422}}
- (catch-ex-info (update-csv! :metabase.upload/append {:file file, :table-id (:id table)}))))
- (testing "Check the data was not uploaded into the table"
- (is (= (rows-with-auto-pk (csv/read-csv original-row))
- (rows-for-table table))))
- (io/delete-file file))))))))
+ (try
+ (testing "Column names get deduplicated"
+ (update-csv! :metabase.upload/append {:file file, :table-id (:id table)})
+ (is (= (rows-with-auto-pk (concat (csv/read-csv original-row)
+ (csv/read-csv appended-row)))
+ (rows-for-table table))))
+ (finally
+ (io/delete-file file))))))))))
(driver/register! ::short-column-test-driver)
(defmethod driver/column-name-length-limit ::short-column-test-driver [_] 10)