The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GDGT-2233: Match colliding upload columns by display name on append/replace (#75090)
Ngoc Khuat6b0ea3600e21
diff --git a/src/metabase/upload/impl.clj b/src/metabase/upload/impl.cljindex 9c207c226cf..7b0a7b2a100 100644--- a/src/metabase/upload/impl.clj+++ b/src/metabase/upload/impl.clj@@ -367,6 +367,33 @@(mapv (comp keyword generator-fn)(for [h header] (normalize-column-name driver h)))))+(defn- match-column-names+ "Return the existing field name that each CSV `header` column should be written to.++ - Columns are matched to fields by name, independent of the order they appear in the CSV.+ - When several headers resolve to the same name, they are matched by display name so each value+ lands in the column it came from.+ - When columns can't be told apart even by display name, they are matched by the order they appear in."+ [driver header name->field]+ (let [positional (mapv name (derive-column-names driver header))+ normalized (mapv #(normalize-column-name driver %) header)+ collisions (into #{} (keep (fn [[col-name freq]] (when (> freq 1) col-name))) (frequencies normalized))]+ (if (empty? collisions)+ positional+ ;; Map each colliding field's display name (scoped to its normalized name) back to its positional+ ;; name, then re-match the colliding columns by display name.+ (let [display->name (into {}+ (for [[norm pos] (map vector normalized positional)+ :when (collisions norm)+ :let [field (name->field pos)]+ :when field]+ [[norm (:display_name field)] pos]))]+ (mapv (fn [norm pos display]+ (if (collisions norm)+ (get display->name [norm display] pos)+ pos))+ normalized positional (derive-display-names driver header))))))+(defn- create-from-csv!"Creates a table from a CSV file. If the table already exists, it will throw an error.Returns the file size, number of rows, and number of columns."@@ -789,13 +816,9 @@auto-pk?without-auto-pk-columns)name->field (m/index-by :name (t2/select :model/Field :table_id (:id table) :active true))- ;; Gotcha: Long column names, which get sanitized and truncated to the same string, will be match to the- ;; database columns based on their order. If their order in the new upload differs from that in previous- ;; uploads, they will be matched incorrectly.- ;; We accept this edge case (customers can reorder CSV columns to fix) rather than rejecting uploads- ;; with ambiguous column names even when the order is consistent (see #44926/#issuecomment-3524373073).- ;; Future idea: match on display names for smart re-ordering.- column-names (map name (derive-column-names driver header))+ ;; Match colliding columns to existing fields by display name so reordering them between+ ;; uploads doesn't write data to the wrong column. See [[match-column-names]] (GDGT-2233).+ column-names (match-column-names driver header name->field)display-names (for [h header] (normalize-display-name h))create-auto-pk? (andauto-pk?diff --git a/test/metabase/upload/impl_test.clj b/test/metabase/upload/impl_test.cljindex aa6b415fc16..875dd41d0af 100644--- a/test/metabase/upload/impl_test.clj+++ b/test/metabase/upload/impl_test.clj@@ -2558,6 +2558,27 @@(finally(io/delete-file file))))))))))+(deftest update-csv-normalized-name-collision-reorder-test+ (testing "Reordered columns that normalize to the same name are matched by display name, not position (GDGT-2233)\n"+ (mt/test-drivers (mt/normal-drivers-with-feature :uploads)+ (with-mysql-local-infile-on-and-off+ (doseq [action (actions-to-test driver/*driver*)]+ (testing (action-testing-str action)+ ;; "Café" and "Cafe" both normalize to "cafe", so the table has columns+ ;; cafe (display "Café") and cafe_2 (display "Cafe").+ (with-upload-table!+ [table (create-from-csv-and-sync-with-defaults!+ :file (csv-file-with ["Café,Cafe" "100,200"]))]+ (let [file (csv-file-with ["Cafe,Café" "500,600"] (mt/random-name))]+ (try+ (update-csv! action {:file file, :table-id (:id table)})+ ;; Even though the appended CSV swaps the column order, the "Café" value (600)+ ;; must land in the cafe column and the "Cafe" value (500) in cafe_2.+ (is (= (set (updated-contents action [[100 200]] [[600 500]]))+ (set (rows-for-table table))))+ (finally+ (io/delete-file file)))))))))))+(driver/register! ::short-column-test-driver)(defmethod driver/column-name-length-limit ::short-column-test-driver [_] 10)