The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add UTF-8 BOM to CSV downloads (#76028)
Nathan Voxland3c58b8226fba
diff --git a/e2e/test/scenarios/sharing/sharing-reproductions.cy.spec.js b/e2e/test/scenarios/sharing/sharing-reproductions.cy.spec.jsindex 0652aa30763..61db08c7ea0 100644--- a/e2e/test/scenarios/sharing/sharing-reproductions.cy.spec.js+++ b/e2e/test/scenarios/sharing/sharing-reproductions.cy.spec.js@@ -1057,11 +1057,13 @@ describe("issue 49525", { tags: "@external" }, () => {url: `http://localhost:${WEB_PORT}/email/${email.id}/attachment/${csvAttachment.generatedFileName}`,encoding: "utf8",}).then((response) => {- const csvContent = response.body;- const rows = csvContent.split("\n");+ // CSV exports begin with a UTF-8 BOM; strip it, and tolerate either+ // \n or \r\n line endings, before asserting on the header row.+ const csvContent = response.body.replace(/^\uFEFF/, "");+ const rows = csvContent.split(/\r?\n/);const headers = rows[0];expect(headers).to.equal(- "Created At: Year,Doohickey,Gadget,Gizmo,Widget,Row totals\r",+ "Created At: Year,Doohickey,Gadget,Gizmo,Widget,Row totals",);});});diff --git a/frontend/src/metabase/redux/downloads.ts b/frontend/src/metabase/redux/downloads.tsindex 0e8c542611c..a1c1fb119d0 100644--- a/frontend/src/metabase/redux/downloads.ts+++ b/frontend/src/metabase/redux/downloads.ts@@ -288,6 +288,7 @@ export const downloadDataset = createAsyncThunk(type ExportParams = {format_rows: boolean;pivot_results: boolean;+ csv_include_bom: boolean;};const getPublicDashcardParams = (@@ -479,6 +480,7 @@ export const getDatasetParams = ({const exportParams: ExportParams = {format_rows: enableFormatting,pivot_results: enablePivot,+ csv_include_bom: true,};const { accessedVia, resourceType } = getDownloadedResourceType({diff --git a/frontend/src/metabase/redux/downloads.unit.spec.ts b/frontend/src/metabase/redux/downloads.unit.spec.tsindex a21014de2b0..8867be8308a 100644--- a/frontend/src/metabase/redux/downloads.unit.spec.ts+++ b/frontend/src/metabase/redux/downloads.unit.spec.ts@@ -149,6 +149,18 @@ describe("getDatasetParams - public question (uuid-based)", () => {expect(url.get("format_rows")).toBe("true");expect(url.get("pivot_results")).toBe("true");});++ it("requests the UTF-8 BOM so exports open correctly in Excel", () => {+ const downloadParams = getDatasetParams({+ type: "csv",+ question,+ result,+ uuid: PUBLIC_UUID,+ });++ const url = new URLSearchParams(downloadParams.params);+ expect(url.get("csv_include_bom")).toBe("true");+ });});describe("readDownloadBlob", () => {diff --git a/src/metabase/channel/email/result_attachment.clj b/src/metabase/channel/email/result_attachment.cljindex 5aee85e4bf4..be62d108aa3 100644--- a/src/metabase/channel/email/result_attachment.clj+++ b/src/metabase/channel/email/result_attachment.clj@@ -31,10 +31,13 @@Results are streamed synchronously. Caller is responsible for closing `os` when this call is complete."[^OutputStream os :- (ms/InstanceOfClass OutputStream)- {:keys [export-format format-rows? pivot?], :as _options} :- [:map- [:export-format :keyword]- [:format-rows? {:optional true} [:maybe :boolean]]- [:pivot? {:optional true} [:maybe :boolean]]]+ {:keys [export-format format-rows? pivot? csv-include-bom?]+ :or {csv-include-bom? true}+ :as _options} :- [:map+ [:export-format :keyword]+ [:format-rows? {:optional true} [:maybe :boolean]]+ [:pivot? {:optional true} [:maybe :boolean]]+ [:csv-include-bom? {:optional true} [:maybe :boolean]]]{{:keys [rows]} :data, database-id :database_id, :as results} :- [:map[:database_id ::lib.schema.id/database]]];; make sure Database/driver info is available for the streaming results writers -- they might need this in order to@@ -51,6 +54,7 @@(-> results(assoc-in [:data :format-rows?] format-rows?)(assoc-in [:data :pivot?] pivot?)+ (assoc-in [:data :csv-include-bom?] csv-include-bom?)(assoc-in [:data :ordered-cols] ordered-cols))viz-settings')(perf/reduce (fn [_ i row]@@ -106,7 +110,7 @@[(when-let [temp-file (and (:include_csv card)(create-temp-file-or-throw! "csv"))](with-open [os (io/output-stream temp-file)]- (stream-api-results-to-export-format! os {:export-format :csv :format-rows? format-rows :pivot? pivot-results} result))+ (stream-api-results-to-export-format! os {:export-format :csv :format-rows? format-rows :pivot? pivot-results :csv-include-bom? true} result))(create-result-attachment-map "csv" filename-prefix temp-file))(when-let [temp-file (and (:include_xls card)(create-temp-file-or-throw! "xlsx"))]diff --git a/src/metabase/channel/render/preview.clj b/src/metabase/channel/render/preview.cljindex 3fbf85eae08..e654d902f2d 100644--- a/src/metabase/channel/render/preview.clj+++ b/src/metabase/channel/render/preview.clj@@ -15,6 +15,7 @@[metabase.channel.render.png :as png][metabase.channel.render.style :as style][metabase.notification.payload.core :as notification.payload]+ [metabase.util :as u][metabase.util.markdown :as markdown][toucan2.core :as t2]))@@ -31,7 +32,7 @@(def ^:private csv-row-limit 10)(defn- csv-to-html-table [csv-string]- (let [rows (csv/read-csv csv-string)]+ (let [rows (csv/read-csv (u/strip-bom csv-string))][:table {:style table-style}(for [row (take (inc csv-row-limit) rows)] ;; inc row-limit to include the header and the expected # of rows[:tr {:style table-style}diff --git a/src/metabase/queries_rest/api/card.clj b/src/metabase/queries_rest/api/card.cljindex e06159a9d5a..c8c0c543cef 100644--- a/src/metabase/queries_rest/api/card.clj+++ b/src/metabase/queries_rest/api/card.clj@@ -918,17 +918,18 @@(api.macros/defendpoint :post "/:card-id/query/:export-format""Run the query associated with a Card, and return its results as a file in the specified format.- `parameters`, `pivot-results?` and `format-rows?` should be passed as application/x-www-form-urlencoded form content+ `csv_include_bom`, `parameters`, `pivot-results?` and `format-rows?` should be passed as application/x-www-form-urlencoded form contentor json in the body. This is because this endpoint is normally used to power 'Download Results' buttons that useHTML `form` actions)."[{:keys [card-id export-format]} :- [:map[:card-id ms/PositiveInt][:export-format ::qp.schema/export-format]]_query-params- {:keys [parameters]- pivot-results? :pivot_results- format-rows? :format_rows- :as _body}+ {:keys [parameters]+ pivot-results? :pivot_results+ format-rows? :format_rows+ csv-include-bom? :csv_include_bom+ :as _body}:- [:map[:parameters {:optional true} [:maybe;; support JSON-encoded parameters for backwards compatibility when with this@@ -942,7 +943,8 @@;; it breaks existing tests[:sequential [:map-of :keyword :any]]]][:format_rows {:default false} ms/BooleanValue]- [:pivot_results {:default false} ms/BooleanValue]]]+ [:pivot_results {:default false} ms/BooleanValue]+ [:csv_include_bom {:default false} ms/BooleanValue]]](qp.card/process-query-for-card(api/check-404 (t2/select-one :model/Card card-id)) export-format:parameters parameters@@ -953,6 +955,7 @@:ignore-cached-results? true:format-rows? format-rows?:pivot? pivot-results?+ :csv-include-bom? (if (some? csv-include-bom?) csv-include-bom? false):js-int-to-string? false}));;; ----------------------------------------------- Sharing is Caring ------------------------------------------------diff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.cljindex 7071f550a94..50a62a1b451 100644--- a/src/metabase/query_processor/api.clj+++ b/src/metabase/query_processor/api.clj@@ -134,6 +134,7 @@{{:keys [was-pivot] :as query} :queryformat-rows :format_rowspivot-results :pivot_results+ csv-include-bom :csv_include_bomvisualization-settings :visualization_settings};; Support JSON-encoded query and viz settings for backwards compatibility for when downloads used to be triggered by;; `<form>` submissions... see https://metaboat.slack.com/archives/C010L1Z4F9S/p1738003606875659@@ -147,7 +148,8 @@(cond-> x(string? x) (json/decode viz-setting-key-fn)))}]][:format_rows {:default false} ms/BooleanValue]- [:pivot_results {:default false} ms/BooleanValue]]]+ [:pivot_results {:default false} ms/BooleanValue]+ [:csv_include_bom {:default false} ms/BooleanValue]]](let [viz-settings (-> visualization-settingsmi/normalize-visualization-settingsmb.viz/norm->db)@@ -158,6 +160,7 @@(dissoc :add-default-userland-constraints? :js-int-to-string?)(assoc :format-rows? (or format-rows false):pivot? (or pivot-results false)+ :csv-include-bom? (if (some? csv-include-bom) csv-include-bom false):process-viz-settings? true:skip-results-metadata? true))))](run-streaming-querydiff --git a/src/metabase/query_processor/middleware/format_rows.clj b/src/metabase/query_processor/middleware/format_rows.cljindex fa295a119f3..06791f8873c 100644--- a/src/metabase/query_processor/middleware/format_rows.clj+++ b/src/metabase/query_processor/middleware/format_rows.clj@@ -98,11 +98,12 @@(defn format-rows"Format individual query result values as needed. Ex: format temporal values as ISO-8601 strings w/ timezone offset."- [{{:keys [format-rows?] :or {format-rows? true}} :middleware, :as _query} rff]+ [{{:keys [format-rows? csv-include-bom?] :or {format-rows? true, csv-include-bom? true}} :middleware, :as _query} rff](fn format-rows-rff* [metadata]- ;; always assoc `:format-rows?` into the metadata so that- ;; the `qp.si/streaming-results-writer` implmementations can apply/not-apply formatting based on the key's value- (let [metadata (assoc metadata :format-rows? format-rows?)]+ ;; always assoc these into the metadata so that the `qp.si/streaming-results-writer` implementations can adjust+ ;; their output based on the values: `:format-rows?` toggles value formatting, and `:csv-include-bom?` toggles the+ ;; leading UTF-8 BOM emitted by the CSV writer.+ (let [metadata (assoc metadata :format-rows? format-rows?, :csv-include-bom? csv-include-bom?)](if format-rows?(format-rows-xform (rff metadata) metadata)(rff metadata)))))diff --git a/src/metabase/query_processor/streaming/csv.clj b/src/metabase/query_processor/streaming/csv.cljindex 9ce61ff51b3..a817e72cd69 100644--- a/src/metabase/query_processor/streaming/csv.clj+++ b/src/metabase/query_processor/streaming/csv.clj@@ -9,6 +9,7 @@[metabase.query-processor.settings :as qp.settings][metabase.query-processor.streaming.common :as streaming.common][metabase.query-processor.streaming.interface :as qp.si]+ [metabase.util :as u][metabase.util.performance :refer [mapv]])(:import(java.io BufferedWriter OutputStream OutputStreamWriter)@@ -67,9 +68,12 @@pivot-data (volatile! nil)enable-pivoted-exports? (qp.settings/enable-pivoted-exports)](reify qp.si/StreamingResultsWriter- (begin! [_ {{:keys [ordered-cols results_timezone format-rows? pivot-export-options pivot?]- :or {format-rows? true- pivot? false}} :data} viz-settings]+ (begin! [_ {{:keys [ordered-cols results_timezone format-rows? pivot-export-options pivot? csv-include-bom?]+ :or {format-rows? true+ pivot? false+ csv-include-bom? true}} :data} viz-settings]+ (when csv-include-bom?+ (.write writer u/utf8-bom))(let [col-names (vec (streaming.common/column-titles ordered-cols viz-settings format-rows?))pivot-grouping-index (pivot.postprocess/pivot-grouping-index col-names)](conddiff --git a/src/metabase/util.cljc b/src/metabase/util.cljcindex e0f196666bc..5b67fe9a7c5 100644--- a/src/metabase/util.cljc+++ b/src/metabase/util.cljc@@ -228,6 +228,18 @@(str (upper-case-en (subs s 0 1))(lower-case-en (subs s 1))))))+(def ^String utf8-bom+ "The UTF-8 byte-order mark"+ "\ufeff")++(defn strip-bom+ "Strip a leading UTF-8 BOM from string `s`, if present. `clojure.data.csv` and many other parsers do not strip it+ automatically, so it can leak into the first cell. Returns `s` unchanged when there is no BOM (or `s` is nil)."+ ^String [^String s]+ (if (and s (str/starts-with? s utf8-bom))+ (subs s 1)+ s))+(defn truncate"Truncate a string to `n` characters."[s n]diff --git a/test/metabase/api/downloads_exports_test.clj b/test/metabase/api/downloads_exports_test.cljindex 58f31377e40..886c93355cb 100644--- a/test/metabase/api/downloads_exports_test.clj+++ b/test/metabase/api/downloads_exports_test.clj@@ -26,6 +26,7 @@[metabase.pulse.test-util :as pulse.test-util][metabase.query-processor.settings :as qp.settings][metabase.test :as mt]+ [metabase.util :as u][toucan2.core :as t2])(:import(org.apache.poi.ss.usermodel DataFormatter)))@@ -55,12 +56,17 @@(mapv name ks)(map #(mapv % ks) result))))+(defn- parse-csv+ "Parse CSV export bytes, stripping the leading UTF-8 BOM"+ [s]+ (csv/read-csv (cond-> s (string? s) u/strip-bom)))+(defn- process-results[export-format results](when (seq results)(case export-format:csv (cond-> results- (not (map? results)) csv/read-csv)+ (not (map? results)) parse-csv):xlsx (read-xlsx results):json (tabulate-maps results))))@@ -639,7 +645,7 @@(format "card/%d/query/csv" pivot-card-id){:format_rows true:pivot_results true})- csv/read-csv)]+ parse-csv)](is (= [["""Doohickey" "Doohickey""Gadget" "Gadget"@@ -712,7 +718,7 @@(format "card/%d/query/csv" pivot-card-id){:format_rows true:pivot_results true})- csv/read-csv)]+ parse-csv)](is (= [["Created At: Month" "Category" "Sum of Price"]["May, 2016" "Doohickey" "144.12"]["May, 2016" "Gadget" "81.58"]@@ -737,7 +743,7 @@(format "card/%d/query/csv" pivot-card-id){:format_rows false:pivot_results true})- csv/read-csv)]+ parse-csv)](is (= [["Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]["2185.89" "3019.2" "2834.88" "3109.31" "11149.28"]]result)))))))@@ -759,7 +765,7 @@(format "card/%d/query/csv" pivot-card-id){:format_rows true:pivot_results true})- csv/read-csv)]+ parse-csv)](is (= [["Category" "Created At: Year" "Sum of Price" "Count"]["Doohickey" "2016" "632.14" "13"]["Doohickey" "2017" "854.19" "17"]@@ -881,7 +887,7 @@(let [result (->> (mt/user-http-request :crowberto :post 200(format "card/%d/query/csv" pivot-card-id){:format_rows true})- csv/read-csv)]+ parse-csv)](is (= [["Category" "Created At: Month" "Sum of Price"]["Doohickey" "May, 2016" "144.12"]["Doohickey" "June, 2016" "82.92"]diff --git a/test/metabase/channel/email/result_attachment_test.clj b/test/metabase/channel/email/result_attachment_test.cljindex a79cb2b82df..3c385ac1dcf 100644--- a/test/metabase/channel/email/result_attachment_test.clj+++ b/test/metabase/channel/email/result_attachment_test.clj@@ -1,11 +1,13 @@(ns metabase.channel.email.result-attachment-test(:require+ [clojure.string :as str][clojure.test :refer :all][metabase.channel.email.result-attachment :as email.result-attachment][metabase.permissions.core :as perms]- [metabase.test :as mt])+ [metabase.test :as mt]+ [metabase.util :as u])(:import- (java.io IOException)))+ (java.io ByteArrayOutputStream IOException)))(set! *warn-on-reflection* true)@@ -22,6 +24,25 @@(with-create-temp-failure!(#'email.result-attachment/create-temp-file-or-throw! "txt")))))+(deftest csv-attachment-includes-utf8-bom-test+ (testing "CSV email attachments start with a UTF-8 BOM so they open correctly in Excel (#75875)"+ (let [results {:database_id (mt/id)+ :row_count 1+ :data {:cols [{:name "n" :base_type :type/Integer}]+ :rows [[1]]+ :viz-settings {}}}+ stream! (fn [opts]+ (let [os (ByteArrayOutputStream.)]+ (#'email.result-attachment/stream-api-results-to-export-format!+ os (merge {:export-format :csv} opts) results)+ (String. (.toByteArray os) "UTF-8")))]+ (testing "included by default"+ (is (str/starts-with? (stream! {}) u/utf8-bom)))+ (testing "included when explicitly requested"+ (is (str/starts-with? (stream! {:csv-include-bom? true}) u/utf8-bom)))+ (testing "omitted when explicitly disabled"+ (is (not (str/starts-with? (stream! {:csv-include-bom? false}) u/utf8-bom)))))))+(deftest result-attachment-uses-passed-creator-id-for-perm-check-test(testing "result-attachment checks the passed creator-id's download perms, not the card author's";; Regression test for GH #71696 (Linear GDGT-2143):diff --git a/test/metabase/embedding_rest/api/embed_test.clj b/test/metabase/embedding_rest/api/embed_test.cljindex 7311f856120..3c55015adb7 100644--- a/test/metabase/embedding_rest/api/embed_test.clj+++ b/test/metabase/embedding_rest/api/embed_test.clj@@ -131,7 +131,7 @@"/csv"(is (= "Count\n100\n"- actual))+ (u/strip-bom actual)))"/xlsx"(let [actual (->> (ByteArrayInputStream. actual)@@ -545,7 +545,7 @@(with-embedding-enabled-and-new-secret-key!(mt/with-temp [:model/Card card (card-with-date-field-filter)](is (= "count\n107\n"- (client/client :get 200 (str (card-query-url card "/csv") "&date=Q1-2014")))))))))+ (u/strip-bom (client/client :get 200 (str (card-query-url card "/csv") "&date=Q1-2014"))))))))))(deftest csv-forward-url-test(mt/test-helpers-set-global-values!@@ -555,7 +555,7 @@(binding [client/*url-prefix* ""](mt/with-temporary-setting-values [site-url (str "http://localhost:" (server.instance/server-port) client/*url-prefix*)](is (= "count\n107\n"- (client/real-client :get 200 (str "embed/question/" (card-token card) ".csv?date=Q1-2014"))))))))))+ (u/strip-bom (client/real-client :get 200 (str "embed/question/" (card-token card) ".csv?date=Q1-2014")))))))))));;; ---------------------------------------- GET /api/embed/dashboard/:token -----------------------------------------@@ -844,10 +844,10 @@{:name "PRICE" :fieldRef [:field (mt/id :venues :price) nil] :enabled true}]}}}](let [results (client/client :get 200 (str (dashcard-url dashcard) "/csv"))](is (= ["Name" "ID" "Category ID" "Price"]- (first (csv/read-csv results)))))+ (first (csv/read-csv (u/strip-bom results))))))(let [eid-results (client/client :get 200 (str (dashcard-url dashcard {} (dashcard->dash-eid dashcard)) "/csv"))](is (= ["Name" "ID" "Category ID" "Price"]- (first (csv/read-csv eid-results))))))))))+ (first (csv/read-csv (u/strip-bom eid-results)))))))))))(deftest generic-query-failed-exception-test(testing (str "...but if the card has an invalid query we should just get a generic \"query failed\" exception "diff --git a/test/metabase/public_sharing_rest/api_test.clj b/test/metabase/public_sharing_rest/api_test.cljindex b2d9e786f6a..97a223cf2fd 100644--- a/test/metabase/public_sharing_rest/api_test.clj+++ b/test/metabase/public_sharing_rest/api_test.clj@@ -306,7 +306,7 @@(client/client :get 200 (str "public/card/" uuid "/query/json?format_rows=true")))))(testing ":csv download response format"(is (= "Count\n100\n"- (client/client :get 200 (str "public/card/" uuid "/query/csv?format_rows=true"), :format :csv))))+ (u/strip-bom (client/client :get 200 (str "public/card/" uuid "/query/csv?format_rows=true"), :format :csv)))))(testing ":xlsx download response format"(is (= [{:col "Count"} {:col 100.0}](parse-xlsx-response@@ -520,11 +520,12 @@(mt/with-temporary-setting-values [enable-public-sharing true](mt/with-temp [:model/Card {uuid :public_uuid} (card-with-date-field-filter)](is (= "count\n107\n"- (client/client :get 200 (str "public/card/" uuid "/query/csv")- :parameters (json/encode [{:id "_DATE_"- :type :date/quarter-year- :target [:dimension [:template-tag :date]]- :value "Q1-2014"}])))))))+ (u/strip-bom+ (client/client :get 200 (str "public/card/" uuid "/query/csv")+ :parameters (json/encode [{:id "_DATE_"+ :type :date/quarter-year+ :target [:dimension [:template-tag :date]]+ :value "Q1-2014"}]))))))))(deftest make-sure-it-also-works-with-the-forwarded-url(mt/test-helpers-set-global-values!@@ -534,11 +535,12 @@(binding [client/*url-prefix* ""](mt/with-temporary-setting-values [site-url (str "http://localhost:" (server.instance/server-port) client/*url-prefix*)](is (= "count\n107\n"- (client/real-client :get 200 (str "public/question/" uuid ".csv")- :parameters (json/encode [{:id "_DATE_"- :type :date/quarter-year- :target [:dimension [:template-tag :date]]- :value "Q1-2014"}]))))))))))+ (u/strip-bom+ (client/real-client :get 200 (str "public/question/" uuid ".csv")+ :parameters (json/encode [{:id "_DATE_"+ :type :date/quarter-year+ :target [:dimension [:template-tag :date]]+ :value "Q1-2014"}])))))))))))(defn- card-with-trendline [](assoc (shared-obj)@@ -1660,18 +1662,18 @@:model/DashboardCard {dashcard-id :id} {:card_id card-id:dashboard_id dashboard-id}](testing "CSV export"- (is (str/starts-with? (client/client :post 200 (format "public/dashboard/%s/dashcard/%d/card/%d/csv"- uuid- dashcard-id- card-id))+ (is (str/starts-with? (u/strip-bom (client/client :post 200 (format "public/dashboard/%s/dashcard/%d/card/%d/csv"+ uuid+ dashcard-id+ card-id)))"ID,State,Latitude,Longitude\n")))(testing "urlencoded requests"- (is (str/starts-with? (client/client :post 200 (format "public/dashboard/%s/dashcard/%d/card/%d/csv"- uuid- dashcard-id- card-id)- {:request-options {:headers {"content-type" "application/x-www-form-urlencoded"}}}- {:format_rows true})+ (is (str/starts-with? (u/strip-bom (client/client :post 200 (format "public/dashboard/%s/dashcard/%d/card/%d/csv"+ uuid+ dashcard-id+ card-id)+ {:request-options {:headers {"content-type" "application/x-www-form-urlencoded"}}}+ {:format_rows true}))"ID,State,Latitude,Longitude\n")))(testing "Invalid id throws 404"(client/client :post 404 (format "public/dashboard/%s/dashcard/%d/card/%d/csv"diff --git a/test/metabase/pulse/pulse_integration_test.clj b/test/metabase/pulse/pulse_integration_test.cljindex 5540f275286..b3b5d06bf8f 100644--- a/test/metabase/pulse/pulse_integration_test.clj+++ b/test/metabase/pulse/pulse_integration_test.clj@@ -202,7 +202,7 @@(= :attachment type)(= "text/csv" content-type))[(strip-timestamp file-name)- (let [[h & r] (csv/read-csv (slurp content))]+ (let [[h & r] (csv/read-csv (u/strip-bom (slurp content)))](zipmap h (apply mapv vector r)))])))(into {}))))@@ -623,7 +623,7 @@(= :attachment type)(= "text/csv" content-type))[(strip-timestamp file-name)- (first (csv/read-csv (slurp content)))])))+ (first (csv/read-csv (u/strip-bom (slurp content))))])))(into {})))](testing "Renaming columns via viz settings is correctly applied to the CSV export"(is (= ["THE_ID" "ORDER TAX" "Total Amount" "Discount Applied ($)" "Amount Ordered" "Effective Tax Rate"]diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.cljindex becf430c577..ed7106ec5a3 100644--- a/test/metabase/queries_rest/api/card_test.clj+++ b/test/metabase/queries_rest/api/card_test.clj@@ -2328,7 +2328,7 @@(is (= ["COUNT(*)""75"](cond-> response- (string? response) str/split-lines))))))))+ (string? response) (-> u/strip-bom str/split-lines)))))))))(deftest csv-download-test-2(testing "with parameters"@@ -2339,7 +2339,7 @@(is (= ["COUNT(*)""8"](cond-> response- (string? response) str/split-lines))))))))+ (string? response) (-> u/strip-bom str/split-lines)))))))))(deftest json-download-test(testing "no parameters"@@ -3860,7 +3860,7 @@(let [q {:database (mt/id):type :native:native {:query "SELECT 2000 AS number, '2024-03-26'::DATE AS date;"}}- output-helper {:csv (fn [output] (->> output csv/read-csv))+ output-helper {:csv (fn [output] (->> output u/strip-bom csv/read-csv)):json (fn [[row]] [(map name (keys row)) (vals row)])}](mt/with-temp [:model/Card {card-id :id} {:dataset_query q:display :tablediff --git a/test/metabase/query_processor/streaming/csv_test.clj b/test/metabase/query_processor/streaming/csv_test.cljindex a97956397ab..3925572a731 100644--- a/test/metabase/query_processor/streaming/csv_test.clj+++ b/test/metabase/query_processor/streaming/csv_test.clj@@ -10,7 +10,8 @@[metabase.query-processor.streaming.interface :as qp.si][metabase.query-processor.test :as qp][metabase.test :as mt]- [metabase.test.data.dataset-definitions :as defs])+ [metabase.test.data.dataset-definitions :as defs]+ [metabase.util :as u])(:import(java.io BufferedOutputStream ByteArrayOutputStream)))@@ -131,6 +132,59 @@(let [bytea (.toByteArray bos)](rest (csv/read-csv (String. bytea))))))))+(deftest csv-export-includes-utf8-bom-test+ (testing "CSV exports start with a UTF-8 BOM so Excel renders non-ASCII chars like £ correctly"+ (driver/with-driver :h2+ (mt/with-metadata-provider (mt/id)+ (with-open [bos (ByteArrayOutputStream.)+ os (BufferedOutputStream. bos)]+ (let [results-writer (qp.si/streaming-results-writer :csv os)]+ (qp.si/begin! results-writer {:data {:ordered-cols [{:base_type :type/*}]}} {})+ (qp.si/write-row! results-writer ["£37.65"] 0 [] {})+ (qp.si/finish! results-writer {:row_count 1}))+ (is (= [(unchecked-byte 0xEF) (unchecked-byte 0xBB) (unchecked-byte 0xBF)]+ (take 3 (seq (.toByteArray bos))))))))))++(deftest csv-export-omits-utf8-bom-when-disabled-test+ (testing "CSV exports omit the UTF-8 BOM when `:csv-include-bom?` is false (#75875)"+ (driver/with-driver :h2+ (mt/with-metadata-provider (mt/id)+ (with-open [bos (ByteArrayOutputStream.)+ os (BufferedOutputStream. bos)]+ (let [results-writer (qp.si/streaming-results-writer :csv os)]+ (qp.si/begin! results-writer {:data {:ordered-cols [{:base_type :type/*}] :csv-include-bom? false}} {})+ (qp.si/write-row! results-writer ["£37.65"] 0 [] {})+ (qp.si/finish! results-writer {:row_count 1}))+ (let [bytes (seq (.toByteArray bos))]+ (is (not= [(unchecked-byte 0xEF) (unchecked-byte 0xBB) (unchecked-byte 0xBF)]+ (take 3 bytes))+ "should not start with a BOM")+ ;; first byte should be the start of the (empty) header line, not the BOM+ (is (not= (unchecked-byte 0xEF) (first bytes)))))))))++(deftest csv-export-bom-respects-csv-include-bom-param-test+ (testing "the csv_include_bom API param controls whether the downloaded CSV starts with a UTF-8 BOM (#75875)"+ (let [bom-prefix? (fn [^String s] (str/starts-with? s u/utf8-bom))+ download (fn [params]+ (mt/user-http-request :crowberto :post 200 "dataset/csv"+ (merge {:query (mt/mbql-query venues {:order-by [[:asc $id]], :limit 1})}+ params)))]+ (testing "omitted by default so external API consumers don't get an unexpected BOM"+ (is (false? (bom-prefix? (download {})))))+ (testing "included when explicitly true"+ (is (true? (bom-prefix? (download {:csv_include_bom true})))))+ (testing "omitted when false"+ (is (false? (bom-prefix? (download {:csv_include_bom false})))))+ (testing "the card query endpoint also honors the param"+ ;; allowing `with-temp` here since it tests against the REST API+ #_{:clj-kondo/ignore [:discouraged-var]}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query venues {:order-by [[:asc $id]], :limit 1})}]+ (let [download-card (fn [params]+ (mt/user-http-request :crowberto :post 200 (format "card/%d/query/csv" card-id) params))]+ (is (false? (bom-prefix? (download-card {}))))+ (is (true? (bom-prefix? (download-card {:csv_include_bom true}))))+ (is (false? (bom-prefix? (download-card {:csv_include_bom false}))))))))))+(deftest lazy-seq-realized-test(testing "Lazy seqs within rows are automatically realized during exports (#26261)"(let [row (first (csv-export [[(lazy-seq [1 2 3])]]))]diff --git a/test/metabase/query_processor/streaming/test_util.clj b/test/metabase/query_processor/streaming/test_util.cljindex a07391f90c8..ba37762956b 100644--- a/test/metabase/query_processor/streaming/test_util.clj+++ b/test/metabase/query_processor/streaming/test_util.clj@@ -33,7 +33,7 @@(defmethod parse-result* :csv[_ ^InputStream is _](with-open [reader (InputStreamReader. is)]- (doall (csv/read-csv reader))))+ (doall (csv/read-csv (u/strip-bom (slurp reader))))))(defmethod parse-result* :xlsx[_ ^InputStream is column-names]diff --git a/test/metabase/query_processor/streaming_test.clj b/test/metabase/query_processor/streaming_test.cljindex bfaa5737505..0c5b1934b20 100644--- a/test/metabase/query_processor/streaming_test.clj+++ b/test/metabase/query_processor/streaming_test.clj@@ -393,7 +393,7 @@[results](if (map? results)(throw (ex-info "Error in CSV export" results))- (csv/read-csv results)))+ (csv/read-csv (u/strip-bom results))))(deftest basic-export-test(do-test!diff --git a/test/metabase/util_test.cljc b/test/metabase/util_test.cljcindex f0519bdfb3d..2f1b33fc48d 100644--- a/test/metabase/util_test.cljc+++ b/test/metabase/util_test.cljc@@ -357,6 +357,17 @@"string" 3 "str""string" 0 ""))+(deftest ^:parallel strip-bom-test+ (are [s expected] (= expected+ (u/strip-bom s))+ nil nil+ "" ""+ "ID,Name" "ID,Name"+ (str u/utf8-bom "ID,Name") "ID,Name"+ (str u/utf8-bom) ""+ ;; only a *leading* BOM is stripped+ (str "ID" u/utf8-bom "Name") (str "ID" u/utf8-bom "Name")))+#?(:clj(deftest capitalize-en-turkish-test(mt/with-locale! "tr"