Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add UTF-8 BOM to CSV downloads (#76028)

Nathan Voxland3c58b8226fba
diff --git a/e2e/test/scenarios/sharing/sharing-reproductions.cy.spec.js b/e2e/test/scenarios/sharing/sharing-reproductions.cy.spec.js
index 0652aa30763..61db08c7ea0 100644
--- a/e2e/test/scenarios/sharing/sharing-reproductions.cy.spec.js
+++ b/e2e/test/scenarios/sharing/sharing-reproductions.cy.spec.js
@@ -1057,11 +1057,13 @@ describe("issue 49525", { tags: "@external" }, () => {
url: `http://localhost:${WEB_PORT}/email/${email.id}/attachment/${csvAttachment.generatedFileName}`,
encoding: "utf8",
}).then((response) => {
- const csvContent = response.body;
- const rows = csvContent.split("\n");
+ // CSV exports begin with a UTF-8 BOM; strip it, and tolerate either
+ // \n or \r\n line endings, before asserting on the header row.
+ const csvContent = response.body.replace(/^\uFEFF/, "");
+ const rows = csvContent.split(/\r?\n/);
const headers = rows[0];
expect(headers).to.equal(
- "Created At: Year,Doohickey,Gadget,Gizmo,Widget,Row totals\r",
+ "Created At: Year,Doohickey,Gadget,Gizmo,Widget,Row totals",
);
});
});
diff --git a/frontend/src/metabase/redux/downloads.ts b/frontend/src/metabase/redux/downloads.ts
index 0e8c542611c..a1c1fb119d0 100644
--- a/frontend/src/metabase/redux/downloads.ts
+++ b/frontend/src/metabase/redux/downloads.ts
@@ -288,6 +288,7 @@ export const downloadDataset = createAsyncThunk(
type ExportParams = {
format_rows: boolean;
pivot_results: boolean;
+ csv_include_bom: boolean;
};
const getPublicDashcardParams = (
@@ -479,6 +480,7 @@ export const getDatasetParams = ({
const exportParams: ExportParams = {
format_rows: enableFormatting,
pivot_results: enablePivot,
+ csv_include_bom: true,
};
const { accessedVia, resourceType } = getDownloadedResourceType({
diff --git a/frontend/src/metabase/redux/downloads.unit.spec.ts b/frontend/src/metabase/redux/downloads.unit.spec.ts
index a21014de2b0..8867be8308a 100644
--- a/frontend/src/metabase/redux/downloads.unit.spec.ts
+++ b/frontend/src/metabase/redux/downloads.unit.spec.ts
@@ -149,6 +149,18 @@ describe("getDatasetParams - public question (uuid-based)", () => {
expect(url.get("format_rows")).toBe("true");
expect(url.get("pivot_results")).toBe("true");
});
+
+ it("requests the UTF-8 BOM so exports open correctly in Excel", () => {
+ const downloadParams = getDatasetParams({
+ type: "csv",
+ question,
+ result,
+ uuid: PUBLIC_UUID,
+ });
+
+ const url = new URLSearchParams(downloadParams.params);
+ expect(url.get("csv_include_bom")).toBe("true");
+ });
});
describe("readDownloadBlob", () => {
diff --git a/src/metabase/channel/email/result_attachment.clj b/src/metabase/channel/email/result_attachment.clj
index 5aee85e4bf4..be62d108aa3 100644
--- a/src/metabase/channel/email/result_attachment.clj
+++ b/src/metabase/channel/email/result_attachment.clj
@@ -31,10 +31,13 @@
Results are streamed synchronously. Caller is responsible for closing `os` when this call is complete."
[^OutputStream os :- (ms/InstanceOfClass OutputStream)
- {:keys [export-format format-rows? pivot?], :as _options} :- [:map
- [:export-format :keyword]
- [:format-rows? {:optional true} [:maybe :boolean]]
- [:pivot? {:optional true} [:maybe :boolean]]]
+ {:keys [export-format format-rows? pivot? csv-include-bom?]
+ :or {csv-include-bom? true}
+ :as _options} :- [:map
+ [:export-format :keyword]
+ [:format-rows? {:optional true} [:maybe :boolean]]
+ [:pivot? {:optional true} [:maybe :boolean]]
+ [:csv-include-bom? {:optional true} [:maybe :boolean]]]
{{:keys [rows]} :data, database-id :database_id, :as results} :- [:map
[:database_id ::lib.schema.id/database]]]
;; make sure Database/driver info is available for the streaming results writers -- they might need this in order to
@@ -51,6 +54,7 @@
(-> results
(assoc-in [:data :format-rows?] format-rows?)
(assoc-in [:data :pivot?] pivot?)
+ (assoc-in [:data :csv-include-bom?] csv-include-bom?)
(assoc-in [:data :ordered-cols] ordered-cols))
viz-settings')
(perf/reduce (fn [_ i row]
@@ -106,7 +110,7 @@
[(when-let [temp-file (and (:include_csv card)
(create-temp-file-or-throw! "csv"))]
(with-open [os (io/output-stream temp-file)]
- (stream-api-results-to-export-format! os {:export-format :csv :format-rows? format-rows :pivot? pivot-results} result))
+ (stream-api-results-to-export-format! os {:export-format :csv :format-rows? format-rows :pivot? pivot-results :csv-include-bom? true} result))
(create-result-attachment-map "csv" filename-prefix temp-file))
(when-let [temp-file (and (:include_xls card)
(create-temp-file-or-throw! "xlsx"))]
diff --git a/src/metabase/channel/render/preview.clj b/src/metabase/channel/render/preview.clj
index 3fbf85eae08..e654d902f2d 100644
--- a/src/metabase/channel/render/preview.clj
+++ b/src/metabase/channel/render/preview.clj
@@ -15,6 +15,7 @@
[metabase.channel.render.png :as png]
[metabase.channel.render.style :as style]
[metabase.notification.payload.core :as notification.payload]
+ [metabase.util :as u]
[metabase.util.markdown :as markdown]
[toucan2.core :as t2]))
@@ -31,7 +32,7 @@
(def ^:private csv-row-limit 10)
(defn- csv-to-html-table [csv-string]
- (let [rows (csv/read-csv csv-string)]
+ (let [rows (csv/read-csv (u/strip-bom csv-string))]
[:table {:style table-style}
(for [row (take (inc csv-row-limit) rows)] ;; inc row-limit to include the header and the expected # of rows
[:tr {:style table-style}
diff --git a/src/metabase/queries_rest/api/card.clj b/src/metabase/queries_rest/api/card.clj
index e06159a9d5a..c8c0c543cef 100644
--- a/src/metabase/queries_rest/api/card.clj
+++ b/src/metabase/queries_rest/api/card.clj
@@ -918,17 +918,18 @@
(api.macros/defendpoint :post "/:card-id/query/:export-format"
"Run the query associated with a Card, and return its results as a file in the specified format.
- `parameters`, `pivot-results?` and `format-rows?` should be passed as application/x-www-form-urlencoded form content
+ `csv_include_bom`, `parameters`, `pivot-results?` and `format-rows?` should be passed as application/x-www-form-urlencoded form content
or json in the body. This is because this endpoint is normally used to power 'Download Results' buttons that use
HTML `form` actions)."
[{:keys [card-id export-format]} :- [:map
[:card-id ms/PositiveInt]
[:export-format ::qp.schema/export-format]]
_query-params
- {:keys [parameters]
- pivot-results? :pivot_results
- format-rows? :format_rows
- :as _body}
+ {:keys [parameters]
+ pivot-results? :pivot_results
+ format-rows? :format_rows
+ csv-include-bom? :csv_include_bom
+ :as _body}
:- [:map
[:parameters {:optional true} [:maybe
;; support JSON-encoded parameters for backwards compatibility when with this
@@ -942,7 +943,8 @@
;; it breaks existing tests
[:sequential [:map-of :keyword :any]]]]
[:format_rows {:default false} ms/BooleanValue]
- [:pivot_results {:default false} ms/BooleanValue]]]
+ [:pivot_results {:default false} ms/BooleanValue]
+ [:csv_include_bom {:default false} ms/BooleanValue]]]
(qp.card/process-query-for-card
(api/check-404 (t2/select-one :model/Card card-id)) export-format
:parameters parameters
@@ -953,6 +955,7 @@
:ignore-cached-results? true
:format-rows? format-rows?
:pivot? pivot-results?
+ :csv-include-bom? (if (some? csv-include-bom?) csv-include-bom? false)
:js-int-to-string? false}))
;;; ----------------------------------------------- Sharing is Caring ------------------------------------------------
diff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.clj
index 7071f550a94..50a62a1b451 100644
--- a/src/metabase/query_processor/api.clj
+++ b/src/metabase/query_processor/api.clj
@@ -134,6 +134,7 @@
{{:keys [was-pivot] :as query} :query
format-rows :format_rows
pivot-results :pivot_results
+ csv-include-bom :csv_include_bom
visualization-settings :visualization_settings}
;; Support JSON-encoded query and viz settings for backwards compatibility for when downloads used to be triggered by
;; `<form>` submissions... see https://metaboat.slack.com/archives/C010L1Z4F9S/p1738003606875659
@@ -147,7 +148,8 @@
(cond-> x
(string? x) (json/decode viz-setting-key-fn)))}]]
[:format_rows {:default false} ms/BooleanValue]
- [:pivot_results {:default false} ms/BooleanValue]]]
+ [:pivot_results {:default false} ms/BooleanValue]
+ [:csv_include_bom {:default false} ms/BooleanValue]]]
(let [viz-settings (-> visualization-settings
mi/normalize-visualization-settings
mb.viz/norm->db)
@@ -158,6 +160,7 @@
(dissoc :add-default-userland-constraints? :js-int-to-string?)
(assoc :format-rows? (or format-rows false)
:pivot? (or pivot-results false)
+ :csv-include-bom? (if (some? csv-include-bom) csv-include-bom false)
:process-viz-settings? true
:skip-results-metadata? true))))]
(run-streaming-query
diff --git a/src/metabase/query_processor/middleware/format_rows.clj b/src/metabase/query_processor/middleware/format_rows.clj
index fa295a119f3..06791f8873c 100644
--- a/src/metabase/query_processor/middleware/format_rows.clj
+++ b/src/metabase/query_processor/middleware/format_rows.clj
@@ -98,11 +98,12 @@
(defn format-rows
"Format individual query result values as needed. Ex: format temporal values as ISO-8601 strings w/ timezone offset."
- [{{:keys [format-rows?] :or {format-rows? true}} :middleware, :as _query} rff]
+ [{{:keys [format-rows? csv-include-bom?] :or {format-rows? true, csv-include-bom? true}} :middleware, :as _query} rff]
(fn format-rows-rff* [metadata]
- ;; always assoc `:format-rows?` into the metadata so that
- ;; the `qp.si/streaming-results-writer` implmementations can apply/not-apply formatting based on the key's value
- (let [metadata (assoc metadata :format-rows? format-rows?)]
+ ;; always assoc these into the metadata so that the `qp.si/streaming-results-writer` implementations can adjust
+ ;; their output based on the values: `:format-rows?` toggles value formatting, and `:csv-include-bom?` toggles the
+ ;; leading UTF-8 BOM emitted by the CSV writer.
+ (let [metadata (assoc metadata :format-rows? format-rows?, :csv-include-bom? csv-include-bom?)]
(if format-rows?
(format-rows-xform (rff metadata) metadata)
(rff metadata)))))
diff --git a/src/metabase/query_processor/streaming/csv.clj b/src/metabase/query_processor/streaming/csv.clj
index 9ce61ff51b3..a817e72cd69 100644
--- a/src/metabase/query_processor/streaming/csv.clj
+++ b/src/metabase/query_processor/streaming/csv.clj
@@ -9,6 +9,7 @@
[metabase.query-processor.settings :as qp.settings]
[metabase.query-processor.streaming.common :as streaming.common]
[metabase.query-processor.streaming.interface :as qp.si]
+ [metabase.util :as u]
[metabase.util.performance :refer [mapv]])
(:import
(java.io BufferedWriter OutputStream OutputStreamWriter)
@@ -67,9 +68,12 @@
pivot-data (volatile! nil)
enable-pivoted-exports? (qp.settings/enable-pivoted-exports)]
(reify qp.si/StreamingResultsWriter
- (begin! [_ {{:keys [ordered-cols results_timezone format-rows? pivot-export-options pivot?]
- :or {format-rows? true
- pivot? false}} :data} viz-settings]
+ (begin! [_ {{:keys [ordered-cols results_timezone format-rows? pivot-export-options pivot? csv-include-bom?]
+ :or {format-rows? true
+ pivot? false
+ csv-include-bom? true}} :data} viz-settings]
+ (when csv-include-bom?
+ (.write writer u/utf8-bom))
(let [col-names (vec (streaming.common/column-titles ordered-cols viz-settings format-rows?))
pivot-grouping-index (pivot.postprocess/pivot-grouping-index col-names)]
(cond
diff --git a/src/metabase/util.cljc b/src/metabase/util.cljc
index e0f196666bc..5b67fe9a7c5 100644
--- a/src/metabase/util.cljc
+++ b/src/metabase/util.cljc
@@ -228,6 +228,18 @@
(str (upper-case-en (subs s 0 1))
(lower-case-en (subs s 1))))))
+(def ^String utf8-bom
+ "The UTF-8 byte-order mark"
+ "\ufeff")
+
+(defn strip-bom
+ "Strip a leading UTF-8 BOM from string `s`, if present. `clojure.data.csv` and many other parsers do not strip it
+ automatically, so it can leak into the first cell. Returns `s` unchanged when there is no BOM (or `s` is nil)."
+ ^String [^String s]
+ (if (and s (str/starts-with? s utf8-bom))
+ (subs s 1)
+ s))
+
(defn truncate
"Truncate a string to `n` characters."
[s n]
diff --git a/test/metabase/api/downloads_exports_test.clj b/test/metabase/api/downloads_exports_test.clj
index 58f31377e40..886c93355cb 100644
--- a/test/metabase/api/downloads_exports_test.clj
+++ b/test/metabase/api/downloads_exports_test.clj
@@ -26,6 +26,7 @@
[metabase.pulse.test-util :as pulse.test-util]
[metabase.query-processor.settings :as qp.settings]
[metabase.test :as mt]
+ [metabase.util :as u]
[toucan2.core :as t2])
(:import
(org.apache.poi.ss.usermodel DataFormatter)))
@@ -55,12 +56,17 @@
(mapv name ks)
(map #(mapv % ks) result))))
+(defn- parse-csv
+ "Parse CSV export bytes, stripping the leading UTF-8 BOM"
+ [s]
+ (csv/read-csv (cond-> s (string? s) u/strip-bom)))
+
(defn- process-results
[export-format results]
(when (seq results)
(case export-format
:csv (cond-> results
- (not (map? results)) csv/read-csv)
+ (not (map? results)) parse-csv)
:xlsx (read-xlsx results)
:json (tabulate-maps results))))
@@ -639,7 +645,7 @@
(format "card/%d/query/csv" pivot-card-id)
{:format_rows true
:pivot_results true})
- csv/read-csv)]
+ parse-csv)]
(is (= [[""
"Doohickey" "Doohickey"
"Gadget" "Gadget"
@@ -712,7 +718,7 @@
(format "card/%d/query/csv" pivot-card-id)
{:format_rows true
:pivot_results true})
- csv/read-csv)]
+ parse-csv)]
(is (= [["Created At: Month" "Category" "Sum of Price"]
["May, 2016" "Doohickey" "144.12"]
["May, 2016" "Gadget" "81.58"]
@@ -737,7 +743,7 @@
(format "card/%d/query/csv" pivot-card-id)
{:format_rows false
:pivot_results true})
- csv/read-csv)]
+ parse-csv)]
(is (= [["Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]
["2185.89" "3019.2" "2834.88" "3109.31" "11149.28"]]
result)))))))
@@ -759,7 +765,7 @@
(format "card/%d/query/csv" pivot-card-id)
{:format_rows true
:pivot_results true})
- csv/read-csv)]
+ parse-csv)]
(is (= [["Category" "Created At: Year" "Sum of Price" "Count"]
["Doohickey" "2016" "632.14" "13"]
["Doohickey" "2017" "854.19" "17"]
@@ -881,7 +887,7 @@
(let [result (->> (mt/user-http-request :crowberto :post 200
(format "card/%d/query/csv" pivot-card-id)
{:format_rows true})
- csv/read-csv)]
+ parse-csv)]
(is (= [["Category" "Created At: Month" "Sum of Price"]
["Doohickey" "May, 2016" "144.12"]
["Doohickey" "June, 2016" "82.92"]
diff --git a/test/metabase/channel/email/result_attachment_test.clj b/test/metabase/channel/email/result_attachment_test.clj
index a79cb2b82df..3c385ac1dcf 100644
--- a/test/metabase/channel/email/result_attachment_test.clj
+++ b/test/metabase/channel/email/result_attachment_test.clj
@@ -1,11 +1,13 @@
(ns metabase.channel.email.result-attachment-test
(:require
+ [clojure.string :as str]
[clojure.test :refer :all]
[metabase.channel.email.result-attachment :as email.result-attachment]
[metabase.permissions.core :as perms]
- [metabase.test :as mt])
+ [metabase.test :as mt]
+ [metabase.util :as u])
(:import
- (java.io IOException)))
+ (java.io ByteArrayOutputStream IOException)))
(set! *warn-on-reflection* true)
@@ -22,6 +24,25 @@
(with-create-temp-failure!
(#'email.result-attachment/create-temp-file-or-throw! "txt")))))
+(deftest csv-attachment-includes-utf8-bom-test
+ (testing "CSV email attachments start with a UTF-8 BOM so they open correctly in Excel (#75875)"
+ (let [results {:database_id (mt/id)
+ :row_count 1
+ :data {:cols [{:name "n" :base_type :type/Integer}]
+ :rows [[1]]
+ :viz-settings {}}}
+ stream! (fn [opts]
+ (let [os (ByteArrayOutputStream.)]
+ (#'email.result-attachment/stream-api-results-to-export-format!
+ os (merge {:export-format :csv} opts) results)
+ (String. (.toByteArray os) "UTF-8")))]
+ (testing "included by default"
+ (is (str/starts-with? (stream! {}) u/utf8-bom)))
+ (testing "included when explicitly requested"
+ (is (str/starts-with? (stream! {:csv-include-bom? true}) u/utf8-bom)))
+ (testing "omitted when explicitly disabled"
+ (is (not (str/starts-with? (stream! {:csv-include-bom? false}) u/utf8-bom)))))))
+
(deftest result-attachment-uses-passed-creator-id-for-perm-check-test
(testing "result-attachment checks the passed creator-id's download perms, not the card author's"
;; Regression test for GH #71696 (Linear GDGT-2143):
diff --git a/test/metabase/embedding_rest/api/embed_test.clj b/test/metabase/embedding_rest/api/embed_test.clj
index 7311f856120..3c55015adb7 100644
--- a/test/metabase/embedding_rest/api/embed_test.clj
+++ b/test/metabase/embedding_rest/api/embed_test.clj
@@ -131,7 +131,7 @@
"/csv"
(is (= "Count\n100\n"
- actual))
+ (u/strip-bom actual)))
"/xlsx"
(let [actual (->> (ByteArrayInputStream. actual)
@@ -545,7 +545,7 @@
(with-embedding-enabled-and-new-secret-key!
(mt/with-temp [:model/Card card (card-with-date-field-filter)]
(is (= "count\n107\n"
- (client/client :get 200 (str (card-query-url card "/csv") "&date=Q1-2014")))))))))
+ (u/strip-bom (client/client :get 200 (str (card-query-url card "/csv") "&date=Q1-2014"))))))))))
(deftest csv-forward-url-test
(mt/test-helpers-set-global-values!
@@ -555,7 +555,7 @@
(binding [client/*url-prefix* ""]
(mt/with-temporary-setting-values [site-url (str "http://localhost:" (server.instance/server-port) client/*url-prefix*)]
(is (= "count\n107\n"
- (client/real-client :get 200 (str "embed/question/" (card-token card) ".csv?date=Q1-2014"))))))))))
+ (u/strip-bom (client/real-client :get 200 (str "embed/question/" (card-token card) ".csv?date=Q1-2014")))))))))))
;;; ---------------------------------------- GET /api/embed/dashboard/:token -----------------------------------------
@@ -844,10 +844,10 @@
{:name "PRICE" :fieldRef [:field (mt/id :venues :price) nil] :enabled true}]}}}]
(let [results (client/client :get 200 (str (dashcard-url dashcard) "/csv"))]
(is (= ["Name" "ID" "Category ID" "Price"]
- (first (csv/read-csv results)))))
+ (first (csv/read-csv (u/strip-bom results))))))
(let [eid-results (client/client :get 200 (str (dashcard-url dashcard {} (dashcard->dash-eid dashcard)) "/csv"))]
(is (= ["Name" "ID" "Category ID" "Price"]
- (first (csv/read-csv eid-results))))))))))
+ (first (csv/read-csv (u/strip-bom eid-results)))))))))))
(deftest generic-query-failed-exception-test
(testing (str "...but if the card has an invalid query we should just get a generic \"query failed\" exception "
diff --git a/test/metabase/public_sharing_rest/api_test.clj b/test/metabase/public_sharing_rest/api_test.clj
index b2d9e786f6a..97a223cf2fd 100644
--- a/test/metabase/public_sharing_rest/api_test.clj
+++ b/test/metabase/public_sharing_rest/api_test.clj
@@ -306,7 +306,7 @@
(client/client :get 200 (str "public/card/" uuid "/query/json?format_rows=true")))))
(testing ":csv download response format"
(is (= "Count\n100\n"
- (client/client :get 200 (str "public/card/" uuid "/query/csv?format_rows=true"), :format :csv))))
+ (u/strip-bom (client/client :get 200 (str "public/card/" uuid "/query/csv?format_rows=true"), :format :csv)))))
(testing ":xlsx download response format"
(is (= [{:col "Count"} {:col 100.0}]
(parse-xlsx-response
@@ -520,11 +520,12 @@
(mt/with-temporary-setting-values [enable-public-sharing true]
(mt/with-temp [:model/Card {uuid :public_uuid} (card-with-date-field-filter)]
(is (= "count\n107\n"
- (client/client :get 200 (str "public/card/" uuid "/query/csv")
- :parameters (json/encode [{:id "_DATE_"
- :type :date/quarter-year
- :target [:dimension [:template-tag :date]]
- :value "Q1-2014"}])))))))
+ (u/strip-bom
+ (client/client :get 200 (str "public/card/" uuid "/query/csv")
+ :parameters (json/encode [{:id "_DATE_"
+ :type :date/quarter-year
+ :target [:dimension [:template-tag :date]]
+ :value "Q1-2014"}]))))))))
(deftest make-sure-it-also-works-with-the-forwarded-url
(mt/test-helpers-set-global-values!
@@ -534,11 +535,12 @@
(binding [client/*url-prefix* ""]
(mt/with-temporary-setting-values [site-url (str "http://localhost:" (server.instance/server-port) client/*url-prefix*)]
(is (= "count\n107\n"
- (client/real-client :get 200 (str "public/question/" uuid ".csv")
- :parameters (json/encode [{:id "_DATE_"
- :type :date/quarter-year
- :target [:dimension [:template-tag :date]]
- :value "Q1-2014"}]))))))))))
+ (u/strip-bom
+ (client/real-client :get 200 (str "public/question/" uuid ".csv")
+ :parameters (json/encode [{:id "_DATE_"
+ :type :date/quarter-year
+ :target [:dimension [:template-tag :date]]
+ :value "Q1-2014"}])))))))))))
(defn- card-with-trendline []
(assoc (shared-obj)
@@ -1660,18 +1662,18 @@
:model/DashboardCard {dashcard-id :id} {:card_id card-id
:dashboard_id dashboard-id}]
(testing "CSV export"
- (is (str/starts-with? (client/client :post 200 (format "public/dashboard/%s/dashcard/%d/card/%d/csv"
- uuid
- dashcard-id
- card-id))
+ (is (str/starts-with? (u/strip-bom (client/client :post 200 (format "public/dashboard/%s/dashcard/%d/card/%d/csv"
+ uuid
+ dashcard-id
+ card-id)))
"ID,State,Latitude,Longitude\n")))
(testing "urlencoded requests"
- (is (str/starts-with? (client/client :post 200 (format "public/dashboard/%s/dashcard/%d/card/%d/csv"
- uuid
- dashcard-id
- card-id)
- {:request-options {:headers {"content-type" "application/x-www-form-urlencoded"}}}
- {:format_rows true})
+ (is (str/starts-with? (u/strip-bom (client/client :post 200 (format "public/dashboard/%s/dashcard/%d/card/%d/csv"
+ uuid
+ dashcard-id
+ card-id)
+ {:request-options {:headers {"content-type" "application/x-www-form-urlencoded"}}}
+ {:format_rows true}))
"ID,State,Latitude,Longitude\n")))
(testing "Invalid id throws 404"
(client/client :post 404 (format "public/dashboard/%s/dashcard/%d/card/%d/csv"
diff --git a/test/metabase/pulse/pulse_integration_test.clj b/test/metabase/pulse/pulse_integration_test.clj
index 5540f275286..b3b5d06bf8f 100644
--- a/test/metabase/pulse/pulse_integration_test.clj
+++ b/test/metabase/pulse/pulse_integration_test.clj
@@ -202,7 +202,7 @@
(= :attachment type)
(= "text/csv" content-type))
[(strip-timestamp file-name)
- (let [[h & r] (csv/read-csv (slurp content))]
+ (let [[h & r] (csv/read-csv (u/strip-bom (slurp content)))]
(zipmap h (apply mapv vector r)))])))
(into {}))))
@@ -623,7 +623,7 @@
(= :attachment type)
(= "text/csv" content-type))
[(strip-timestamp file-name)
- (first (csv/read-csv (slurp content)))])))
+ (first (csv/read-csv (u/strip-bom (slurp content))))])))
(into {})))]
(testing "Renaming columns via viz settings is correctly applied to the CSV export"
(is (= ["THE_ID" "ORDER TAX" "Total Amount" "Discount Applied ($)" "Amount Ordered" "Effective Tax Rate"]
diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.clj
index becf430c577..ed7106ec5a3 100644
--- a/test/metabase/queries_rest/api/card_test.clj
+++ b/test/metabase/queries_rest/api/card_test.clj
@@ -2328,7 +2328,7 @@
(is (= ["COUNT(*)"
"75"]
(cond-> response
- (string? response) str/split-lines))))))))
+ (string? response) (-> u/strip-bom str/split-lines)))))))))
(deftest csv-download-test-2
(testing "with parameters"
@@ -2339,7 +2339,7 @@
(is (= ["COUNT(*)"
"8"]
(cond-> response
- (string? response) str/split-lines))))))))
+ (string? response) (-> u/strip-bom str/split-lines)))))))))
(deftest json-download-test
(testing "no parameters"
@@ -3860,7 +3860,7 @@
(let [q {:database (mt/id)
:type :native
:native {:query "SELECT 2000 AS number, '2024-03-26'::DATE AS date;"}}
- output-helper {:csv (fn [output] (->> output csv/read-csv))
+ output-helper {:csv (fn [output] (->> output u/strip-bom csv/read-csv))
:json (fn [[row]] [(map name (keys row)) (vals row)])}]
(mt/with-temp [:model/Card {card-id :id} {:dataset_query q
:display :table
diff --git a/test/metabase/query_processor/streaming/csv_test.clj b/test/metabase/query_processor/streaming/csv_test.clj
index a97956397ab..3925572a731 100644
--- a/test/metabase/query_processor/streaming/csv_test.clj
+++ b/test/metabase/query_processor/streaming/csv_test.clj
@@ -10,7 +10,8 @@
[metabase.query-processor.streaming.interface :as qp.si]
[metabase.query-processor.test :as qp]
[metabase.test :as mt]
- [metabase.test.data.dataset-definitions :as defs])
+ [metabase.test.data.dataset-definitions :as defs]
+ [metabase.util :as u])
(:import
(java.io BufferedOutputStream ByteArrayOutputStream)))
@@ -131,6 +132,59 @@
(let [bytea (.toByteArray bos)]
(rest (csv/read-csv (String. bytea))))))))
+(deftest csv-export-includes-utf8-bom-test
+ (testing "CSV exports start with a UTF-8 BOM so Excel renders non-ASCII chars like £ correctly"
+ (driver/with-driver :h2
+ (mt/with-metadata-provider (mt/id)
+ (with-open [bos (ByteArrayOutputStream.)
+ os (BufferedOutputStream. bos)]
+ (let [results-writer (qp.si/streaming-results-writer :csv os)]
+ (qp.si/begin! results-writer {:data {:ordered-cols [{:base_type :type/*}]}} {})
+ (qp.si/write-row! results-writer ["£37.65"] 0 [] {})
+ (qp.si/finish! results-writer {:row_count 1}))
+ (is (= [(unchecked-byte 0xEF) (unchecked-byte 0xBB) (unchecked-byte 0xBF)]
+ (take 3 (seq (.toByteArray bos))))))))))
+
+(deftest csv-export-omits-utf8-bom-when-disabled-test
+ (testing "CSV exports omit the UTF-8 BOM when `:csv-include-bom?` is false (#75875)"
+ (driver/with-driver :h2
+ (mt/with-metadata-provider (mt/id)
+ (with-open [bos (ByteArrayOutputStream.)
+ os (BufferedOutputStream. bos)]
+ (let [results-writer (qp.si/streaming-results-writer :csv os)]
+ (qp.si/begin! results-writer {:data {:ordered-cols [{:base_type :type/*}] :csv-include-bom? false}} {})
+ (qp.si/write-row! results-writer ["£37.65"] 0 [] {})
+ (qp.si/finish! results-writer {:row_count 1}))
+ (let [bytes (seq (.toByteArray bos))]
+ (is (not= [(unchecked-byte 0xEF) (unchecked-byte 0xBB) (unchecked-byte 0xBF)]
+ (take 3 bytes))
+ "should not start with a BOM")
+ ;; first byte should be the start of the (empty) header line, not the BOM
+ (is (not= (unchecked-byte 0xEF) (first bytes)))))))))
+
+(deftest csv-export-bom-respects-csv-include-bom-param-test
+ (testing "the csv_include_bom API param controls whether the downloaded CSV starts with a UTF-8 BOM (#75875)"
+ (let [bom-prefix? (fn [^String s] (str/starts-with? s u/utf8-bom))
+ download (fn [params]
+ (mt/user-http-request :crowberto :post 200 "dataset/csv"
+ (merge {:query (mt/mbql-query venues {:order-by [[:asc $id]], :limit 1})}
+ params)))]
+ (testing "omitted by default so external API consumers don't get an unexpected BOM"
+ (is (false? (bom-prefix? (download {})))))
+ (testing "included when explicitly true"
+ (is (true? (bom-prefix? (download {:csv_include_bom true})))))
+ (testing "omitted when false"
+ (is (false? (bom-prefix? (download {:csv_include_bom false})))))
+ (testing "the card query endpoint also honors the param"
+ ;; allowing `with-temp` here since it tests against the REST API
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query venues {:order-by [[:asc $id]], :limit 1})}]
+ (let [download-card (fn [params]
+ (mt/user-http-request :crowberto :post 200 (format "card/%d/query/csv" card-id) params))]
+ (is (false? (bom-prefix? (download-card {}))))
+ (is (true? (bom-prefix? (download-card {:csv_include_bom true}))))
+ (is (false? (bom-prefix? (download-card {:csv_include_bom false}))))))))))
+
(deftest lazy-seq-realized-test
(testing "Lazy seqs within rows are automatically realized during exports (#26261)"
(let [row (first (csv-export [[(lazy-seq [1 2 3])]]))]
diff --git a/test/metabase/query_processor/streaming/test_util.clj b/test/metabase/query_processor/streaming/test_util.clj
index a07391f90c8..ba37762956b 100644
--- a/test/metabase/query_processor/streaming/test_util.clj
+++ b/test/metabase/query_processor/streaming/test_util.clj
@@ -33,7 +33,7 @@
(defmethod parse-result* :csv
[_ ^InputStream is _]
(with-open [reader (InputStreamReader. is)]
- (doall (csv/read-csv reader))))
+ (doall (csv/read-csv (u/strip-bom (slurp reader))))))
(defmethod parse-result* :xlsx
[_ ^InputStream is column-names]
diff --git a/test/metabase/query_processor/streaming_test.clj b/test/metabase/query_processor/streaming_test.clj
index bfaa5737505..0c5b1934b20 100644
--- a/test/metabase/query_processor/streaming_test.clj
+++ b/test/metabase/query_processor/streaming_test.clj
@@ -393,7 +393,7 @@
[results]
(if (map? results)
(throw (ex-info "Error in CSV export" results))
- (csv/read-csv results)))
+ (csv/read-csv (u/strip-bom results))))
(deftest basic-export-test
(do-test!
diff --git a/test/metabase/util_test.cljc b/test/metabase/util_test.cljc
index f0519bdfb3d..2f1b33fc48d 100644
--- a/test/metabase/util_test.cljc
+++ b/test/metabase/util_test.cljc
@@ -357,6 +357,17 @@
"string" 3 "str"
"string" 0 ""))
+(deftest ^:parallel strip-bom-test
+ (are [s expected] (= expected
+ (u/strip-bom s))
+ nil nil
+ "" ""
+ "ID,Name" "ID,Name"
+ (str u/utf8-bom "ID,Name") "ID,Name"
+ (str u/utf8-bom) ""
+ ;; only a *leading* BOM is stripped
+ (str "ID" u/utf8-bom "Name") (str "ID" u/utf8-bom "Name")))
+
#?(:clj
(deftest capitalize-en-turkish-test
(mt/with-locale! "tr"