Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add configurable CSV field separator setting (#69827)

Edward Paget42aebbf13710
diff --git a/src/metabase/query_processor/settings.clj b/src/metabase/query_processor/settings.clj
index 5b76ab95df4..02c218144e9 100644
--- a/src/metabase/query_processor/settings.clj
+++ b/src/metabase/query_processor/settings.clj
@@ -65,3 +65,12 @@
(if (nil? limit)
limit
(max limit *minimum-download-row-limit*)))))
+
+(defsetting csv-field-separator
+ (deferred-tru "Character to use as field separator in CSV exports. Defaults to comma (,). Common alternatives include semicolon (;) for European locales and tab (\\t).")
+ :type :string
+ :default ","
+ :visibility :public
+ :export? true
+ :encryption :no
+ :audit :getter)
diff --git a/src/metabase/query_processor/streaming/csv.clj b/src/metabase/query_processor/streaming/csv.clj
index d8ef32c0f4a..241ed1f3ea3 100644
--- a/src/metabase/query_processor/streaming/csv.clj
+++ b/src/metabase/query_processor/streaming/csv.clj
@@ -28,17 +28,18 @@
:write-keepalive-newlines? false}))
(defn- write-csv
- "Custom implementation of `clojure.data.csv/write-csv` with a more efficient quote? predicate and no support for
- options (we don't use them)."
+ "Custom implementation of `clojure.data.csv/write-csv` with a more efficient quote? predicate and configurable
+ separator."
[writer data]
- (let [separator \,
+ (let [separator-str (qp.settings/csv-field-separator)
+ ^char separator (if (= separator-str "\\t") \tab (first separator-str)) ; Handle tab specially
quote \"
quote? (fn [^String s]
(let [n (.length s)]
(loop [i 0]
(if (>= i n) false
(let [ch (.charAt s (unchecked-int i))]
- (if (or (= ch \,) ;; separator
+ (if (or (= ch separator)
(= ch \") ;; quote
(= ch \return)
(= ch \newline))
diff --git a/test/metabase/query_processor/streaming/csv_test.clj b/test/metabase/query_processor/streaming/csv_test.clj
index 17c1e975e5d..2bfea85599f 100644
--- a/test/metabase/query_processor/streaming/csv_test.clj
+++ b/test/metabase/query_processor/streaming/csv_test.clj
@@ -146,3 +146,50 @@
results (qp/process-query (assoc (mt/native-query {:query query}) :middleware {:format-rows? false}))]
(is (= [["2020-06-03" "2020-06-03T23:41:23"]]
(csv-export (mt/rows results)))))))
+
+(deftest csv-field-separator-default-test
+ (testing "Default comma separator works correctly"
+ (let [rows [["ID" "Name" "Value"]
+ ["1" "Test" "100"]
+ ["2" "Data" "200"]]
+ result (csv-export rows)
+ header (first result)]
+ (is (= ["ID" "Name" "Value"] header)))))
+
+(deftest csv-field-separator-semicolon-test
+ (testing "Semicolon separator via HTTP API"
+ (mt/with-temporary-setting-values [csv-field-separator ";"]
+ (let [result (mt/user-http-request :crowberto :post 200 "dataset/csv"
+ {:query (mt/mbql-query venues {:order-by [[:asc $id]], :limit 2})
+ :format_rows true})
+ lines (str/split-lines result)
+ header (first lines)]
+ (is (str/includes? header ";")
+ "Header should use semicolon separator")
+ (is (not (re-find #"(?<![\"\\]),(?![\"\\])" header))
+ "Header should not contain unquoted commas as separators")))))
+
+(deftest csv-field-separator-tab-test
+ (testing "Tab separator via HTTP API"
+ (mt/with-temporary-setting-values [csv-field-separator "\\t"]
+ (let [result (mt/user-http-request :crowberto :post 200 "dataset/csv"
+ {:query (mt/mbql-query venues {:order-by [[:asc $id]], :limit 2})
+ :format_rows true})
+ lines (str/split-lines result)
+ header (first lines)]
+ (is (str/includes? header "\t")
+ "Header should use tab separator")))))
+
+(deftest csv-separator-quoting-test
+ (testing "Values containing the separator should be properly quoted"
+ (mt/with-temporary-setting-values [csv-field-separator ";"]
+ (let [result (mt/user-http-request
+ :crowberto :post 200 "dataset/csv"
+ {:query {:database (mt/id)
+ :type :native
+ :native {:query "SELECT 'Test' as name, 'Value;with;semicolons' as description"}}
+ :format_rows false})
+ lines (str/split-lines result)]
+ (is (= 2 (count lines)) "Should have header and one data row")
+ (is (str/includes? (second lines) "\"")
+ "Value containing separator should be quoted")))))
diff --git a/test_resources/serialization_baseline/settings.yaml b/test_resources/serialization_baseline/settings.yaml
index 92d434cdb58..55408373352 100644
--- a/test_resources/serialization_baseline/settings.yaml
+++ b/test_resources/serialization_baseline/settings.yaml
@@ -10,6 +10,7 @@ available-fonts: null
available-locales: null
available-timezones: null
breakout-bins-num: null
+csv-field-separator: nil
custom-formatting: null
custom-geojson: null
custom-geojson-enabled: null