The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add configurable CSV field separator setting (#69827)
Edward Paget42aebbf13710
diff --git a/src/metabase/query_processor/settings.clj b/src/metabase/query_processor/settings.cljindex 5b76ab95df4..02c218144e9 100644--- a/src/metabase/query_processor/settings.clj+++ b/src/metabase/query_processor/settings.clj@@ -65,3 +65,12 @@(if (nil? limit)limit(max limit *minimum-download-row-limit*)))))++(defsetting csv-field-separator+ (deferred-tru "Character to use as field separator in CSV exports. Defaults to comma (,). Common alternatives include semicolon (;) for European locales and tab (\\t).")+ :type :string+ :default ","+ :visibility :public+ :export? true+ :encryption :no+ :audit :getter)diff --git a/src/metabase/query_processor/streaming/csv.clj b/src/metabase/query_processor/streaming/csv.cljindex d8ef32c0f4a..241ed1f3ea3 100644--- a/src/metabase/query_processor/streaming/csv.clj+++ b/src/metabase/query_processor/streaming/csv.clj@@ -28,17 +28,18 @@:write-keepalive-newlines? false}))(defn- write-csv- "Custom implementation of `clojure.data.csv/write-csv` with a more efficient quote? predicate and no support for- options (we don't use them)."+ "Custom implementation of `clojure.data.csv/write-csv` with a more efficient quote? predicate and configurable+ separator."[writer data]- (let [separator \,+ (let [separator-str (qp.settings/csv-field-separator)+ ^char separator (if (= separator-str "\\t") \tab (first separator-str)) ; Handle tab speciallyquote \"quote? (fn [^String s](let [n (.length s)](loop [i 0](if (>= i n) false(let [ch (.charAt s (unchecked-int i))]- (if (or (= ch \,) ;; separator+ (if (or (= ch separator)(= ch \") ;; quote(= ch \return)(= ch \newline))diff --git a/test/metabase/query_processor/streaming/csv_test.clj b/test/metabase/query_processor/streaming/csv_test.cljindex 17c1e975e5d..2bfea85599f 100644--- a/test/metabase/query_processor/streaming/csv_test.clj+++ b/test/metabase/query_processor/streaming/csv_test.clj@@ -146,3 +146,50 @@results (qp/process-query (assoc (mt/native-query {:query query}) :middleware {:format-rows? false}))](is (= [["2020-06-03" "2020-06-03T23:41:23"]](csv-export (mt/rows results)))))))++(deftest csv-field-separator-default-test+ (testing "Default comma separator works correctly"+ (let [rows [["ID" "Name" "Value"]+ ["1" "Test" "100"]+ ["2" "Data" "200"]]+ result (csv-export rows)+ header (first result)]+ (is (= ["ID" "Name" "Value"] header)))))++(deftest csv-field-separator-semicolon-test+ (testing "Semicolon separator via HTTP API"+ (mt/with-temporary-setting-values [csv-field-separator ";"]+ (let [result (mt/user-http-request :crowberto :post 200 "dataset/csv"+ {:query (mt/mbql-query venues {:order-by [[:asc $id]], :limit 2})+ :format_rows true})+ lines (str/split-lines result)+ header (first lines)]+ (is (str/includes? header ";")+ "Header should use semicolon separator")+ (is (not (re-find #"(?<![\"\\]),(?![\"\\])" header))+ "Header should not contain unquoted commas as separators")))))++(deftest csv-field-separator-tab-test+ (testing "Tab separator via HTTP API"+ (mt/with-temporary-setting-values [csv-field-separator "\\t"]+ (let [result (mt/user-http-request :crowberto :post 200 "dataset/csv"+ {:query (mt/mbql-query venues {:order-by [[:asc $id]], :limit 2})+ :format_rows true})+ lines (str/split-lines result)+ header (first lines)]+ (is (str/includes? header "\t")+ "Header should use tab separator")))))++(deftest csv-separator-quoting-test+ (testing "Values containing the separator should be properly quoted"+ (mt/with-temporary-setting-values [csv-field-separator ";"]+ (let [result (mt/user-http-request+ :crowberto :post 200 "dataset/csv"+ {:query {:database (mt/id)+ :type :native+ :native {:query "SELECT 'Test' as name, 'Value;with;semicolons' as description"}}+ :format_rows false})+ lines (str/split-lines result)]+ (is (= 2 (count lines)) "Should have header and one data row")+ (is (str/includes? (second lines) "\"")+ "Value containing separator should be quoted")))))diff --git a/test_resources/serialization_baseline/settings.yaml b/test_resources/serialization_baseline/settings.yamlindex 92d434cdb58..55408373352 100644--- a/test_resources/serialization_baseline/settings.yaml+++ b/test_resources/serialization_baseline/settings.yaml@@ -10,6 +10,7 @@ available-fonts: nullavailable-locales: nullavailable-timezones: nullbreakout-bins-num: null+csv-field-separator: nilcustom-formatting: nullcustom-geojson: nullcustom-geojson-enabled: null