The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GDGT-304: Fix unthreadsafe number parsing in csv upload (#75119)
Ngoc Khuat5528bf4740f4
diff --git a/src/metabase/upload/parsing.clj b/src/metabase/upload/parsing.cljindex 6375570d041..212189b12f2 100644--- a/src/metabase/upload/parsing.clj+++ b/src/metabase/upload/parsing.clj@@ -157,19 +157,30 @@[s](str/replace s currency-regex ""))-(let [us (NumberFormat/getInstance (Locale. "en" "US"))- de (NumberFormat/getInstance (Locale. "de" "DE"))- fr (NumberFormat/getInstance (Locale. "fr" "FR"))- ch (NumberFormat/getInstance (Locale. "de" "CH"))]+(defn- number-format-thread-local+ "A `ThreadLocal` holding one `NumberFormat` per thread for `locale`."+ ^ThreadLocal [^Locale locale]+ (ThreadLocal/withInitial+ #(NumberFormat/getInstance locale)))++;; NumberFormat is not thread-safe: it keeps mutable parse state inside itself, so parsing with a+;; shared instance lets concurrent uploads corrupt each other - either throwing or silently+;; returning the wrong number. We give each thread its own formatter per locale via ThreadLocal,+;; so a thread reuses its copy across every cell instead of reallocating one per value.+;; See https://linear.app/metabase/issue/GDGT-304+(let [us (number-format-thread-local (Locale. "en" "US"))+ de (number-format-thread-local (Locale. "de" "DE"))+ fr (number-format-thread-local (Locale. "fr" "FR"))+ ch (number-format-thread-local (Locale. "de" "CH"))](defn- parse-plain-number [number-separators s](let [has-parens? (re-matches #"\(.*\)" s)deparenthesized-s (str/replace s #"[()]" "")parse-pos (ParsePosition. 0)parsed-number (case number-separators- ("." ".,") (. us parse deparenthesized-s parse-pos)- ",." (. de parse deparenthesized-s parse-pos)- ", " (. fr parse (str/replace deparenthesized-s \space \u00A0) parse-pos) ; \u00A0 is a non-breaking space- ".’" (. ch parse deparenthesized-s parse-pos))]+ ("." ".,") (. ^NumberFormat (.get us) parse deparenthesized-s parse-pos)+ ",." (. ^NumberFormat (.get de) parse deparenthesized-s parse-pos)+ ", " (. ^NumberFormat (.get fr) parse (str/replace deparenthesized-s \space \u00A0) parse-pos) ; \u00A0 is a non-breaking space+ ".’" (. ^NumberFormat (.get ch) parse deparenthesized-s parse-pos))](let [parsed-idx (.getIndex parse-pos)](when-not (= parsed-idx (count deparenthesized-s))(throw (ex-info "Unexpected trailing characters - this is probably not a number"diff --git a/test/metabase/upload/parsing_test.clj b/test/metabase/upload/parsing_test.cljnew file mode 100644index 00000000000..09327afaf08--- /dev/null+++ b/test/metabase/upload/parsing_test.clj@@ -0,0 +1,34 @@+(ns metabase.upload.parsing-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.upload.parsing :as upload-parsing]+ [metabase.upload.types :as upload-types]))++(deftest parse-number-is-thread-safe-test+ ;; `metabase.upload.parsing` kept one `NumberFormat` instance per locale, shared by the whole+ ;; process. `NumberFormat` is not thread-safe, so two concurrent uploads corrupted each other's+ ;; internal parse state - either throwing "''…'' is not a recognizable number" or, worse, silently+ ;; returning the wrong number. https://linear.app/metabase/issue/GDGT-304+ (testing "concurrent number parsing must not throw or return wrong values (GDGT-304)"+ ;; the five separator settings exercise all four shared formatters+ ;; ("." / ".," -> US, ",." -> DE, ", " -> FR, ".’" -> CH); "241" is a clean integer in each+ (doseq [separators ["." ".," ",." ", " ".’"]]+ (let [parser (upload-parsing/upload-type->parser ::upload-types/int {:number-separators separators})+ n-threads 50+ n-iters 1000+ mistakes (->> (repeatedly n-threads+ (fn []+ (future+ (reduce (fn [bad _]+ (+ bad (try+ (if (= (biginteger 241) (parser "241")) 0 1)+ (catch Throwable _ 1))))+ 0+ (range n-iters)))))+ doall+ (map deref)+ (reduce +))]+ (testing (format "separators %s" (pr-str separators))+ (is (zero? mistakes)+ (format "%d/%d concurrent parses of \"241\" threw or returned the wrong value"+ mistakes (* n-threads n-iters))))))))