The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Speed up the ratchet scan 2.5x with a regex masker (#84112)
Chris Truter4d17822f03a2
diff --git a/dev/src/dev/kondo_ratchet.clj b/dev/src/dev/kondo_ratchet.cljindex f035e134f63..1fc2bb24df5 100644--- a/dev/src/dev/kondo_ratchet.clj+++ b/dev/src/dev/kondo_ratchet.clj@@ -474,6 +474,19 @@(def ^:private namespaced-ignore-prefix-re(re-pattern (str reader-form-start "#:clj-kondo" ignore-marker-boundary)))+;; One alternative per thing to mask: a string, a line comment, a char literal.+;; The string's closing quote is captured so that an unterminated string, which runs to the end of the input,+;; can be told apart. The loops are unrolled and possessive: `(?:a|b)*` recurses once per character and+;; overflows the regex engine's stack on a long string.+(def ^:private maskable-re+ #"\"[^\"\\]*+(?:\\[\s\S]?[^\"\\]*+)*+(\")?|;[^\n]*|\\[\s\S]?")++(defn- blank+ "`s` with everything but its newlines replaced by spaces, at the same length."+ [s]+ ;; a character class matches a whole code point, so the run is measured in chars to keep the length+ (str/replace s #"[^\n]+" (fn [^String run] (.repeat " " (.length run)))))+(defn mask-strings-and-comments"`content` with string-literal and line-comment interiors replaced by spaces, newlines kept.Same length as the input, so offsets and line numbers carry over.@@ -482,35 +495,18 @@[[has-justification-comment?]] can locate real trailing comments."[content](let [sb (StringBuilder. ^String content)- n (count content)]- (loop [i 0, state :code]- (if (>= i n)- (str sb)- (let [c (.charAt sb i)]- (case state- :code (case c- \" (recur (inc i) :string)- \; (recur (inc i) :comment)- ;; char literal: mask the next char so it can't open a string or start a comment- \\ (do (when (< (inc i) n)- (when-not (= (.charAt sb (inc i)) \newline)- (.setCharAt sb (inc i) \space)))- (recur (+ i 2) :code))- (recur (inc i) :code))- :string (case c- \" (recur (inc i) :code)- \\ (do (.setCharAt sb i \space)- (when (< (inc i) n)- (when-not (= (.charAt sb (inc i)) \newline)- (.setCharAt sb (inc i) \space)))- (recur (+ i 2) :string))- \newline (recur (inc i) :string)- (do (.setCharAt sb i \space)- (recur (inc i) :string)))- :comment (if (= c \newline)- (recur (inc i) :code)- (do (.setCharAt sb i \space)- (recur (inc i) :comment)))))))))+ m (re-matcher maskable-re content)]+ (while (.find m)+ ;; the opening quote, `;` or backslash survives, and so does a string's closing quote+ (let [start (inc (.start m))+ end (cond+ (.group m 1) (dec (.end m))+ ;; a char literal masks one char, though the regex takes a whole code point+ (= \\ (.charAt content (.start m))) (min (.end m) (inc start))+ :else (.end m))]+ (when (< start end)+ (.replace sb start end ^String (blank (subs content start end))))))+ (str sb)))(defn- linter-keywords[vector-contents]diff --git a/dev/test/metabase/core/kondo_ratchet_test.clj b/dev/test/metabase/core/kondo_ratchet_test.cljindex 26e0d751778..ce0fd39ea8f 100644--- a/dev/test/metabase/core/kondo_ratchet_test.clj+++ b/dev/test/metabase/core/kondo_ratchet_test.clj@@ -45,7 +45,23 @@;; start a comment"(f \" \")" "(f \"a\\\"b c\")""[\\ \" \"]" "[\\\" \"abc\"]"- "(f \\ \" \")" "(f \\; \"a;b\")")+ "(f \\ \" \")" "(f \\; \"a;b\")"+ ;; input that ends mid-token: an unterminated string runs to the end, a trailing backslash is kept+ "" ""+ "\"" "\""+ "(f \" " "(f \"never closed"+ "(f \" " "(f \"ends on escape\\"+ "(f) \\" "(f) \\"+ "(f) ; " "(f) ; no newline"+ "\" \n \" \\\n ; " "\"a\\\nb\" \\\n ; c")+ (testing "characters outside the basic plane keep their two-char length, and a char literal masks one char"+ (is (= "(f \" \") ; \n(g \\ \uDE00)"+ (kondo-ratchet/mask-strings-and-comments+ "(f \"😀 café\") ; 😀\n(g \\😀)"))))+ (testing "a string with more escapes than a recursive regex could match"+ (is (= (str "(def x \"" (str/join (repeat 60000 \space)) "\")")+ (kondo-ratchet/mask-strings-and-comments+ (str "(def x \"" (str/join (repeat 20000 "\\\"a")) "\")")))))(testing "masking preserves length and newline positions"(let [content "(f \"a\nb\") ; c\n(g)"masked (kondo-ratchet/mask-strings-and-comments content)]