Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Speed up the ratchet scan 2.5x with a regex masker (#84112)

Chris Truter4d17822f03a2
diff --git a/dev/src/dev/kondo_ratchet.clj b/dev/src/dev/kondo_ratchet.clj
index f035e134f63..1fc2bb24df5 100644
--- a/dev/src/dev/kondo_ratchet.clj
+++ b/dev/src/dev/kondo_ratchet.clj
@@ -474,6 +474,19 @@
(def ^:private namespaced-ignore-prefix-re
(re-pattern (str reader-form-start "#:clj-kondo" ignore-marker-boundary)))
+;; One alternative per thing to mask: a string, a line comment, a char literal.
+;; The string's closing quote is captured so that an unterminated string, which runs to the end of the input,
+;; can be told apart. The loops are unrolled and possessive: `(?:a|b)*` recurses once per character and
+;; overflows the regex engine's stack on a long string.
+(def ^:private maskable-re
+ #"\"[^\"\\]*+(?:\\[\s\S]?[^\"\\]*+)*+(\")?|;[^\n]*|\\[\s\S]?")
+
+(defn- blank
+ "`s` with everything but its newlines replaced by spaces, at the same length."
+ [s]
+ ;; a character class matches a whole code point, so the run is measured in chars to keep the length
+ (str/replace s #"[^\n]+" (fn [^String run] (.repeat " " (.length run)))))
+
(defn mask-strings-and-comments
"`content` with string-literal and line-comment interiors replaced by spaces, newlines kept.
Same length as the input, so offsets and line numbers carry over.
@@ -482,35 +495,18 @@
[[has-justification-comment?]] can locate real trailing comments."
[content]
(let [sb (StringBuilder. ^String content)
- n (count content)]
- (loop [i 0, state :code]
- (if (>= i n)
- (str sb)
- (let [c (.charAt sb i)]
- (case state
- :code (case c
- \" (recur (inc i) :string)
- \; (recur (inc i) :comment)
- ;; char literal: mask the next char so it can't open a string or start a comment
- \\ (do (when (< (inc i) n)
- (when-not (= (.charAt sb (inc i)) \newline)
- (.setCharAt sb (inc i) \space)))
- (recur (+ i 2) :code))
- (recur (inc i) :code))
- :string (case c
- \" (recur (inc i) :code)
- \\ (do (.setCharAt sb i \space)
- (when (< (inc i) n)
- (when-not (= (.charAt sb (inc i)) \newline)
- (.setCharAt sb (inc i) \space)))
- (recur (+ i 2) :string))
- \newline (recur (inc i) :string)
- (do (.setCharAt sb i \space)
- (recur (inc i) :string)))
- :comment (if (= c \newline)
- (recur (inc i) :code)
- (do (.setCharAt sb i \space)
- (recur (inc i) :comment)))))))))
+ m (re-matcher maskable-re content)]
+ (while (.find m)
+ ;; the opening quote, `;` or backslash survives, and so does a string's closing quote
+ (let [start (inc (.start m))
+ end (cond
+ (.group m 1) (dec (.end m))
+ ;; a char literal masks one char, though the regex takes a whole code point
+ (= \\ (.charAt content (.start m))) (min (.end m) (inc start))
+ :else (.end m))]
+ (when (< start end)
+ (.replace sb start end ^String (blank (subs content start end))))))
+ (str sb)))
(defn- linter-keywords
[vector-contents]
diff --git a/dev/test/metabase/core/kondo_ratchet_test.clj b/dev/test/metabase/core/kondo_ratchet_test.clj
index 26e0d751778..ce0fd39ea8f 100644
--- a/dev/test/metabase/core/kondo_ratchet_test.clj
+++ b/dev/test/metabase/core/kondo_ratchet_test.clj
@@ -45,7 +45,23 @@
;; start a comment
"(f \" \")" "(f \"a\\\"b c\")"
"[\\ \" \"]" "[\\\" \"abc\"]"
- "(f \\ \" \")" "(f \\; \"a;b\")")
+ "(f \\ \" \")" "(f \\; \"a;b\")"
+ ;; input that ends mid-token: an unterminated string runs to the end, a trailing backslash is kept
+ "" ""
+ "\"" "\""
+ "(f \" " "(f \"never closed"
+ "(f \" " "(f \"ends on escape\\"
+ "(f) \\" "(f) \\"
+ "(f) ; " "(f) ; no newline"
+ "\" \n \" \\\n ; " "\"a\\\nb\" \\\n ; c")
+ (testing "characters outside the basic plane keep their two-char length, and a char literal masks one char"
+ (is (= "(f \" \") ; \n(g \\ \uDE00)"
+ (kondo-ratchet/mask-strings-and-comments
+ "(f \"😀 café\") ; 😀\n(g \\😀)"))))
+ (testing "a string with more escapes than a recursive regex could match"
+ (is (= (str "(def x \"" (str/join (repeat 60000 \space)) "\")")
+ (kondo-ratchet/mask-strings-and-comments
+ (str "(def x \"" (str/join (repeat 20000 "\\\"a")) "\")")))))
(testing "masking preserves length and newline positions"
(let [content "(f \"a\nb\") ; c\n(g)"
masked (kondo-ratchet/mask-strings-and-comments content)]