Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Strip large literal lists (IN, tuple IN, ARRAY) before GraalPy SQL parsing (#79028)

Alexander Polyankin9d391efca1b3
diff --git a/src/metabase/sql_parsing/core.clj b/src/metabase/sql_parsing/core.clj
index 99414315ae1..f9fa32d9e40 100644
--- a/src/metabase/sql_parsing/core.clj
+++ b/src/metabase/sql_parsing/core.clj
@@ -39,19 +39,21 @@
(boolean (and (:sql-parsing/error data)
(= "ParseError" (:sql-parsing/python-error-type data))))))
-;;; ----------------------------------------- VALUES clause stripping ------------------------------------------
+;;; ------------------------------------- Large literal-list stripping -----------------------------------------
-;; Large VALUES clauses (thousands of tuples) cause GraalPy OOM due to ~114KB per AST node
-;; (vs ~1-2KB on CPython). We strip them on the JVM side before passing SQL to Python,
-;; because GraalPy is too slow at character-by-character string scanning over multi-MB inputs.
+;; Large literal lists — VALUES clauses, IN lists (flat or tuple), and ARRAY literals with
+;; thousands of items — cause GraalPy OOM/timeouts due to ~114KB per AST node (vs ~1-2KB on
+;; CPython). We strip them on the JVM side before passing SQL to Python, because GraalPy is too
+;; slow at character-by-character string scanning over multi-MB inputs.
-(def ^:private ^:const values-strip-threshold
- "Only strip VALUES clauses with more than this many tuples."
+(def ^:private ^:const strip-threshold
+ "Strip VALUES clauses with more than this many tuples, and IN lists / ARRAY literals with at
+ least this many commas."
100)
-(def ^:private values-keyword-pattern
- "Pattern to find VALUES keyword followed by opening paren."
- (re-pattern "(?i)\\bVALUES\\s*\\("))
+(def ^:private literal-list-keyword-pattern
+ "Pattern to find a VALUES, IN, or ARRAY keyword followed by its opening delimiter."
+ (re-pattern "(?i)\\b(?:VALUES\\s*\\(|IN\\s*\\(|ARRAY\\s*\\[)"))
(defn- skip-whitespace
"Return the first non-whitespace position at or after `pos`."
@@ -61,47 +63,51 @@
(recur (inc p))
p)))
+(defn- skip-string-literal
+ "Starting at an opening quote (single or double), advance past the matching closing quote,
+ treating a doubled quote as an escape. Returns the position immediately after the closing quote."
+ ^long [^String sql ^long pos ^long n]
+ (let [q (.charAt sql pos)]
+ (loop [i (inc pos)]
+ (if (>= i n)
+ i
+ (if (= (.charAt sql i) q)
+ (if (and (< (inc i) n) (= (.charAt sql (inc i)) q))
+ (recur (+ i 2)) ; escaped (doubled) quote
+ (inc i))
+ (recur (inc i)))))))
+
(defn- skip-balanced-parens
"Starting at an opening `(`, advance past the matching `)`.
- Handles nested parens and SQL string literals (single and double quoted, with doubled-quote escaping).
- Returns the position immediately after the closing `)`."
+ Handles nested parens and SQL string literals. Returns the position immediately after the closing `)`."
^long [^String sql ^long pos ^long n]
- (loop [pos pos, depth (int 0), in-str false, str-ch \space]
+ (loop [pos pos, depth 0]
(if (>= pos n)
pos
(let [ch (.charAt sql pos)]
(cond
- ;; Inside a string literal — look for the closing quote
- in-str
- (if (= ch str-ch)
- (if (and (< (inc pos) n) (= (.charAt sql (inc pos)) str-ch))
- (recur (+ pos 2) depth true str-ch) ; escaped (doubled) quote
- (recur (inc pos) depth false str-ch))
- (recur (inc pos) depth true str-ch))
-
- (or (= ch \') (= ch \")) (recur (inc pos) depth true ch)
- (= ch \() (recur (inc pos) (inc depth) false str-ch)
- (= ch \)) (if (= depth 1)
- (inc pos) ; done
- (recur (inc pos) (dec depth) false str-ch))
- :else (recur (inc pos) depth false str-ch))))))
+ (or (= ch \') (= ch \")) (recur (skip-string-literal sql pos n) depth)
+ (= ch \() (recur (inc pos) (inc depth))
+ (= ch \)) (if (= depth 1)
+ (inc pos) ; done
+ (recur (inc pos) (dec depth)))
+ :else (recur (inc pos) depth))))))
(defn- count-top-level-commas
"Count top-level comma-separated items inside a tuple's content string.
`(1, 'a', 3)` → inner content `1, 'a', 3` → 3 items."
^long [^String content]
(let [n (.length content)]
- (loop [i 0, depth (int 0), in-str false, str-ch \space, items (int 1)]
+ (loop [i 0, depth 0, items 1]
(if (>= i n)
items
(let [ch (.charAt content i)]
(cond
- in-str (recur (inc i) depth (not= ch str-ch) str-ch items)
- (or (= ch \') (= ch \")) (recur (inc i) depth true ch items)
- (= ch \() (recur (inc i) (inc depth) false str-ch items)
- (= ch \)) (recur (inc i) (dec depth) false str-ch items)
- (and (= ch \,) (= depth 0)) (recur (inc i) depth false str-ch (inc items))
- :else (recur (inc i) depth false str-ch items)))))))
+ (or (= ch \') (= ch \")) (recur (skip-string-literal content i n) depth items)
+ (= ch \() (recur (inc i) (inc depth) items)
+ (= ch \)) (recur (inc i) (dec depth) items)
+ (and (= ch \,) (= depth 0)) (recur (inc i) depth (inc items))
+ :else (recur (inc i) depth items)))))))
(defn- count-and-skip-tuples
"Starting after the first tuple, count how many more `, (...)` tuples follow.
@@ -122,58 +128,113 @@
(let [nulls (str/join ", " (repeat col-count "NULL"))]
(str original-keyword " (" nulls ")")))
-(defn- extract-values-keyword
- "Extract just the VALUES keyword text from a regex match like `VALUES (`."
+(defn- extract-keyword
+ "Extract just the keyword text from a regex match like `VALUES (`, `IN (`, or `ARRAY [`."
^String [^String sql ^long match-start ^long match-end]
(-> (.substring sql match-start match-end)
str/trimr
- (str/replace #"\($" "")
+ (str/replace #"[\(\[]$" "")
str/trimr))
-(defn- strip-large-values*
- "Walk through SQL, replacing any VALUES clause with more than `values-strip-threshold`
- tuples with a single-row NULL placeholder. Preserves column count from the first tuple."
+(defn- strip-values-at
+ "Decide whether to strip the VALUES clause whose `VALUES (` match spans [match-start, match-end).
+ Returns [replacement resume-pos]: a single-row NULL placeholder (column count taken from the
+ first tuple) covering the SQL up to resume-pos, or [nil match-end] to leave the clause untouched."
+ [^String sql ^long match-start ^long match-end ^long n]
+ (let [paren-start (dec match-end)
+ first-end (skip-balanced-parens sql paren-start n)
+ [tuple-count end-pos] (count-and-skip-tuples sql first-end n)]
+ (if (and (> (long tuple-count) strip-threshold)
+ (> first-end (inc paren-start)))
+ [(make-null-placeholder (extract-keyword sql match-start match-end)
+ (count-top-level-commas (.substring sql (inc paren-start) (dec first-end))))
+ end-pos]
+ [nil match-end])))
+
+(defn- simple-list-end
+ "Scan the list opening at `open-pos` (`(` or `[`). If it contains only numbers, single-quoted
+ strings, signs, commas, whitespace, and balanced parens (VALUES-style tuples), return
+ [comma-count end-pos] with end-pos just past the matching closing delimiter. Any other
+ character — subqueries, column references, casts, bind parameters — returns nil, leaving the
+ list untouched."
+ [^String sql ^long open-pos ^long n]
+ (let [close-ch (char (if (= (.charAt sql open-pos) \[) \] \)))]
+ (loop [i (inc open-pos), depth 1, commas 0]
+ (when (< i n)
+ (let [ch (.charAt sql i)]
+ (cond
+ (and (= ch close-ch) (= depth 1)) [commas (inc i)]
+ (= ch \') (recur (skip-string-literal sql i n) depth commas)
+ (= ch \() (recur (inc i) (inc depth) commas)
+ (= ch \)) (when (> depth 1) (recur (inc i) (dec depth) commas))
+ (= ch \,) (recur (inc i) depth (inc commas))
+ (or (Character/isDigit ch)
+ (Character/isWhitespace ch)
+ (= ch \.) (= ch \-) (= ch \+))
+ (recur (inc i) depth commas)
+
+ :else nil))))))
+
+(defn- strip-list-at
+ "Decide whether to strip the IN list or ARRAY literal whose keyword match spans
+ [match-start, match-end). Returns [replacement resume-pos]: `IN (NULL)` / `ARRAY[NULL]`
+ covering the whole list, or [nil match-end] to leave it untouched (resuming just inside the
+ delimiter, so a large list nested in a subquery — `IN (SELECT ... WHERE x IN (...))` — is
+ still found)."
+ [^String sql ^long match-start ^long match-end ^long n]
+ (let [open-pos (dec match-end)
+ [commas end-pos] (simple-list-end sql open-pos n)]
+ (if (and commas (>= (long commas) strip-threshold))
+ [(str (extract-keyword sql match-start match-end)
+ (if (= (.charAt sql open-pos) \[) "[NULL]" " (NULL)"))
+ end-pos]
+ [nil match-end])))
+
+(defn- strip-large-literal-lists*
+ "Single pass over `sql`, replacing every oversized literal list — VALUES clause or literal-only
+ IN list — with a NULL placeholder. Returns `sql` itself when nothing was stripped."
^String [^String sql]
- (let [matcher (re-matcher values-keyword-pattern sql)
- n (int (.length sql))]
+ (let [matcher (re-matcher literal-list-keyword-pattern sql)
+ n (.length sql)]
(if-not (.find matcher)
sql
- (let [_ (.reset matcher)
- sb (StringBuilder.)]
- (loop [i (int 0)]
- (if-not (.find matcher i)
- (-> sb (.append sql i n) .toString)
- (let [match-start (.start matcher)
- match-end (.end matcher)
- _ (.append sb sql (int i) (int match-start))
- ;; Parse the first tuple to learn its column count
- paren-start (dec (int match-end))
- first-end (skip-balanced-parens sql paren-start n)
- first-inner (when (> first-end (inc paren-start))
- (.substring sql (inc paren-start) (dec (int first-end))))
- ;; Scan remaining tuples
- [tuple-count end-pos] (count-and-skip-tuples sql first-end n)]
- (if (and (> (int tuple-count) values-strip-threshold) first-inner)
- (do (.append sb (make-null-placeholder
- (extract-values-keyword sql match-start match-end)
- (count-top-level-commas first-inner)))
- (recur (int end-pos)))
- (do (.append sb sql (int match-start) (int end-pos))
- (recur (int end-pos)))))))))))
-
-(defn strip-large-values
- "Replace large VALUES clauses with a single-row NULL placeholder.
-
- Preserves the column count from the first tuple and all surrounding SQL structure.
- Only triggers when a VALUES clause has more than `values-strip-threshold` tuples.
+ (let [sb (StringBuilder.)]
+ (loop [i 0, stripped? false, match? true]
+ (if-not match?
+ (if stripped?
+ (-> sb (.append sql (int i) (int n)) .toString)
+ sql)
+ (let [match-start (.start matcher)
+ match-end (.end matcher)
+ [replacement resume] (let [ch (.charAt sql match-start)]
+ (if (or (= ch \V) (= ch \v))
+ (strip-values-at sql match-start match-end n)
+ (strip-list-at sql match-start match-end n)))]
+ (if replacement
+ (-> sb (.append sql (int i) (int match-start)) (.append ^String replacement))
+ (.append sb sql (int i) (int resume)))
+ (recur (long resume)
+ (or stripped? (some? replacement))
+ (.find matcher (int resume))))))))))
+
+(defn strip-large-literal-lists
+ "Replace large literal lists with NULL placeholders: VALUES clauses with more than
+ [[strip-threshold]] tuples become a single-row NULL tuple (preserving the column count from the
+ first tuple), and simple IN lists / ARRAY literals — numbers, single-quoted strings, and
+ VALUES-style tuples of them — with at least [[strip-threshold]] commas become `IN (NULL)` /
+ `ARRAY[NULL]`. All surrounding SQL structure is preserved.
This runs on the JVM side (fast) before passing SQL to GraalPy (slow at char scanning).
- On any error, returns the original SQL unchanged so parsing can proceed normally."
+ On any error, returns the original SQL unchanged so parsing can proceed normally.
+
+ Best-effort by design: stripping only feeds fail-soft analysis, and execution paths restore the
+ original SQL when anything was stripped ([[is-single-stmt-of-type?]]), so edge cases like
+ keyword-shaped text inside string literals or comments are deliberately not guarded against."
^String [^String sql]
(try
- (strip-large-values* sql)
+ (strip-large-literal-lists* sql)
(catch Exception e
- (log/warnf "Error stripping VALUES clauses, passing SQL through unchanged: %s" (ex-message e))
+ (log/warnf "Error stripping large literal lists, passing SQL through unchanged: %s" (ex-message e))
sql)))
;;; -------------------------------------------------- Public API --------------------------------------------------
@@ -187,7 +248,7 @@
This is the pure parsing layer - it returns what's literally in the SQL.
Default schema resolution happens in the matching layer (core.clj)."
[dialect sql]
- (protocol/referenced-tables (parser) dialect (strip-large-values sql)))
+ (protocol/referenced-tables (parser) dialect (strip-large-literal-lists sql)))
(defn referenced-fields
"Extract field references from SQL, returning only fields from actual database tables.
@@ -213,7 +274,7 @@
(referenced-fields \"bigquery\" \"SELECT * FROM myproject.analytics.events\")
=> [[\"myproject\" \"analytics\" \"events\" \"*\"]]"
[dialect sql]
- (protocol/referenced-fields (parser) dialect (strip-large-values sql)))
+ (protocol/referenced-fields (parser) dialect (strip-large-literal-lists sql)))
(defn returned-columns-lineage
"Extract column lineage from SQL query, showing which output columns depend on which source columns.
@@ -233,7 +294,7 @@
(returned-columns-lineage \"postgres\" \"SELECT id + 1 as computed FROM users\" nil schema)
=> [[\"computed\" false [[[nil \"users\" \"id\"]]]]]"
[dialect sql default-table-schema sqlglot-schema]
- (protocol/returned-columns-lineage (parser) dialect (strip-large-values sql) default-table-schema sqlglot-schema))
+ (protocol/returned-columns-lineage (parser) dialect (strip-large-literal-lists sql) default-table-schema sqlglot-schema))
(defn validate-query
"Validate a SQL query against a schema using sqlglot's qualify optimizer.
@@ -265,7 +326,7 @@
- \"invalid_expression\": Syntax/parse error
- \"unhandled\": Other errors"
[dialect sql default-table-schema & [sqlglot-schema]]
- (protocol/validate-query (parser) dialect (strip-large-values sql) default-table-schema sqlglot-schema))
+ (protocol/validate-query (parser) dialect (strip-large-literal-lists sql) default-table-schema sqlglot-schema))
(defn simple-query?
"Check if SQL is a simple SELECT without LIMIT, OFFSET, or CTEs.
@@ -285,7 +346,7 @@
(simple-query? nil \"SELECT * FROM users LIMIT 10\")
=> {:is_simple false :reason \"Contains a LIMIT\"}"
[dialect sql]
- (protocol/simple-query (parser) dialect (strip-large-values sql)))
+ (protocol/simple-query (parser) dialect (strip-large-literal-lists sql)))
(defn add-into-clause
"Add an INTO clause to a SELECT statement for SQL Server SELECT INTO syntax.
@@ -417,7 +478,7 @@
'fail soft' pattern used for parsing failures."
[dialect sql]
(try
- (let [raw (protocol/field-references (parser) dialect (strip-large-values sql))
+ (let [raw (protocol/field-references (parser) dialect (strip-large-literal-lists sql))
used-fields (or (:used-fields raw) (:used_fields raw) (get raw "used_fields") [])
returned-fields (or (:returned-fields raw) (:returned_fields raw) (get raw "returned_fields") [])
errors (or (:errors raw) (get raw "errors") [])]
@@ -456,12 +517,13 @@
"Validates that a query is a single read statement (SELECT) or a single write statement (INSERT, UPDATE, DELETE)
and returns the query reconstructed from the parsed AST."
[dialect sql stmt-type]
- (let [stripped-sql (strip-large-values sql)
+ (let [stripped-sql (strip-large-literal-lists sql)
result (-> (protocol/single-stmt-of-type (parser) dialect stripped-sql stmt-type)
(perf/update-keys (comp keyword u/->kebab-case-en)))]
;; The `:sql` in the `result` is the reconstructed SQL from the SQLGlot parser.
- ;; We generally want to use the reconstructed SQL, but if the original SQL had its values stripped (to avoid GraalPy OOM)
- ;; then we need to return the original SQL to preserve the values. (#74284)
+ ;; We generally want to use the reconstructed SQL, but if the original SQL had its VALUES/IN
+ ;; literal lists stripped (to avoid GraalPy OOM) then we need to return the original SQL to
+ ;; preserve the values. (#74284)
(cond-> result
(not= sql stripped-sql) (assoc :sql sql))))
diff --git a/test/metabase/sql_parsing/core_test.clj b/test/metabase/sql_parsing/core_test.clj
index 561569cc314..27dd4ac3f35 100644
--- a/test/metabase/sql_parsing/core_test.clj
+++ b/test/metabase/sql_parsing/core_test.clj
@@ -1,6 +1,7 @@
(ns metabase.sql-parsing.core-test
(:require
[clojure.java.io :as io]
+ [clojure.string :as str]
[clojure.test :refer :all]
[metabase.sql-parsing.core :as sql-parsing]
[metabase.util :as u]))
@@ -351,7 +352,6 @@
"Nonexistent table reference is parsed and will fail at execution time"))))
(comment
- (require '[clojure.string :as str])
(def query-corpus-path "/Users/bcm/dv/mb/query_corpus/")
(def sentinel (re-pattern "\n-----end-query-----\n"))
@@ -437,56 +437,328 @@
"\n Expected: " (pr-str (normalize-fields expected))
"\n Actual: " (pr-str (normalize-fields actual))))))))))
-;;; ----------------------------------------- Large VALUES stripping tests -----------------------------------------
+;;; -------------------------------------- Large literal-list stripping tests --------------------------------------
(deftest ^:parallel strip-large-values-test
(testing "Small VALUES clauses are preserved"
(let [sql "SELECT * FROM (VALUES (1, 'a'), (2, 'b'), (3, 'c')) AS t(id, name)"]
- (is (= sql (sql-parsing/strip-large-values sql)))))
+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))
(testing "No VALUES keyword returns SQL unchanged"
(let [sql "SELECT * FROM users WHERE id = 1"]
- (is (= sql (sql-parsing/strip-large-values sql)))))
+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))
(testing "Large VALUES clause is replaced with NULLs preserving column count"
- (let [tuples (clojure.string/join ", " (map #(format "(%d, '%s', %d)" % (str "name" %) (* % 10))
- (range 200)))
+ (let [tuples (str/join ", " (map #(format "(%d, '%s', %d)" % (str "name" %) (* % 10))
+ (range 200)))
sql (str "SELECT * FROM (VALUES " tuples ") AS t(id, name, score)")
- result (sql-parsing/strip-large-values sql)]
- (is (clojure.string/includes? result "VALUES (NULL, NULL, NULL)"))
- (is (clojure.string/includes? result "AS t(id, name, score)"))
- (is (not (clojure.string/includes? result "name0")))))
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (is (str/includes? result "VALUES (NULL, NULL, NULL)"))
+ (is (str/includes? result "AS t(id, name, score)"))
+ (is (not (str/includes? result "name0")))))
(testing "Multiple large VALUES clauses are all stripped"
- (let [tuples1 (clojure.string/join ", " (map #(format "(%d)" %) (range 200)))
- tuples2 (clojure.string/join ", " (map #(format "(%d, %d)" % (* % 2)) (range 200)))
+ (let [tuples1 (str/join ", " (map #(format "(%d)" %) (range 200)))
+ tuples2 (str/join ", " (map #(format "(%d, %d)" % (* % 2)) (range 200)))
sql (str "WITH a AS (SELECT * FROM (VALUES " tuples1 ") AS t(x)), "
"b AS (SELECT * FROM (VALUES " tuples2 ") AS t(y, z)) "
"SELECT * FROM a JOIN b ON a.x = b.y")
- result (sql-parsing/strip-large-values sql)]
- (is (clojure.string/includes? result "VALUES (NULL)"))
- (is (clojure.string/includes? result "VALUES (NULL, NULL)"))))
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (is (str/includes? result "VALUES (NULL)"))
+ (is (str/includes? result "VALUES (NULL, NULL)"))))
(testing "VALUES keyword casing is preserved"
- (let [tuples (clojure.string/join ", " (map #(format "(%d)" %) (range 200)))
+ (let [tuples (str/join ", " (map #(format "(%d)" %) (range 200)))
sql (str "select * from (values " tuples ") as t(x)")
- result (sql-parsing/strip-large-values sql)]
- (is (clojure.string/includes? result "values (NULL)"))))
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (is (str/includes? result "values (NULL)"))))
(testing "VALUES inside a string literal is not stripped"
(let [sql "SELECT 'INSERT INTO foo VALUES (1,2,3)' AS example FROM bar"]
- (is (= sql (sql-parsing/strip-large-values sql)))))
+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))
(testing "Column named values is not stripped"
(let [sql "SELECT values FROM my_table WHERE values > 10"]
- (is (= sql (sql-parsing/strip-large-values sql)))))
+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))
(testing "VALUES keyword not followed by paren is not stripped"
(let [sql "SELECT * FROM t WHERE col IN (SELECT values FROM other)"]
- (is (= sql (sql-parsing/strip-large-values sql)))))
+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))
(testing "INSERT INTO ... VALUES is stripped when large"
- (let [tuples (clojure.string/join ", " (map #(format "(%d, 'x')" %) (range 200)))
+ (let [tuples (str/join ", " (map #(format "(%d, 'x')" %) (range 200)))
sql (str "INSERT INTO foo VALUES " tuples)
- result (sql-parsing/strip-large-values sql)]
- (is (clojure.string/includes? result "VALUES (NULL, NULL)"))
- (is (not (clojure.string/includes? result "(0, 'x')"))))))
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (is (str/includes? result "VALUES (NULL, NULL)"))
+ (is (not (str/includes? result "(0, 'x')"))))))
+
+(deftest ^:parallel strip-large-in-lists-test
+ (testing "Small IN lists are preserved"
+ (let [sql "SELECT * FROM t WHERE id IN (1, 2, 3)"]
+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))
+ (testing "Large literal IN list is replaced with IN (NULL)"
+ (let [sql (str "SELECT * FROM t WHERE id IN (" (str/join ", " (range 200)) ") AND active = true")
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (is (= "SELECT * FROM t WHERE id IN (NULL) AND active = true" result))))
+ (testing "Large IN list of strings, negatives, and decimals is stripped"
+ (let [items (concat (map #(format "'name %d'" %) (range 100))
+ (map #(str "-" %) (range 50))
+ ["1.5" "+2"])
+ sql (str "SELECT * FROM t WHERE name IN (" (str/join ", " items) ")")
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (is (= "SELECT * FROM t WHERE name IN (NULL)" result))))
+ (testing "IN keyword casing is preserved"
+ (let [sql (str "SELECT * FROM t WHERE id in (" (str/join ", " (range 200)) ")")
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (is (str/includes? result "in (NULL)"))))
+ (testing "NOT IN with a large literal list is stripped"
+ (let [sql (str "SELECT * FROM t WHERE id NOT IN (" (str/join ", " (range 200)) ")")
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (is (= "SELECT * FROM t WHERE id NOT IN (NULL)" result))))
+ (testing "Multiple large IN lists are all stripped"
+ (let [in1 (str/join ", " (range 200))
+ in2 (str/join ", " (map #(format "'x%d'" %) (range 200)))
+ sql (str "SELECT * FROM t WHERE a IN (" in1 ") OR b IN (" in2 ")")
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (is (= "SELECT * FROM t WHERE a IN (NULL) OR b IN (NULL)" result))))
+ (testing "IN with a subquery is not stripped"
+ (let [sql "SELECT * FROM t WHERE id IN (SELECT id FROM other)"]
+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))
+ (testing "A large IN list nested inside an unstripped IN subquery is still stripped"
+ (let [sql (str "SELECT * FROM t WHERE id IN (SELECT id FROM other WHERE x IN ("
+ (str/join ", " (range 200)) "))")
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (is (= "SELECT * FROM t WHERE id IN (SELECT id FROM other WHERE x IN (NULL))" result))))
+ (testing "Large IN lists containing non-literals are not stripped"
+ (doseq [extra-item ["col_ref" "foo(1)" "?" "$1" "{{param}}" "1e5" "DATE '2024-01-01'" "NULL"]]
+ (let [sql (str "SELECT * FROM t WHERE id IN (" (str/join ", " (concat (range 200) [extra-item])) ")")]
+ (is (= sql (sql-parsing/strip-large-literal-lists sql))
+ (str "list containing " (pr-str extra-item) " should not be stripped")))))
+ (testing "Word boundaries: JOIN ( and INT( do not trigger IN matching"
+ (let [sql "SELECT CAST(x AS INT) FROM a JOIN (SELECT * FROM b) c ON a.id = c.id"]
+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))
+ (testing "Large VALUES and IN in the same statement are both stripped"
+ (let [tuples (str/join ", " (map #(format "(%d)" %) (range 200)))
+ sql (str "WITH v AS (SELECT * FROM (VALUES " tuples ") AS t(x)) "
+ "SELECT * FROM v WHERE x IN (" (str/join ", " (range 200)) ")")
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (is (= "WITH v AS (SELECT * FROM (VALUES (NULL)) AS t(x)) SELECT * FROM v WHERE x IN (NULL)" result)))))
+
+(deftest ^:parallel large-in-list-referenced-tables-test
+ (testing "referenced-tables and referenced-fields work on queries with massive IN lists"
+ (let [sql (str "SELECT a.name FROM accounts a WHERE a.id IN (" (str/join ", " (range 20000)) ")")]
+ (is (= [[nil nil "accounts"]] (sql-parsing/referenced-tables "postgres" sql)))
+ (is (= [[nil nil "accounts" "id"] [nil nil "accounts" "name"]]
+ (sort (sql-parsing/referenced-fields "postgres" sql)))))))
+
+(defn- in-list
+ "Build `IN (0, 1, ..., n-1)` with `n` items."
+ [n]
+ (str "IN (" (str/join ", " (range n)) ")"))
+
+(deftest ^:parallel strip-threshold-boundary-test
+ (testing "IN lists at or below the 100-item threshold are preserved, above it stripped"
+ (doseq [n [1 2 50 99 100]]
+ (let [sql (str "SELECT * FROM t WHERE id " (in-list n))]
+ (is (= sql (sql-parsing/strip-large-literal-lists sql))
+ (str n " items should not be stripped"))))
+ (doseq [n [101 102 150 1000 10000]]
+ (let [sql (str "SELECT * FROM t WHERE id " (in-list n))]
+ (is (= "SELECT * FROM t WHERE id IN (NULL)" (sql-parsing/strip-large-literal-lists sql))
+ (str n " items should be stripped")))))
+ (testing "VALUES at or below the 100-tuple threshold are preserved, above it stripped"
+ (doseq [[n stripped?] {1 false, 100 false, 101 true, 1000 true}]
+ (let [tuples (str/join ", " (map #(format "(%d, %d)" % %) (range n)))
+ sql (str "SELECT * FROM (VALUES " tuples ") AS t(x, y)")
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (if stripped?
+ (is (= "SELECT * FROM (VALUES (NULL, NULL)) AS t(x, y)" result)
+ (str n " tuples should be stripped"))
+ (is (= sql result)
+ (str n " tuples should not be stripped")))))))
+
+(deftest ^:parallel strip-in-list-literal-shapes-test
+ (testing "Lists of each literal shape are stripped"
+ (doseq [[shape item-fn] {"integers" str
+ "decimals" #(str % ".5")
+ "negatives" #(str "-" %)
+ "plus-signed" #(str "+" %)
+ "single-quoted" #(format "'name %d'" %)
+ "commas-inside-strings" #(format "'a,%d,b'" %)
+ "escaped-quotes" #(format "'it''s %d'" %)}]
+ (let [sql (str "SELECT * FROM t WHERE x IN (" (str/join ", " (map item-fn (range 150))) ")")]
+ (is (= "SELECT * FROM t WHERE x IN (NULL)" (sql-parsing/strip-large-literal-lists sql))
+ (str shape " should be stripped")))))
+ (testing "Whitespace variants are stripped"
+ (doseq [sql [(str "SELECT * FROM t WHERE x IN(" (str/join "," (range 150)) ")")
+ (str "SELECT * FROM t WHERE x IN\n (" (str/join " ,\n" (range 150)) ")")
+ (str "SELECT * FROM t WHERE x in (" (str/join ",\t" (range 150)) ")")]]
+ (is (str/includes? (sql-parsing/strip-large-literal-lists sql) "(NULL)")
+ (pr-str (subs sql 0 40))))))
+
+(deftest ^:parallel strip-in-list-disqualifiers-test
+ (testing "One non-literal item anywhere in a large list prevents stripping"
+ (doseq [bad ["col_ref" "some_column" "foo(1)" "?" "$1" "{{param}}" "%(name)s"
+ "1e5" "NULL" "DATE '2024-01-01'" "TIMESTAMP '2024-01-01 00:00:00'"
+ "1::int" "x.y" "CURRENT_DATE" "-- comment" "héllo" "N'abc'"
+ "\"quoted_identifier\"" "null1"]
+ position [:first :middle :last]]
+ (let [items (map str (range 150))
+ items (case position
+ :first (cons bad items)
+ :middle (concat (take 75 items) [bad] (drop 75 items))
+ :last (concat items [bad]))
+ sql (str "SELECT * FROM t WHERE x IN (" (str/join ", " items) ")")]
+ (is (= sql (sql-parsing/strip-large-literal-lists sql))
+ (str (pr-str bad) " at " position " should prevent stripping")))))
+ (testing "Subqueries are never stripped regardless of size"
+ (let [sql "SELECT * FROM t WHERE x IN (SELECT id FROM u WHERE y = 'a' AND z IN ('b', 'c'))"]
+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))
+ (testing "A large list of double-quoted identifiers is not stripped (they are column references
+ in identifier-quoting dialects, not strings)"
+ (let [sql (str "SELECT * FROM t WHERE x IN (" (str/join ", " (map #(format "\"col_%d\"" %) (range 150))) ")")]
+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql))))))
+
+(deftest ^:parallel strip-preserves-surroundings-test
+ (testing "Everything around a stripped list is preserved byte-for-byte"
+ (let [prefix "WITH cte AS (SELECT 1 AS x) SELECT t.a, cte.x FROM t JOIN cte ON TRUE WHERE t.id "
+ suffix " AND t.status = 'active' ORDER BY t.a LIMIT 5"
+ sql (str prefix (in-list 150) suffix)]
+ (is (= (str prefix "IN (NULL)" suffix)
+ (sql-parsing/strip-large-literal-lists sql)))))
+ (testing "Several lists in one statement are stripped/kept independently"
+ (let [sql (str "SELECT * FROM t WHERE a " (in-list 150)
+ " OR b " (in-list 3)
+ " OR c NOT " (in-list 200)
+ " OR d IN (SELECT x FROM u)")]
+ (is (= (str "SELECT * FROM t WHERE a IN (NULL)"
+ " OR b " (in-list 3)
+ " OR c NOT IN (NULL)"
+ " OR d IN (SELECT x FROM u)")
+ (sql-parsing/strip-large-literal-lists sql))))))
+
+(deftest ^:parallel strip-nesting-test
+ (testing "A large IN list inside a small VALUES tuple is stripped"
+ (let [sql (str "SELECT * FROM (VALUES ((SELECT count(*) FROM u WHERE u.id " (in-list 150) "))) t")]
+ (is (= "SELECT * FROM (VALUES ((SELECT count(*) FROM u WHERE u.id IN (NULL)))) t"
+ (sql-parsing/strip-large-literal-lists sql)))))
+ (testing "A large IN list inside a stripped VALUES clause disappears with it"
+ (let [tuples (str/join ", " (map #(format "(%d)" %) (range 150)))
+ sql (str "SELECT * FROM (VALUES " tuples ", ((SELECT 1 WHERE x " (in-list 150) "))) t")
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (is (= "SELECT * FROM (VALUES (NULL)) t" result))))
+ (testing "IN subqueries nest arbitrarily deep and the innermost large list is still found"
+ (let [sql (str "SELECT * FROM a WHERE x IN "
+ "(SELECT x FROM b WHERE y IN "
+ "(SELECT y FROM c WHERE z " (in-list 150) "))")]
+ (is (= (str "SELECT * FROM a WHERE x IN "
+ "(SELECT x FROM b WHERE y IN "
+ "(SELECT y FROM c WHERE z IN (NULL)))")
+ (sql-parsing/strip-large-literal-lists sql))))))
+
+(deftest ^:parallel strip-idempotency-and-identity-test
+ (testing "Stripping is idempotent"
+ (doseq [sql [(str "SELECT * FROM t WHERE id " (in-list 150))
+ (str "INSERT INTO foo VALUES " (str/join ", " (map #(format "(%d)" %) (range 150))))]]
+ (let [once (sql-parsing/strip-large-literal-lists sql)]
+ (is (= once (sql-parsing/strip-large-literal-lists once))))))
+ (testing "When nothing is stripped the original string instance is returned"
+ (doseq [sql ["SELECT * FROM t"
+ "SELECT * FROM t WHERE id IN (1, 2, 3)"
+ "SELECT * FROM t WHERE id IN (SELECT id FROM u)"
+ "SELECT * FROM (VALUES (1), (2)) AS t(x)"]]
+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql))))))
+
+(deftest ^:parallel strip-malformed-sql-test
+ (testing "Malformed or degenerate input never throws and is left unchanged"
+ (doseq [sql [""
+ "IN ("
+ "VALUES ("
+ (str "SELECT * FROM t WHERE id IN (" (str/join ", " (range 150))) ; no closing paren
+ (str "SELECT * FROM t WHERE id IN ('unterminated string, " (str/join ", " (range 150)) ")")
+ "SELECT * FROM t WHERE id IN ()"
+ "IN () VALUES ()"
+ "in( )"]]
+ (is (= sql (sql-parsing/strip-large-literal-lists sql))
+ (pr-str (subs sql 0 (min 40 (count sql))))))))
+
+(deftest ^:parallel strip-keyword-false-positives-test
+ (testing "Words containing 'in'/'values' and quoted identifiers are not treated as keywords"
+ (doseq [sql ["SELECT margin FROM t WHERE margin > 10"
+ "SELECT * FROM t JOIN (SELECT * FROM u) v ON t.id = v.id"
+ "SELECT CAST(x AS INT) FROM t"
+ "SELECT * FROM logins (1, 2, 3)"
+ "SELECT \"in\" FROM t"
+ "SELECT t.values FROM t WHERE t.values > 10"]]
+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql)) sql))))
+
+(deftest ^:parallel strip-tuple-in-list-test
+ (testing "Tuple IN lists below the comma threshold are preserved, above it stripped"
+ ;; n two-element tuples contain 2n-1 commas; the threshold is 100 commas, so 51 tuples strip
+ (doseq [[n stripped?] {1 false, 50 false, 51 true, 1000 true}]
+ (let [tuples (str/join ", " (map #(format "(%d, %d)" % (* % 2)) (range n)))
+ sql (str "SELECT * FROM t WHERE (a, b) IN (" tuples ")")
+ result (sql-parsing/strip-large-literal-lists sql)]
+ (if stripped?
+ (is (= "SELECT * FROM t WHERE (a, b) IN (NULL)" result)
+ (str n " tuples should be stripped"))
+ (is (identical? sql result)
+ (str n " tuples should not be stripped"))))))
+ (testing "NOT IN and string tuples"
+ (let [tuples (str/join ", " (map #(format "(%d, '%d')" % %) (range 150)))
+ sql (str "SELECT * FROM t WHERE (a, b) NOT IN (" tuples ")")]
+ (is (= "SELECT * FROM t WHERE (a, b) NOT IN (NULL)"
+ (sql-parsing/strip-large-literal-lists sql)))))
+ (testing "Tuple lists containing non-simple tuples are not stripped"
+ (doseq [bad ["(SELECT 1, 2)" "(x, 1)" "(foo(1), 2)" "(?, ?)"]]
+ (let [tuples (str/join ", " (concat (map #(format "(%d, %d)" % %) (range 150)) [bad]))
+ sql (str "SELECT * FROM t WHERE (a, b) IN (" tuples ")")]
+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql))
+ (str "tuple list containing " (pr-str bad) " should not be stripped"))))))
+
+(defn- array-literal
+ "Build `ARRAY[0, 1, ..., n-1]` with `n` items."
+ [n]
+ (str "ARRAY[" (str/join ", " (range n)) "]"))
+
+(deftest ^:parallel strip-array-literal-test
+ (testing "ARRAY literals at or below the threshold are preserved, above it stripped"
+ (doseq [n [1 100]]
+ (let [sql (str "SELECT * FROM t WHERE id = ANY(" (array-literal n) ")")]
+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql))
+ (str n " items should not be stripped"))))
+ (doseq [n [101 1000]]
+ (let [sql (str "SELECT * FROM t WHERE id = ANY(" (array-literal n) ")")]
+ (is (= "SELECT * FROM t WHERE id = ANY(ARRAY[NULL])" (sql-parsing/strip-large-literal-lists sql))
+ (str n " items should be stripped")))))
+ (testing "Casing and string arrays"
+ (let [sql (str "SELECT array['" (str/join "', '" (range 150)) "'] AS a FROM t")]
+ (is (= "SELECT array[NULL] AS a FROM t" (sql-parsing/strip-large-literal-lists sql)))))
+ (testing "Array literals containing non-literals are not stripped"
+ (doseq [bad ["col_ref" "foo(1)" "ARRAY[1]" "1:5"]]
+ (let [sql (str "SELECT ARRAY[" (str/join ", " (concat (range 150) [bad])) "] FROM t")]
+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql))
+ (str "array containing " (pr-str bad) " should not be stripped")))))
+ (testing "Array subscript access on a column named array is untouched"
+ (doseq [sql ["SELECT array[1] FROM t"
+ "SELECT arr[1] FROM t"
+ "SELECT arr[1:5] FROM t"]]
+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql)) sql)))
+ (testing "A large ARRAY nested inside an unstripped IN subquery is still stripped"
+ (let [sql (str "SELECT * FROM t WHERE id IN (SELECT id FROM u WHERE tags && " (array-literal 150) ")")]
+ (is (= "SELECT * FROM t WHERE id IN (SELECT id FROM u WHERE tags && ARRAY[NULL])"
+ (sql-parsing/strip-large-literal-lists sql))))))
+
+(deftest ^:parallel stripped-sql-parses-e2e-test
+ (testing "every analysis entry point tolerates SQL whose lists were stripped"
+ (let [sql (str "SELECT a.name FROM accounts a WHERE a.id " (in-list 150))]
+ (is (= [[nil nil "accounts"]] (sql-parsing/referenced-tables "postgres" sql)))
+ (is (= {:is_simple true} (sql-parsing/simple-query? "postgres" sql)))
+ (is (=? {:status "ok"} (sql-parsing/validate-query "postgres" sql nil)))
+ (is (=? {:used-fields set? :returned-fields vector? :errors #{}}
+ (sql-parsing/field-references "postgres" sql)))))
+ (testing "tuple IN lists and ARRAY literals also parse after stripping"
+ (doseq [sql [(str "SELECT a.name FROM accounts a WHERE (a.id, a.org_id) IN ("
+ (str/join ", " (map #(format "(%d, %d)" % %) (range 20000))) ")")
+ (str "SELECT a.name FROM accounts a WHERE a.id = ANY(" (array-literal 20000) ")")]]
+ (is (= [[nil nil "accounts"]] (sql-parsing/referenced-tables "postgres" sql)))
+ (is (=? {:errors #{}} (sql-parsing/field-references "postgres" sql))))))
(deftest ^:parallel large-values-referenced-tables-test
(testing "referenced-tables works on queries with massive VALUES clauses"
- (let [tuples (clojure.string/join ", " (map #(format "(%d, %d)" % (mod % 100)) (range 20000)))
+ (let [tuples (str/join ", " (map #(format "(%d, %d)" % (mod % 100)) (range 20000)))
sql (str "WITH lookup AS (SELECT * FROM (VALUES " tuples
") AS v(id, score)) "
"SELECT a.name, l.score FROM accounts a "
diff --git a/test/metabase/sql_tools/core_test.clj b/test/metabase/sql_tools/core_test.clj
index 90c0d339274..d780c9e15e7 100644
--- a/test/metabase/sql_tools/core_test.clj
+++ b/test/metabase/sql_tools/core_test.clj
@@ -241,4 +241,14 @@
(str "SELECT 1; " values-query) false false
(str "SET ROLE none; " values-query) false false
(str values-query "; SELECT 1") false false
- (str values-query "; SET ROLE none") false false))))
+ (str values-query "; SET ROLE none") false false)))
+ (testing "we don't remove large IN lists, tuple lists, or arrays when validating impersonated queries"
+ (doseq [query [(str "SELECT x FROM t WHERE x IN (" (str/join ", " (range 105)) ")")
+ (str "SELECT x FROM t WHERE (x, y) IN (" (str/join ", " (map #(format "(%d, %d)" % %) (range 105))) ")")
+ (str "SELECT x FROM t WHERE x = ANY(ARRAY[" (str/join ", " (range 105)) "])")]]
+ (are [sql is-single-stmt? allowed-stmt-type?]
+ (= {:is-single-stmt? is-single-stmt? :allowed-stmt-type? allowed-stmt-type? :sql sql}
+ (sql-tools/is-single-stmt-of-type? :postgres sql "read"))
+ query true true
+ (str "SELECT 1; " query) false false
+ (str query "; SELECT 1") false false))))