The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Strip large literal lists (IN, tuple IN, ARRAY) before GraalPy SQL parsing (#79028)
Alexander Polyankin9d391efca1b3
diff --git a/src/metabase/sql_parsing/core.clj b/src/metabase/sql_parsing/core.cljindex 99414315ae1..f9fa32d9e40 100644--- a/src/metabase/sql_parsing/core.clj+++ b/src/metabase/sql_parsing/core.clj@@ -39,19 +39,21 @@(boolean (and (:sql-parsing/error data)(= "ParseError" (:sql-parsing/python-error-type data))))))-;;; ----------------------------------------- VALUES clause stripping ------------------------------------------+;;; ------------------------------------- Large literal-list stripping ------------------------------------------;; Large VALUES clauses (thousands of tuples) cause GraalPy OOM due to ~114KB per AST node-;; (vs ~1-2KB on CPython). We strip them on the JVM side before passing SQL to Python,-;; because GraalPy is too slow at character-by-character string scanning over multi-MB inputs.+;; Large literal lists — VALUES clauses, IN lists (flat or tuple), and ARRAY literals with+;; thousands of items — cause GraalPy OOM/timeouts due to ~114KB per AST node (vs ~1-2KB on+;; CPython). We strip them on the JVM side before passing SQL to Python, because GraalPy is too+;; slow at character-by-character string scanning over multi-MB inputs.-(def ^:private ^:const values-strip-threshold- "Only strip VALUES clauses with more than this many tuples."+(def ^:private ^:const strip-threshold+ "Strip VALUES clauses with more than this many tuples, and IN lists / ARRAY literals with at+ least this many commas."100)-(def ^:private values-keyword-pattern- "Pattern to find VALUES keyword followed by opening paren."- (re-pattern "(?i)\\bVALUES\\s*\\("))+(def ^:private literal-list-keyword-pattern+ "Pattern to find a VALUES, IN, or ARRAY keyword followed by its opening delimiter."+ (re-pattern "(?i)\\b(?:VALUES\\s*\\(|IN\\s*\\(|ARRAY\\s*\\[)"))(defn- skip-whitespace"Return the first non-whitespace position at or after `pos`."@@ -61,47 +63,51 @@(recur (inc p))p)))+(defn- skip-string-literal+ "Starting at an opening quote (single or double), advance past the matching closing quote,+ treating a doubled quote as an escape. Returns the position immediately after the closing quote."+ ^long [^String sql ^long pos ^long n]+ (let [q (.charAt sql pos)]+ (loop [i (inc pos)]+ (if (>= i n)+ i+ (if (= (.charAt sql i) q)+ (if (and (< (inc i) n) (= (.charAt sql (inc i)) q))+ (recur (+ i 2)) ; escaped (doubled) quote+ (inc i))+ (recur (inc i)))))))+(defn- skip-balanced-parens"Starting at an opening `(`, advance past the matching `)`.- Handles nested parens and SQL string literals (single and double quoted, with doubled-quote escaping).- Returns the position immediately after the closing `)`."+ Handles nested parens and SQL string literals. Returns the position immediately after the closing `)`."^long [^String sql ^long pos ^long n]- (loop [pos pos, depth (int 0), in-str false, str-ch \space]+ (loop [pos pos, depth 0](if (>= pos n)pos(let [ch (.charAt sql pos)](cond- ;; Inside a string literal — look for the closing quote- in-str- (if (= ch str-ch)- (if (and (< (inc pos) n) (= (.charAt sql (inc pos)) str-ch))- (recur (+ pos 2) depth true str-ch) ; escaped (doubled) quote- (recur (inc pos) depth false str-ch))- (recur (inc pos) depth true str-ch))-- (or (= ch \') (= ch \")) (recur (inc pos) depth true ch)- (= ch \() (recur (inc pos) (inc depth) false str-ch)- (= ch \)) (if (= depth 1)- (inc pos) ; done- (recur (inc pos) (dec depth) false str-ch))- :else (recur (inc pos) depth false str-ch))))))+ (or (= ch \') (= ch \")) (recur (skip-string-literal sql pos n) depth)+ (= ch \() (recur (inc pos) (inc depth))+ (= ch \)) (if (= depth 1)+ (inc pos) ; done+ (recur (inc pos) (dec depth)))+ :else (recur (inc pos) depth))))))(defn- count-top-level-commas"Count top-level comma-separated items inside a tuple's content string.`(1, 'a', 3)` → inner content `1, 'a', 3` → 3 items."^long [^String content](let [n (.length content)]- (loop [i 0, depth (int 0), in-str false, str-ch \space, items (int 1)]+ (loop [i 0, depth 0, items 1](if (>= i n)items(let [ch (.charAt content i)](cond- in-str (recur (inc i) depth (not= ch str-ch) str-ch items)- (or (= ch \') (= ch \")) (recur (inc i) depth true ch items)- (= ch \() (recur (inc i) (inc depth) false str-ch items)- (= ch \)) (recur (inc i) (dec depth) false str-ch items)- (and (= ch \,) (= depth 0)) (recur (inc i) depth false str-ch (inc items))- :else (recur (inc i) depth false str-ch items)))))))+ (or (= ch \') (= ch \")) (recur (skip-string-literal content i n) depth items)+ (= ch \() (recur (inc i) (inc depth) items)+ (= ch \)) (recur (inc i) (dec depth) items)+ (and (= ch \,) (= depth 0)) (recur (inc i) depth (inc items))+ :else (recur (inc i) depth items)))))))(defn- count-and-skip-tuples"Starting after the first tuple, count how many more `, (...)` tuples follow.@@ -122,58 +128,113 @@(let [nulls (str/join ", " (repeat col-count "NULL"))](str original-keyword " (" nulls ")")))-(defn- extract-values-keyword- "Extract just the VALUES keyword text from a regex match like `VALUES (`."+(defn- extract-keyword+ "Extract just the keyword text from a regex match like `VALUES (`, `IN (`, or `ARRAY [`."^String [^String sql ^long match-start ^long match-end](-> (.substring sql match-start match-end)str/trimr- (str/replace #"\($" "")+ (str/replace #"[\(\[]$" "")str/trimr))-(defn- strip-large-values*- "Walk through SQL, replacing any VALUES clause with more than `values-strip-threshold`- tuples with a single-row NULL placeholder. Preserves column count from the first tuple."+(defn- strip-values-at+ "Decide whether to strip the VALUES clause whose `VALUES (` match spans [match-start, match-end).+ Returns [replacement resume-pos]: a single-row NULL placeholder (column count taken from the+ first tuple) covering the SQL up to resume-pos, or [nil match-end] to leave the clause untouched."+ [^String sql ^long match-start ^long match-end ^long n]+ (let [paren-start (dec match-end)+ first-end (skip-balanced-parens sql paren-start n)+ [tuple-count end-pos] (count-and-skip-tuples sql first-end n)]+ (if (and (> (long tuple-count) strip-threshold)+ (> first-end (inc paren-start)))+ [(make-null-placeholder (extract-keyword sql match-start match-end)+ (count-top-level-commas (.substring sql (inc paren-start) (dec first-end))))+ end-pos]+ [nil match-end])))++(defn- simple-list-end+ "Scan the list opening at `open-pos` (`(` or `[`). If it contains only numbers, single-quoted+ strings, signs, commas, whitespace, and balanced parens (VALUES-style tuples), return+ [comma-count end-pos] with end-pos just past the matching closing delimiter. Any other+ character — subqueries, column references, casts, bind parameters — returns nil, leaving the+ list untouched."+ [^String sql ^long open-pos ^long n]+ (let [close-ch (char (if (= (.charAt sql open-pos) \[) \] \)))]+ (loop [i (inc open-pos), depth 1, commas 0]+ (when (< i n)+ (let [ch (.charAt sql i)]+ (cond+ (and (= ch close-ch) (= depth 1)) [commas (inc i)]+ (= ch \') (recur (skip-string-literal sql i n) depth commas)+ (= ch \() (recur (inc i) (inc depth) commas)+ (= ch \)) (when (> depth 1) (recur (inc i) (dec depth) commas))+ (= ch \,) (recur (inc i) depth (inc commas))+ (or (Character/isDigit ch)+ (Character/isWhitespace ch)+ (= ch \.) (= ch \-) (= ch \+))+ (recur (inc i) depth commas)++ :else nil))))))++(defn- strip-list-at+ "Decide whether to strip the IN list or ARRAY literal whose keyword match spans+ [match-start, match-end). Returns [replacement resume-pos]: `IN (NULL)` / `ARRAY[NULL]`+ covering the whole list, or [nil match-end] to leave it untouched (resuming just inside the+ delimiter, so a large list nested in a subquery — `IN (SELECT ... WHERE x IN (...))` — is+ still found)."+ [^String sql ^long match-start ^long match-end ^long n]+ (let [open-pos (dec match-end)+ [commas end-pos] (simple-list-end sql open-pos n)]+ (if (and commas (>= (long commas) strip-threshold))+ [(str (extract-keyword sql match-start match-end)+ (if (= (.charAt sql open-pos) \[) "[NULL]" " (NULL)"))+ end-pos]+ [nil match-end])))++(defn- strip-large-literal-lists*+ "Single pass over `sql`, replacing every oversized literal list — VALUES clause or literal-only+ IN list — with a NULL placeholder. Returns `sql` itself when nothing was stripped."^String [^String sql]- (let [matcher (re-matcher values-keyword-pattern sql)- n (int (.length sql))]+ (let [matcher (re-matcher literal-list-keyword-pattern sql)+ n (.length sql)](if-not (.find matcher)sql- (let [_ (.reset matcher)- sb (StringBuilder.)]- (loop [i (int 0)]- (if-not (.find matcher i)- (-> sb (.append sql i n) .toString)- (let [match-start (.start matcher)- match-end (.end matcher)- _ (.append sb sql (int i) (int match-start))- ;; Parse the first tuple to learn its column count- paren-start (dec (int match-end))- first-end (skip-balanced-parens sql paren-start n)- first-inner (when (> first-end (inc paren-start))- (.substring sql (inc paren-start) (dec (int first-end))))- ;; Scan remaining tuples- [tuple-count end-pos] (count-and-skip-tuples sql first-end n)]- (if (and (> (int tuple-count) values-strip-threshold) first-inner)- (do (.append sb (make-null-placeholder- (extract-values-keyword sql match-start match-end)- (count-top-level-commas first-inner)))- (recur (int end-pos)))- (do (.append sb sql (int match-start) (int end-pos))- (recur (int end-pos)))))))))))--(defn strip-large-values- "Replace large VALUES clauses with a single-row NULL placeholder.-- Preserves the column count from the first tuple and all surrounding SQL structure.- Only triggers when a VALUES clause has more than `values-strip-threshold` tuples.+ (let [sb (StringBuilder.)]+ (loop [i 0, stripped? false, match? true]+ (if-not match?+ (if stripped?+ (-> sb (.append sql (int i) (int n)) .toString)+ sql)+ (let [match-start (.start matcher)+ match-end (.end matcher)+ [replacement resume] (let [ch (.charAt sql match-start)]+ (if (or (= ch \V) (= ch \v))+ (strip-values-at sql match-start match-end n)+ (strip-list-at sql match-start match-end n)))]+ (if replacement+ (-> sb (.append sql (int i) (int match-start)) (.append ^String replacement))+ (.append sb sql (int i) (int resume)))+ (recur (long resume)+ (or stripped? (some? replacement))+ (.find matcher (int resume))))))))))++(defn strip-large-literal-lists+ "Replace large literal lists with NULL placeholders: VALUES clauses with more than+ [[strip-threshold]] tuples become a single-row NULL tuple (preserving the column count from the+ first tuple), and simple IN lists / ARRAY literals — numbers, single-quoted strings, and+ VALUES-style tuples of them — with at least [[strip-threshold]] commas become `IN (NULL)` /+ `ARRAY[NULL]`. All surrounding SQL structure is preserved.This runs on the JVM side (fast) before passing SQL to GraalPy (slow at char scanning).- On any error, returns the original SQL unchanged so parsing can proceed normally."+ On any error, returns the original SQL unchanged so parsing can proceed normally.++ Best-effort by design: stripping only feeds fail-soft analysis, and execution paths restore the+ original SQL when anything was stripped ([[is-single-stmt-of-type?]]), so edge cases like+ keyword-shaped text inside string literals or comments are deliberately not guarded against."^String [^String sql](try- (strip-large-values* sql)+ (strip-large-literal-lists* sql)(catch Exception e- (log/warnf "Error stripping VALUES clauses, passing SQL through unchanged: %s" (ex-message e))+ (log/warnf "Error stripping large literal lists, passing SQL through unchanged: %s" (ex-message e))sql)));;; -------------------------------------------------- Public API --------------------------------------------------@@ -187,7 +248,7 @@This is the pure parsing layer - it returns what's literally in the SQL.Default schema resolution happens in the matching layer (core.clj)."[dialect sql]- (protocol/referenced-tables (parser) dialect (strip-large-values sql)))+ (protocol/referenced-tables (parser) dialect (strip-large-literal-lists sql)))(defn referenced-fields"Extract field references from SQL, returning only fields from actual database tables.@@ -213,7 +274,7 @@(referenced-fields \"bigquery\" \"SELECT * FROM myproject.analytics.events\")=> [[\"myproject\" \"analytics\" \"events\" \"*\"]]"[dialect sql]- (protocol/referenced-fields (parser) dialect (strip-large-values sql)))+ (protocol/referenced-fields (parser) dialect (strip-large-literal-lists sql)))(defn returned-columns-lineage"Extract column lineage from SQL query, showing which output columns depend on which source columns.@@ -233,7 +294,7 @@(returned-columns-lineage \"postgres\" \"SELECT id + 1 as computed FROM users\" nil schema)=> [[\"computed\" false [[[nil \"users\" \"id\"]]]]]"[dialect sql default-table-schema sqlglot-schema]- (protocol/returned-columns-lineage (parser) dialect (strip-large-values sql) default-table-schema sqlglot-schema))+ (protocol/returned-columns-lineage (parser) dialect (strip-large-literal-lists sql) default-table-schema sqlglot-schema))(defn validate-query"Validate a SQL query against a schema using sqlglot's qualify optimizer.@@ -265,7 +326,7 @@- \"invalid_expression\": Syntax/parse error- \"unhandled\": Other errors"[dialect sql default-table-schema & [sqlglot-schema]]- (protocol/validate-query (parser) dialect (strip-large-values sql) default-table-schema sqlglot-schema))+ (protocol/validate-query (parser) dialect (strip-large-literal-lists sql) default-table-schema sqlglot-schema))(defn simple-query?"Check if SQL is a simple SELECT without LIMIT, OFFSET, or CTEs.@@ -285,7 +346,7 @@(simple-query? nil \"SELECT * FROM users LIMIT 10\")=> {:is_simple false :reason \"Contains a LIMIT\"}"[dialect sql]- (protocol/simple-query (parser) dialect (strip-large-values sql)))+ (protocol/simple-query (parser) dialect (strip-large-literal-lists sql)))(defn add-into-clause"Add an INTO clause to a SELECT statement for SQL Server SELECT INTO syntax.@@ -417,7 +478,7 @@'fail soft' pattern used for parsing failures."[dialect sql](try- (let [raw (protocol/field-references (parser) dialect (strip-large-values sql))+ (let [raw (protocol/field-references (parser) dialect (strip-large-literal-lists sql))used-fields (or (:used-fields raw) (:used_fields raw) (get raw "used_fields") [])returned-fields (or (:returned-fields raw) (:returned_fields raw) (get raw "returned_fields") [])errors (or (:errors raw) (get raw "errors") [])]@@ -456,12 +517,13 @@"Validates that a query is a single read statement (SELECT) or a single write statement (INSERT, UPDATE, DELETE)and returns the query reconstructed from the parsed AST."[dialect sql stmt-type]- (let [stripped-sql (strip-large-values sql)+ (let [stripped-sql (strip-large-literal-lists sql)result (-> (protocol/single-stmt-of-type (parser) dialect stripped-sql stmt-type)(perf/update-keys (comp keyword u/->kebab-case-en)))];; The `:sql` in the `result` is the reconstructed SQL from the SQLGlot parser.- ;; We generally want to use the reconstructed SQL, but if the original SQL had its values stripped (to avoid GraalPy OOM)- ;; then we need to return the original SQL to preserve the values. (#74284)+ ;; We generally want to use the reconstructed SQL, but if the original SQL had its VALUES/IN+ ;; literal lists stripped (to avoid GraalPy OOM) then we need to return the original SQL to+ ;; preserve the values. (#74284)(cond-> result(not= sql stripped-sql) (assoc :sql sql))))diff --git a/test/metabase/sql_parsing/core_test.clj b/test/metabase/sql_parsing/core_test.cljindex 561569cc314..27dd4ac3f35 100644--- a/test/metabase/sql_parsing/core_test.clj+++ b/test/metabase/sql_parsing/core_test.clj@@ -1,6 +1,7 @@(ns metabase.sql-parsing.core-test(:require[clojure.java.io :as io]+ [clojure.string :as str][clojure.test :refer :all][metabase.sql-parsing.core :as sql-parsing][metabase.util :as u]))@@ -351,7 +352,6 @@"Nonexistent table reference is parsed and will fail at execution time"))))(comment- (require '[clojure.string :as str])(def query-corpus-path "/Users/bcm/dv/mb/query_corpus/")(def sentinel (re-pattern "\n-----end-query-----\n"))@@ -437,56 +437,328 @@"\n Expected: " (pr-str (normalize-fields expected))"\n Actual: " (pr-str (normalize-fields actual))))))))))-;;; ----------------------------------------- Large VALUES stripping tests -----------------------------------------+;;; -------------------------------------- Large literal-list stripping tests --------------------------------------(deftest ^:parallel strip-large-values-test(testing "Small VALUES clauses are preserved"(let [sql "SELECT * FROM (VALUES (1, 'a'), (2, 'b'), (3, 'c')) AS t(id, name)"]- (is (= sql (sql-parsing/strip-large-values sql)))))+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))(testing "No VALUES keyword returns SQL unchanged"(let [sql "SELECT * FROM users WHERE id = 1"]- (is (= sql (sql-parsing/strip-large-values sql)))))+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))(testing "Large VALUES clause is replaced with NULLs preserving column count"- (let [tuples (clojure.string/join ", " (map #(format "(%d, '%s', %d)" % (str "name" %) (* % 10))- (range 200)))+ (let [tuples (str/join ", " (map #(format "(%d, '%s', %d)" % (str "name" %) (* % 10))+ (range 200)))sql (str "SELECT * FROM (VALUES " tuples ") AS t(id, name, score)")- result (sql-parsing/strip-large-values sql)]- (is (clojure.string/includes? result "VALUES (NULL, NULL, NULL)"))- (is (clojure.string/includes? result "AS t(id, name, score)"))- (is (not (clojure.string/includes? result "name0")))))+ result (sql-parsing/strip-large-literal-lists sql)]+ (is (str/includes? result "VALUES (NULL, NULL, NULL)"))+ (is (str/includes? result "AS t(id, name, score)"))+ (is (not (str/includes? result "name0")))))(testing "Multiple large VALUES clauses are all stripped"- (let [tuples1 (clojure.string/join ", " (map #(format "(%d)" %) (range 200)))- tuples2 (clojure.string/join ", " (map #(format "(%d, %d)" % (* % 2)) (range 200)))+ (let [tuples1 (str/join ", " (map #(format "(%d)" %) (range 200)))+ tuples2 (str/join ", " (map #(format "(%d, %d)" % (* % 2)) (range 200)))sql (str "WITH a AS (SELECT * FROM (VALUES " tuples1 ") AS t(x)), ""b AS (SELECT * FROM (VALUES " tuples2 ") AS t(y, z)) ""SELECT * FROM a JOIN b ON a.x = b.y")- result (sql-parsing/strip-large-values sql)]- (is (clojure.string/includes? result "VALUES (NULL)"))- (is (clojure.string/includes? result "VALUES (NULL, NULL)"))))+ result (sql-parsing/strip-large-literal-lists sql)]+ (is (str/includes? result "VALUES (NULL)"))+ (is (str/includes? result "VALUES (NULL, NULL)"))))(testing "VALUES keyword casing is preserved"- (let [tuples (clojure.string/join ", " (map #(format "(%d)" %) (range 200)))+ (let [tuples (str/join ", " (map #(format "(%d)" %) (range 200)))sql (str "select * from (values " tuples ") as t(x)")- result (sql-parsing/strip-large-values sql)]- (is (clojure.string/includes? result "values (NULL)"))))+ result (sql-parsing/strip-large-literal-lists sql)]+ (is (str/includes? result "values (NULL)"))))(testing "VALUES inside a string literal is not stripped"(let [sql "SELECT 'INSERT INTO foo VALUES (1,2,3)' AS example FROM bar"]- (is (= sql (sql-parsing/strip-large-values sql)))))+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))(testing "Column named values is not stripped"(let [sql "SELECT values FROM my_table WHERE values > 10"]- (is (= sql (sql-parsing/strip-large-values sql)))))+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))(testing "VALUES keyword not followed by paren is not stripped"(let [sql "SELECT * FROM t WHERE col IN (SELECT values FROM other)"]- (is (= sql (sql-parsing/strip-large-values sql)))))+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))(testing "INSERT INTO ... VALUES is stripped when large"- (let [tuples (clojure.string/join ", " (map #(format "(%d, 'x')" %) (range 200)))+ (let [tuples (str/join ", " (map #(format "(%d, 'x')" %) (range 200)))sql (str "INSERT INTO foo VALUES " tuples)- result (sql-parsing/strip-large-values sql)]- (is (clojure.string/includes? result "VALUES (NULL, NULL)"))- (is (not (clojure.string/includes? result "(0, 'x')"))))))+ result (sql-parsing/strip-large-literal-lists sql)]+ (is (str/includes? result "VALUES (NULL, NULL)"))+ (is (not (str/includes? result "(0, 'x')"))))))++(deftest ^:parallel strip-large-in-lists-test+ (testing "Small IN lists are preserved"+ (let [sql "SELECT * FROM t WHERE id IN (1, 2, 3)"]+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))+ (testing "Large literal IN list is replaced with IN (NULL)"+ (let [sql (str "SELECT * FROM t WHERE id IN (" (str/join ", " (range 200)) ") AND active = true")+ result (sql-parsing/strip-large-literal-lists sql)]+ (is (= "SELECT * FROM t WHERE id IN (NULL) AND active = true" result))))+ (testing "Large IN list of strings, negatives, and decimals is stripped"+ (let [items (concat (map #(format "'name %d'" %) (range 100))+ (map #(str "-" %) (range 50))+ ["1.5" "+2"])+ sql (str "SELECT * FROM t WHERE name IN (" (str/join ", " items) ")")+ result (sql-parsing/strip-large-literal-lists sql)]+ (is (= "SELECT * FROM t WHERE name IN (NULL)" result))))+ (testing "IN keyword casing is preserved"+ (let [sql (str "SELECT * FROM t WHERE id in (" (str/join ", " (range 200)) ")")+ result (sql-parsing/strip-large-literal-lists sql)]+ (is (str/includes? result "in (NULL)"))))+ (testing "NOT IN with a large literal list is stripped"+ (let [sql (str "SELECT * FROM t WHERE id NOT IN (" (str/join ", " (range 200)) ")")+ result (sql-parsing/strip-large-literal-lists sql)]+ (is (= "SELECT * FROM t WHERE id NOT IN (NULL)" result))))+ (testing "Multiple large IN lists are all stripped"+ (let [in1 (str/join ", " (range 200))+ in2 (str/join ", " (map #(format "'x%d'" %) (range 200)))+ sql (str "SELECT * FROM t WHERE a IN (" in1 ") OR b IN (" in2 ")")+ result (sql-parsing/strip-large-literal-lists sql)]+ (is (= "SELECT * FROM t WHERE a IN (NULL) OR b IN (NULL)" result))))+ (testing "IN with a subquery is not stripped"+ (let [sql "SELECT * FROM t WHERE id IN (SELECT id FROM other)"]+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))+ (testing "A large IN list nested inside an unstripped IN subquery is still stripped"+ (let [sql (str "SELECT * FROM t WHERE id IN (SELECT id FROM other WHERE x IN ("+ (str/join ", " (range 200)) "))")+ result (sql-parsing/strip-large-literal-lists sql)]+ (is (= "SELECT * FROM t WHERE id IN (SELECT id FROM other WHERE x IN (NULL))" result))))+ (testing "Large IN lists containing non-literals are not stripped"+ (doseq [extra-item ["col_ref" "foo(1)" "?" "$1" "{{param}}" "1e5" "DATE '2024-01-01'" "NULL"]]+ (let [sql (str "SELECT * FROM t WHERE id IN (" (str/join ", " (concat (range 200) [extra-item])) ")")]+ (is (= sql (sql-parsing/strip-large-literal-lists sql))+ (str "list containing " (pr-str extra-item) " should not be stripped")))))+ (testing "Word boundaries: JOIN ( and INT( do not trigger IN matching"+ (let [sql "SELECT CAST(x AS INT) FROM a JOIN (SELECT * FROM b) c ON a.id = c.id"]+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))+ (testing "Large VALUES and IN in the same statement are both stripped"+ (let [tuples (str/join ", " (map #(format "(%d)" %) (range 200)))+ sql (str "WITH v AS (SELECT * FROM (VALUES " tuples ") AS t(x)) "+ "SELECT * FROM v WHERE x IN (" (str/join ", " (range 200)) ")")+ result (sql-parsing/strip-large-literal-lists sql)]+ (is (= "WITH v AS (SELECT * FROM (VALUES (NULL)) AS t(x)) SELECT * FROM v WHERE x IN (NULL)" result)))))++(deftest ^:parallel large-in-list-referenced-tables-test+ (testing "referenced-tables and referenced-fields work on queries with massive IN lists"+ (let [sql (str "SELECT a.name FROM accounts a WHERE a.id IN (" (str/join ", " (range 20000)) ")")]+ (is (= [[nil nil "accounts"]] (sql-parsing/referenced-tables "postgres" sql)))+ (is (= [[nil nil "accounts" "id"] [nil nil "accounts" "name"]]+ (sort (sql-parsing/referenced-fields "postgres" sql)))))))++(defn- in-list+ "Build `IN (0, 1, ..., n-1)` with `n` items."+ [n]+ (str "IN (" (str/join ", " (range n)) ")"))++(deftest ^:parallel strip-threshold-boundary-test+ (testing "IN lists at or below the 100-item threshold are preserved, above it stripped"+ (doseq [n [1 2 50 99 100]]+ (let [sql (str "SELECT * FROM t WHERE id " (in-list n))]+ (is (= sql (sql-parsing/strip-large-literal-lists sql))+ (str n " items should not be stripped"))))+ (doseq [n [101 102 150 1000 10000]]+ (let [sql (str "SELECT * FROM t WHERE id " (in-list n))]+ (is (= "SELECT * FROM t WHERE id IN (NULL)" (sql-parsing/strip-large-literal-lists sql))+ (str n " items should be stripped")))))+ (testing "VALUES at or below the 100-tuple threshold are preserved, above it stripped"+ (doseq [[n stripped?] {1 false, 100 false, 101 true, 1000 true}]+ (let [tuples (str/join ", " (map #(format "(%d, %d)" % %) (range n)))+ sql (str "SELECT * FROM (VALUES " tuples ") AS t(x, y)")+ result (sql-parsing/strip-large-literal-lists sql)]+ (if stripped?+ (is (= "SELECT * FROM (VALUES (NULL, NULL)) AS t(x, y)" result)+ (str n " tuples should be stripped"))+ (is (= sql result)+ (str n " tuples should not be stripped")))))))++(deftest ^:parallel strip-in-list-literal-shapes-test+ (testing "Lists of each literal shape are stripped"+ (doseq [[shape item-fn] {"integers" str+ "decimals" #(str % ".5")+ "negatives" #(str "-" %)+ "plus-signed" #(str "+" %)+ "single-quoted" #(format "'name %d'" %)+ "commas-inside-strings" #(format "'a,%d,b'" %)+ "escaped-quotes" #(format "'it''s %d'" %)}]+ (let [sql (str "SELECT * FROM t WHERE x IN (" (str/join ", " (map item-fn (range 150))) ")")]+ (is (= "SELECT * FROM t WHERE x IN (NULL)" (sql-parsing/strip-large-literal-lists sql))+ (str shape " should be stripped")))))+ (testing "Whitespace variants are stripped"+ (doseq [sql [(str "SELECT * FROM t WHERE x IN(" (str/join "," (range 150)) ")")+ (str "SELECT * FROM t WHERE x IN\n (" (str/join " ,\n" (range 150)) ")")+ (str "SELECT * FROM t WHERE x in (" (str/join ",\t" (range 150)) ")")]]+ (is (str/includes? (sql-parsing/strip-large-literal-lists sql) "(NULL)")+ (pr-str (subs sql 0 40))))))++(deftest ^:parallel strip-in-list-disqualifiers-test+ (testing "One non-literal item anywhere in a large list prevents stripping"+ (doseq [bad ["col_ref" "some_column" "foo(1)" "?" "$1" "{{param}}" "%(name)s"+ "1e5" "NULL" "DATE '2024-01-01'" "TIMESTAMP '2024-01-01 00:00:00'"+ "1::int" "x.y" "CURRENT_DATE" "-- comment" "héllo" "N'abc'"+ "\"quoted_identifier\"" "null1"]+ position [:first :middle :last]]+ (let [items (map str (range 150))+ items (case position+ :first (cons bad items)+ :middle (concat (take 75 items) [bad] (drop 75 items))+ :last (concat items [bad]))+ sql (str "SELECT * FROM t WHERE x IN (" (str/join ", " items) ")")]+ (is (= sql (sql-parsing/strip-large-literal-lists sql))+ (str (pr-str bad) " at " position " should prevent stripping")))))+ (testing "Subqueries are never stripped regardless of size"+ (let [sql "SELECT * FROM t WHERE x IN (SELECT id FROM u WHERE y = 'a' AND z IN ('b', 'c'))"]+ (is (= sql (sql-parsing/strip-large-literal-lists sql)))))+ (testing "A large list of double-quoted identifiers is not stripped (they are column references+ in identifier-quoting dialects, not strings)"+ (let [sql (str "SELECT * FROM t WHERE x IN (" (str/join ", " (map #(format "\"col_%d\"" %) (range 150))) ")")]+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql))))))++(deftest ^:parallel strip-preserves-surroundings-test+ (testing "Everything around a stripped list is preserved byte-for-byte"+ (let [prefix "WITH cte AS (SELECT 1 AS x) SELECT t.a, cte.x FROM t JOIN cte ON TRUE WHERE t.id "+ suffix " AND t.status = 'active' ORDER BY t.a LIMIT 5"+ sql (str prefix (in-list 150) suffix)]+ (is (= (str prefix "IN (NULL)" suffix)+ (sql-parsing/strip-large-literal-lists sql)))))+ (testing "Several lists in one statement are stripped/kept independently"+ (let [sql (str "SELECT * FROM t WHERE a " (in-list 150)+ " OR b " (in-list 3)+ " OR c NOT " (in-list 200)+ " OR d IN (SELECT x FROM u)")]+ (is (= (str "SELECT * FROM t WHERE a IN (NULL)"+ " OR b " (in-list 3)+ " OR c NOT IN (NULL)"+ " OR d IN (SELECT x FROM u)")+ (sql-parsing/strip-large-literal-lists sql))))))++(deftest ^:parallel strip-nesting-test+ (testing "A large IN list inside a small VALUES tuple is stripped"+ (let [sql (str "SELECT * FROM (VALUES ((SELECT count(*) FROM u WHERE u.id " (in-list 150) "))) t")]+ (is (= "SELECT * FROM (VALUES ((SELECT count(*) FROM u WHERE u.id IN (NULL)))) t"+ (sql-parsing/strip-large-literal-lists sql)))))+ (testing "A large IN list inside a stripped VALUES clause disappears with it"+ (let [tuples (str/join ", " (map #(format "(%d)" %) (range 150)))+ sql (str "SELECT * FROM (VALUES " tuples ", ((SELECT 1 WHERE x " (in-list 150) "))) t")+ result (sql-parsing/strip-large-literal-lists sql)]+ (is (= "SELECT * FROM (VALUES (NULL)) t" result))))+ (testing "IN subqueries nest arbitrarily deep and the innermost large list is still found"+ (let [sql (str "SELECT * FROM a WHERE x IN "+ "(SELECT x FROM b WHERE y IN "+ "(SELECT y FROM c WHERE z " (in-list 150) "))")]+ (is (= (str "SELECT * FROM a WHERE x IN "+ "(SELECT x FROM b WHERE y IN "+ "(SELECT y FROM c WHERE z IN (NULL)))")+ (sql-parsing/strip-large-literal-lists sql))))))++(deftest ^:parallel strip-idempotency-and-identity-test+ (testing "Stripping is idempotent"+ (doseq [sql [(str "SELECT * FROM t WHERE id " (in-list 150))+ (str "INSERT INTO foo VALUES " (str/join ", " (map #(format "(%d)" %) (range 150))))]]+ (let [once (sql-parsing/strip-large-literal-lists sql)]+ (is (= once (sql-parsing/strip-large-literal-lists once))))))+ (testing "When nothing is stripped the original string instance is returned"+ (doseq [sql ["SELECT * FROM t"+ "SELECT * FROM t WHERE id IN (1, 2, 3)"+ "SELECT * FROM t WHERE id IN (SELECT id FROM u)"+ "SELECT * FROM (VALUES (1), (2)) AS t(x)"]]+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql))))))++(deftest ^:parallel strip-malformed-sql-test+ (testing "Malformed or degenerate input never throws and is left unchanged"+ (doseq [sql [""+ "IN ("+ "VALUES ("+ (str "SELECT * FROM t WHERE id IN (" (str/join ", " (range 150))) ; no closing paren+ (str "SELECT * FROM t WHERE id IN ('unterminated string, " (str/join ", " (range 150)) ")")+ "SELECT * FROM t WHERE id IN ()"+ "IN () VALUES ()"+ "in( )"]]+ (is (= sql (sql-parsing/strip-large-literal-lists sql))+ (pr-str (subs sql 0 (min 40 (count sql))))))))++(deftest ^:parallel strip-keyword-false-positives-test+ (testing "Words containing 'in'/'values' and quoted identifiers are not treated as keywords"+ (doseq [sql ["SELECT margin FROM t WHERE margin > 10"+ "SELECT * FROM t JOIN (SELECT * FROM u) v ON t.id = v.id"+ "SELECT CAST(x AS INT) FROM t"+ "SELECT * FROM logins (1, 2, 3)"+ "SELECT \"in\" FROM t"+ "SELECT t.values FROM t WHERE t.values > 10"]]+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql)) sql))))++(deftest ^:parallel strip-tuple-in-list-test+ (testing "Tuple IN lists below the comma threshold are preserved, above it stripped"+ ;; n two-element tuples contain 2n-1 commas; the threshold is 100 commas, so 51 tuples strip+ (doseq [[n stripped?] {1 false, 50 false, 51 true, 1000 true}]+ (let [tuples (str/join ", " (map #(format "(%d, %d)" % (* % 2)) (range n)))+ sql (str "SELECT * FROM t WHERE (a, b) IN (" tuples ")")+ result (sql-parsing/strip-large-literal-lists sql)]+ (if stripped?+ (is (= "SELECT * FROM t WHERE (a, b) IN (NULL)" result)+ (str n " tuples should be stripped"))+ (is (identical? sql result)+ (str n " tuples should not be stripped"))))))+ (testing "NOT IN and string tuples"+ (let [tuples (str/join ", " (map #(format "(%d, '%d')" % %) (range 150)))+ sql (str "SELECT * FROM t WHERE (a, b) NOT IN (" tuples ")")]+ (is (= "SELECT * FROM t WHERE (a, b) NOT IN (NULL)"+ (sql-parsing/strip-large-literal-lists sql)))))+ (testing "Tuple lists containing non-simple tuples are not stripped"+ (doseq [bad ["(SELECT 1, 2)" "(x, 1)" "(foo(1), 2)" "(?, ?)"]]+ (let [tuples (str/join ", " (concat (map #(format "(%d, %d)" % %) (range 150)) [bad]))+ sql (str "SELECT * FROM t WHERE (a, b) IN (" tuples ")")]+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql))+ (str "tuple list containing " (pr-str bad) " should not be stripped"))))))++(defn- array-literal+ "Build `ARRAY[0, 1, ..., n-1]` with `n` items."+ [n]+ (str "ARRAY[" (str/join ", " (range n)) "]"))++(deftest ^:parallel strip-array-literal-test+ (testing "ARRAY literals at or below the threshold are preserved, above it stripped"+ (doseq [n [1 100]]+ (let [sql (str "SELECT * FROM t WHERE id = ANY(" (array-literal n) ")")]+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql))+ (str n " items should not be stripped"))))+ (doseq [n [101 1000]]+ (let [sql (str "SELECT * FROM t WHERE id = ANY(" (array-literal n) ")")]+ (is (= "SELECT * FROM t WHERE id = ANY(ARRAY[NULL])" (sql-parsing/strip-large-literal-lists sql))+ (str n " items should be stripped")))))+ (testing "Casing and string arrays"+ (let [sql (str "SELECT array['" (str/join "', '" (range 150)) "'] AS a FROM t")]+ (is (= "SELECT array[NULL] AS a FROM t" (sql-parsing/strip-large-literal-lists sql)))))+ (testing "Array literals containing non-literals are not stripped"+ (doseq [bad ["col_ref" "foo(1)" "ARRAY[1]" "1:5"]]+ (let [sql (str "SELECT ARRAY[" (str/join ", " (concat (range 150) [bad])) "] FROM t")]+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql))+ (str "array containing " (pr-str bad) " should not be stripped")))))+ (testing "Array subscript access on a column named array is untouched"+ (doseq [sql ["SELECT array[1] FROM t"+ "SELECT arr[1] FROM t"+ "SELECT arr[1:5] FROM t"]]+ (is (identical? sql (sql-parsing/strip-large-literal-lists sql)) sql)))+ (testing "A large ARRAY nested inside an unstripped IN subquery is still stripped"+ (let [sql (str "SELECT * FROM t WHERE id IN (SELECT id FROM u WHERE tags && " (array-literal 150) ")")]+ (is (= "SELECT * FROM t WHERE id IN (SELECT id FROM u WHERE tags && ARRAY[NULL])"+ (sql-parsing/strip-large-literal-lists sql))))))++(deftest ^:parallel stripped-sql-parses-e2e-test+ (testing "every analysis entry point tolerates SQL whose lists were stripped"+ (let [sql (str "SELECT a.name FROM accounts a WHERE a.id " (in-list 150))]+ (is (= [[nil nil "accounts"]] (sql-parsing/referenced-tables "postgres" sql)))+ (is (= {:is_simple true} (sql-parsing/simple-query? "postgres" sql)))+ (is (=? {:status "ok"} (sql-parsing/validate-query "postgres" sql nil)))+ (is (=? {:used-fields set? :returned-fields vector? :errors #{}}+ (sql-parsing/field-references "postgres" sql)))))+ (testing "tuple IN lists and ARRAY literals also parse after stripping"+ (doseq [sql [(str "SELECT a.name FROM accounts a WHERE (a.id, a.org_id) IN ("+ (str/join ", " (map #(format "(%d, %d)" % %) (range 20000))) ")")+ (str "SELECT a.name FROM accounts a WHERE a.id = ANY(" (array-literal 20000) ")")]]+ (is (= [[nil nil "accounts"]] (sql-parsing/referenced-tables "postgres" sql)))+ (is (=? {:errors #{}} (sql-parsing/field-references "postgres" sql))))))(deftest ^:parallel large-values-referenced-tables-test(testing "referenced-tables works on queries with massive VALUES clauses"- (let [tuples (clojure.string/join ", " (map #(format "(%d, %d)" % (mod % 100)) (range 20000)))+ (let [tuples (str/join ", " (map #(format "(%d, %d)" % (mod % 100)) (range 20000)))sql (str "WITH lookup AS (SELECT * FROM (VALUES " tuples") AS v(id, score)) ""SELECT a.name, l.score FROM accounts a "diff --git a/test/metabase/sql_tools/core_test.clj b/test/metabase/sql_tools/core_test.cljindex 90c0d339274..d780c9e15e7 100644--- a/test/metabase/sql_tools/core_test.clj+++ b/test/metabase/sql_tools/core_test.clj@@ -241,4 +241,14 @@(str "SELECT 1; " values-query) false false(str "SET ROLE none; " values-query) false false(str values-query "; SELECT 1") false false- (str values-query "; SET ROLE none") false false))))+ (str values-query "; SET ROLE none") false false)))+ (testing "we don't remove large IN lists, tuple lists, or arrays when validating impersonated queries"+ (doseq [query [(str "SELECT x FROM t WHERE x IN (" (str/join ", " (range 105)) ")")+ (str "SELECT x FROM t WHERE (x, y) IN (" (str/join ", " (map #(format "(%d, %d)" % %) (range 105))) ")")+ (str "SELECT x FROM t WHERE x = ANY(ARRAY[" (str/join ", " (range 105)) "])")]]+ (are [sql is-single-stmt? allowed-stmt-type?]+ (= {:is-single-stmt? is-single-stmt? :allowed-stmt-type? allowed-stmt-type? :sql sql}+ (sql-tools/is-single-stmt-of-type? :postgres sql "read"))+ query true true+ (str "SELECT 1; " query) false false+ (str query "; SELECT 1") false false))))