Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add support for table-functions and selecting tables directly (#65074)

William39ceced98f87
diff --git a/deps.edn b/deps.edn
index 9d77f71d052..b01b9d6f7f2 100644
--- a/deps.edn
+++ b/deps.edn
@@ -90,7 +90,7 @@
:git/sha "4998a1fdd9d3713d7034d04509e9212204773bf2"
:git/url "https://github.com/eerohele/pp"}
;; TODO: replace back to normal artifact
- io.github.metabase/macaw {:mvn/version "0.2.28"} ; Parse native SQL queries
+ io.github.metabase/macaw {:mvn/version "0.2.29"} ; Parse native SQL queries
io.github.resilience4j/resilience4j-retry ^:antq/exclude ; 2.x do not support Java 8
{:mvn/version "1.7.1"} ; Support for retrying operations
io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"} ; prometheus jvm collector
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
index ebf53f41b05..224e0713e34 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
@@ -87,6 +87,15 @@
driver
(fake-query mp "select bad from products"))))))))
+(deftest ^:parallel validate-table-function-query-test
+ (testing "can validate queries using table functions"
+ (let [mp (deps.tu/default-metadata-provider)
+ driver (:engine (lib.metadata/database mp))]
+ (is (= []
+ (deps.native-validation/validate-native-query
+ driver
+ (fake-query mp "select i from my_function(1, 100)")))))))
+
(deftest ^:parallel validate-native-query-with-subquery-columns-test
(testing "validate-native-query should detect invalid columns in subqueries"
(let [mp (deps.tu/default-metadata-provider)
@@ -348,6 +357,15 @@
:base-type :type/Text,
:effective-type :type/Text,
:semantic-type :type/Category}]))
+ (testing "Using a table function"
+ (check-result-metadata
+ driver mp
+ "select ids from my_function(1, 100)"
+ [{:name "IDS",
+ :lib/desired-column-alias "IDS",
+ :display-name "Ids",
+ :effective-type :type/*,
+ :semantic-type :Semantic/*}]))
(testing "Selecting a bad table wildcard"
(check-result-metadata
driver mp
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index f6ae81e4aba..c359d708ee8 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -225,7 +225,21 @@
;; inner query, it can also refer to something in the outer query.
;; sql.references/field-references organizes source-cols into a list of lists
;; to account for this.
- (->> (mapcat (partial resolve-field driver metadata-provider) source-col-set)
+ (->> (mapcat (fn [current-col]
+ ;; :unknown-columns is a placeholder for "we know there are columns being
+ ;; returned, but have no way of knowing what those are -- this is primarily
+ ;; used for table-functions like `select * from my_func()`. If we encounter
+ ;; something like that, assume that the query is valid and make up a matching
+ ;; column to avoid false positives.
+ (if (= (:type current-col) :unknown-columns)
+ (let [name (:column col-spec)]
+ [{:base-type :type/*
+ :name name
+ :display-name (->> name (u.humanization/name->human-readable-name :simple))
+ :effective-type :type/*
+ :semantic-type :Semantic/*}])
+ (resolve-field driver metadata-provider current-col)))
+ source-col-set)
(some #(when (= (:name %) (:column col-spec))
%))))))]
(assoc found :lib/desired-column-alias (or alias name))
@@ -272,6 +286,10 @@
:effective-type (apply lca :type/* (map :effective-type member-fields))
:semantic-type (apply lca :Semantic/* (map :semantic-type member-fields))}]))
+(defmethod resolve-field :unknown-columns
+ [_driver _metadata-provider _col-spec]
+ [])
+
(mu/defmethod driver/native-result-metadata :sql
[driver :- :keyword
native-query :- :metabase.lib.schema/native-only-query]
diff --git a/src/metabase/driver/sql/references.clj b/src/metabase/driver/sql/references.clj
index e13dcc90018..cf09c05d5c2 100644
--- a/src/metabase/driver/sql/references.clj
+++ b/src/metabase/driver/sql/references.clj
@@ -23,6 +23,10 @@
[:database {:optional true} :string]
[:table-alias {:optional true} :string]]]])
+(mr/def ::unknown-columns
+ [:map
+ [:type [:= :unknown-columns]]])
+
(mr/def ::custom-field
[:map
[:type [:= :custom-field]]
@@ -47,6 +51,7 @@
[:multi {:dispatch :type}
[:single-column [:ref ::single-column]]
[:all-columns [:ref ::all-columns]]
+ [:unknown-columns [:ref ::unknown-columns]]
[:custom-field [:ref ::custom-field]]
[:composite-field [:ref ::composite-field]]
[:invalid-table-wildcard [:ref ::invalid-table-wildcard]]])
@@ -82,7 +87,7 @@
sublist))
sources))
-(defn- get-column [driver sources raw-col]
+(defn- get-column [driver sources raw-col {:keys [return-table-matches?]}]
(if (and (nil? (:table raw-col))
(nil? (:schema raw-col))
(nil? (:database raw-col))
@@ -100,14 +105,27 @@
valid-sources)
source-column (some #(when (= column (or (:alias %) (:column %)))
%)
- (first source-columns))]
- [(if source-column
- (cond-> source-column
- alias (assoc :alias alias))
- (-> {:column column
- :alias alias
- :type :single-column}
- (assoc :source-columns source-columns)))])))
+ (first source-columns))
+ source-matches (when (nil? table)
+ (some (fn [inner-sources]
+ (some #(when (= column (or (-> % :names :table-alias) (-> % :names :table)))
+ (:returned-fields %))
+ inner-sources))
+ sources))]
+ (cond
+ ;; we have a direct column match
+ source-column [(cond-> source-column
+ alias (assoc :alias alias))]
+ ;; we have a match to a table
+ source-matches (when return-table-matches?
+ [{:type :custom-field
+ :alias (or alias column)
+ :used-fields (set source-matches)}])
+ ;; we don't have a direct match, so show the possible columns this could match to
+ :else [(-> {:column column
+ :alias alias
+ :type :single-column}
+ (assoc :source-columns source-columns))]))))
(defmulti find-used-fields
"Finds the fields used in a given sql expression."
@@ -122,7 +140,7 @@
(defmethod find-used-fields [:sql :macaw.ast/column]
[driver sources _withs expr]
- (get-column driver sources expr))
+ (get-column driver sources expr {:return-table-matches? false}))
(defmethod find-used-fields [:sql :macaw.ast/unary-expression]
[driver sources withs expr]
@@ -190,7 +208,7 @@
(defmethod find-returned-fields [:sql :macaw.ast/column]
[driver sources _withs expr]
- (get-column driver sources expr))
+ (get-column driver sources expr {:return-table-matches? true}))
(defmethod find-returned-fields [:sql :macaw.ast/wildcard]
[_driver sources _withs _expr]
@@ -300,6 +318,13 @@
:names (when-let [alias (:table-alias expr)]
{:table-alias (sql.normalize/normalize-name driver alias)})})
+(defmethod field-references-impl [:sql :macaw.ast/table-function]
+ [driver _outside-sources _withs expr]
+ {:used-fields #{}
+ :returned-fields [{:type :unknown-columns}]
+ :names (when-let [alias (:table-alias expr)]
+ {:table-alias (sql.normalize/normalize-name driver alias)})})
+
(defmethod field-references-impl :default
[_driver _outside-sources _outside-withs _expr]
{:used-fields #{}
diff --git a/test/metabase/driver/sql/references_test.clj b/test/metabase/driver/sql/references_test.clj
index 111559da457..cce2360162b 100644
--- a/test/metabase/driver/sql/references_test.clj
+++ b/test/metabase/driver/sql/references_test.clj
@@ -7,13 +7,13 @@
(defn- ->references [query]
(->> query macaw/parsed-query macaw/->ast (sql.references/field-references :sql)))
-(deftest garbage-test
+(deftest ^:parallel garbage-test
(is (= {:used-fields #{}
:returned-fields []
:bad-sql true}
(->references "nothing"))))
-(deftest basic-select-test
+(deftest ^:parallel basic-select-test
(is (= {:used-fields
#{{:column "a",
:alias nil,
@@ -34,7 +34,7 @@
:source-columns [[{:type :all-columns, :table {:table "products"}}]]}]}
(->references "select * from (select a, b from products)"))))
-(deftest basic-join-test
+(deftest ^:parallel basic-join-test
(is (= {:used-fields
#{{:column "product_id",
:alias nil,
@@ -56,7 +56,7 @@
:source-columns [[{:type :all-columns, :table {:table "orders"}}]]}]}
(->references "select products.*, orders.id from products inner join orders on products.id = orders.product_id"))))
-(deftest indeterminite-join-test
+(deftest ^:parallel indeterminite-join-test
(is (= {:used-fields
#{{:column "id",
:alias nil,
@@ -81,7 +81,7 @@
{:type :all-columns, :table {:table "products"}}]]}]}
(->references "select id from products inner join orders on products.id = orders.product_id"))))
-(deftest table-wildcard-join-test
+(deftest ^:parallel table-wildcard-join-test
(is (= {:used-fields
#{{:column "id",
:alias nil,
@@ -97,7 +97,7 @@
[{:type :all-columns, :table {:table "orders"}}]}
(->references "select orders.* from products inner join orders on products.id = orders.product_id"))))
-(deftest wildcard-join-test
+(deftest ^:parallel wildcard-join-test
(is (= {:used-fields
#{{:column "id",
:alias nil,
@@ -114,7 +114,7 @@
{:type :all-columns, :table {:table "products"}}]}
(->references "select * from products inner join orders on products.id = orders.product_id"))))
-(deftest basic-alias-test
+(deftest ^:parallel basic-alias-test
(is (= {:used-fields
#{{:column "c",
:alias "d",
@@ -135,7 +135,7 @@
:source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}]}
(->references "select p.a as b, p.c as d from products p"))))
-(deftest basic-nested-query-test
+(deftest ^:parallel basic-nested-query-test
(is (= {:used-fields
#{{:column "b",
:alias nil,
@@ -160,7 +160,7 @@
:table {:table "products"}}]]}]}
(->references "select a, b from (select a, b from products)"))))
-(deftest renamed-nested-query-test
+(deftest ^:parallel renamed-nested-query-test
(is (= {:used-fields
#{{:column "a",
:alias "b",
@@ -180,7 +180,7 @@
:table {:table "products"}}]]}]}
(->references "select b as c from (select a as b from products)"))))
-(deftest broken-nested-query-test
+(deftest ^:parallel broken-nested-query-test
(is (= {:used-fields
#{{:column "b",
:alias nil,
@@ -206,7 +206,7 @@
:source-columns [[{:type :all-columns, :table {:table "products"}}]]}]]}]}
(->references "select a from (select b from products)"))))
-(deftest different-case-nested-query-test
+(deftest ^:parallel different-case-nested-query-test
(is (= {:used-fields
#{{:column "a",
:alias nil,
@@ -221,7 +221,7 @@
:table {:table "orders"}}]]}]}
(->references "select A from (select a from orders)"))))
-(deftest wildcard-nested-query-test
+(deftest ^:parallel wildcard-nested-query-test
(is (= {:used-fields
#{{:column "a",
:alias nil,
@@ -236,7 +236,7 @@
:table {:table "orders"}}]]}]}
(->references "select * from (select a from orders)"))))
-(deftest table-wildcard-nested-query-test
+(deftest ^:parallel table-wildcard-nested-query-test
(is (= {:used-fields
#{{:column "a",
:alias nil,
@@ -251,7 +251,7 @@
:table {:table "orders"}}]]}]}
(->references "select o.* from (select a from orders) o"))))
-(deftest bad-table-wildcard-nested-query-test
+(deftest ^:parallel bad-table-wildcard-nested-query-test
(is (= {:used-fields
#{{:column "a",
:alias nil,
@@ -262,7 +262,7 @@
[{:type :invalid-table-wildcard, :table "foo"}]}
(->references "select foo.* from (select a from orders)"))))
-(deftest bad-table-name-test
+(deftest ^:parallel bad-table-name-test
(is (= {:used-fields
#{{:column "a",
:alias nil,
@@ -275,7 +275,7 @@
:source-columns []}]}
(->references "select bad.a from products"))))
-(deftest basic-where-test
+(deftest ^:parallel basic-where-test
(is (= {:used-fields
#{{:column "category",
:alias nil,
@@ -286,17 +286,17 @@
:table {:table "products"}}]}
(->references "select * from products where category = 'hello'"))))
-(deftest basic-aggregation-test
+(deftest ^:parallel basic-aggregation-test
(is (=? {:used-fields #{},
:returned-fields [{:alias nil, :type :custom-field, :used-fields #{}}]}
(->references "select count(*) from products"))))
-(deftest named-aggregation-test
+(deftest ^:parallel named-aggregation-test
(is (=? {:used-fields #{},
:returned-fields [{:alias "count", :type :custom-field, :used-fields #{}}]}
(->references "select count(*) as count from products"))))
-(deftest extra-names-test
+(deftest ^:parallel extra-names-test
(is (= {:used-fields
#{{:column "col",
:alias nil,
@@ -317,7 +317,7 @@
:table "table"}}]]}]}
(->references "select db.schema.table.col from db.schema.table"))))
-(deftest basic-grouping-test
+(deftest ^:parallel basic-grouping-test
(is (= {:used-fields
#{{:column "category",
:alias nil,
@@ -346,7 +346,7 @@
:source-columns [[{:type :all-columns, :table {:table "orders"}}]]}}}]}
(->references "select sum(total) as sum from orders group by category"))))
-(deftest grouping-order-by-test
+(deftest ^:parallel grouping-order-by-test
(is (= {:used-fields
#{{:alias "cards_created",
:type :custom-field,
@@ -385,7 +385,7 @@
:source-columns [[{:type :all-columns, :table {:table "report_card"}}]]}}}]}
(->references "SELECT creator_id, COUNT(creator_id) AS cards_created FROM report_card GROUP BY creator_id ORDER BY cards_created DESC"))))
-(deftest basic-arg-test
+(deftest ^:parallel basic-arg-test
(is (= {:used-fields
#{{:column "category",
:alias nil,
@@ -396,7 +396,7 @@
:table {:table "products"}}]}
(->references "select * from products where category = ?"))))
-(deftest basic-case-test
+(deftest ^:parallel basic-case-test
(is (=? {:used-fields
#{{:column "tax",
:alias nil,
@@ -434,7 +434,7 @@
:table {:table "orders"}}]]}}}]}
(->references "select case when total < 0 then -subtotal else tax end from orders"))))
-(deftest switch-case-test
+(deftest ^:parallel switch-case-test
(is (=? {:used-fields
#{{:column "category",
:alias nil,
@@ -452,7 +452,7 @@
:table {:table "products"}}]]}}}]}
(->references "select case category when 'Gizmo' then 'is gizmo' else 'is not gizmo' end from products"))))
-(deftest basic-select-subquery-test
+(deftest ^:parallel basic-select-subquery-test
(is (= {:used-fields
#{{:column "product_id",
:alias nil,
@@ -482,7 +482,7 @@
{:type :all-columns, :table {:table "orders"}}]}
(->references "select (select category from products where products.id = orders.product_id), * from orders"))))
-(deftest named-select-subquery-test
+(deftest ^:parallel named-select-subquery-test
(is (= {:used-fields
#{{:column "product_id",
:alias nil,
@@ -512,7 +512,7 @@
{:type :all-columns, :table {:table "orders"}}]}
(->references "select (select category from products where products.id = orders.product_id) as category2, * from orders"))))
-(deftest nested-select-subquery-test
+(deftest ^:parallel nested-select-subquery-test
(is (= {:used-fields
#{{:column "product_id",
:alias nil,
@@ -542,7 +542,7 @@
{:type :all-columns, :table {:table "orders"}}]}
(->references "select (select (select category from products where products.id = orders.product_id)), * from orders"))))
-(deftest select-subquery-with-normal-subquery-test
+(deftest ^:parallel select-subquery-with-normal-subquery-test
(is (= {:used-fields
#{{:column "product_id",
:alias nil,
@@ -572,7 +572,7 @@
{:type :all-columns, :table {:table "orders"}}]}
(->references "select (select * from (select category from products where products.id = orders.product_id) sub), * from orders"))))
-(deftest nested-select-subquery-with-reference-to-middle-select-test
+(deftest ^:parallel nested-select-subquery-with-reference-to-middle-select-test
(is (= {:used-fields
#{{:column "id",
:alias nil,
@@ -597,7 +597,7 @@
[{:type :all-columns, :table {:table "orders"}}]]}]}
(->references "select (select (select category) from products where products.id = orders.product_id) from orders"))))
-(deftest nested-select-subquery-with-direct-match-in-outer-query
+(deftest ^:parallel nested-select-subquery-with-direct-match-in-outer-query
(is (= {:used-fields
#{{:column "a",
:alias nil,
@@ -624,7 +624,7 @@
:source-columns [[{:type :all-columns, :table {:table "t2"}}]]}]]}]}
(->references "select (select a from t1) from (select a from t2)"))))
-(deftest basic-exists-test
+(deftest ^:parallel basic-exists-test
(is (= {:used-fields
#{{:column "name",
:alias nil,
@@ -661,7 +661,7 @@
FROM users u
WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"))))
-(deftest basic-not-test
+(deftest ^:parallel basic-not-test
(is (= {:used-fields
#{{:column "category",
:alias nil,
@@ -672,7 +672,7 @@ WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"))))
:table {:table "products"}}]}
(->references "select * from products where not (category = 'Gizmo')"))))
-(deftest basic-is-null-test
+(deftest ^:parallel basic-is-null-test
(is (= {:used-fields
#{{:column "category",
:alias nil,
@@ -683,7 +683,7 @@ WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"))))
:table {:table "products"}}]}
(->references "select * from products where category is null"))))
-(deftest negated-is-null-test
+(deftest ^:parallel negated-is-null-test
(is (= {:used-fields
#{{:column "category",
:alias nil,
@@ -694,7 +694,7 @@ WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"))))
:table {:table "products"}}]}
(->references "select * from products where category is not null"))))
-(deftest basic-between-test
+(deftest ^:parallel basic-between-test
(is (= {:used-fields
#{{:column "created_at",
:alias nil,
@@ -711,7 +711,27 @@ WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"))))
:returned-fields [{:type :all-columns, :table {:table "orders"}}]}
(->references "select * from orders where created_at between left and right"))))
-(deftest basic-cte-test
+(deftest ^:parallel basic-select-table-alias-test
+ (is (= {:used-fields #{},
+ :returned-fields
+ [{:type :custom-field,
+ :alias "o",
+ :used-fields #{{:type :all-columns, :table {:table "orders"}}}}]}
+ (->references "select o from (select * from orders) as o"))))
+
+(deftest ^:parallel select-table-alias-with-alias-test
+ (is (= {:used-fields #{},
+ :returned-fields
+ [{:type :custom-field,
+ :alias "o2",
+ :used-fields #{{:type :all-columns, :table {:table "orders"}}}}]}
+ (->references "select o as o2 from (select * from orders) as o"))))
+
+(deftest ^:parallel table-function-test
+ (is (= {:used-fields #{}, :returned-fields [{:type :unknown-columns}]}
+ (->references "select i.* from generate_series(1, 500) as i"))))
+
+(deftest ^:parallel basic-cte-test
(is (= {:used-fields
#{{:column "name",
:alias nil,
@@ -742,7 +762,7 @@ WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"))))
(->references "WITH active_users AS (SELECT id, name FROM users WHERE active = true)
SELECT * FROM active_users"))))
-(deftest unused-cte-test
+(deftest ^:parallel unused-cte-test
(is (= {:used-fields
#{{:column "name",
:alias nil,
@@ -764,7 +784,7 @@ SELECT * FROM active_users"))))
(->references "WITH active_users AS (SELECT id, name FROM users WHERE active = true)
SELECT * FROM products"))))
-(deftest shadowed-cte-test
+(deftest ^:parallel shadowed-cte-test
(is (= {:used-fields
#{{:column "y",
:alias "x",
@@ -823,7 +843,7 @@ SELECT
c
FROM c;"))))
-(deftest recursive-cte-test
+(deftest ^:parallel recursive-cte-test
(is (= {:used-fields
#{{:column "id",
:alias nil,
@@ -931,7 +951,7 @@ FROM c;"))))
)
SELECT name, level FROM emp_hierarchy"))))
-(deftest basic-union-test
+(deftest ^:parallel basic-union-test
(is (= {:used-fields
#{{:column "id",
:alias nil,
@@ -984,7 +1004,7 @@ SELECT name, level FROM emp_hierarchy"))))
UNION
SELECT id, name FROM archived_users"))))
-(deftest row-number-test
+(deftest ^:parallel row-number-test
(is (= {:used-fields
#{{:column "salary",
:alias nil,
@@ -1013,7 +1033,7 @@ SELECT id, name FROM archived_users"))))
(->references "SELECT name, ROW_NUMBER() OVER (ORDER BY salary DESC) AS rank
FROM employees"))))
-(deftest partition-by-test
+(deftest ^:parallel partition-by-test
(is (= {:used-fields
#{{:column "salary",
:alias nil,
@@ -1047,7 +1067,7 @@ FROM employees"))))
RANK() OVER (PARTITION BY department ORDER BY salary DESC) AS dept_rank
FROM employees"))))
-(deftest week-test
+(deftest ^:parallel week-test
(is (= {:used-fields
#{{:column "created_at",
:alias nil,
@@ -1089,7 +1109,7 @@ ORDER BY
) + INTERVAL '-1 day'
) ASC"))))
-(deftest week-of-year-test
+(deftest ^:parallel week-of-year-test
(is (= {:used-fields
#{{:column "created_at",
:alias nil,
@@ -1164,7 +1184,7 @@ ORDER BY
;; This test is horrifically long and makes cam's eyes bleed because this checks that a complex query compiles to the
;; right ast. Hopefully, other tests will catch any error caught by this test, but this is a bit of a failsafe.
^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
-(deftest complicated-test-1
+(deftest ^:parallel complicated-test-1
(is (= {:used-fields
#{{:column "title",
:alias "title",
@@ -1336,7 +1356,7 @@ ORDER BY
;; This test is horrifically long and makes cam's eyes bleed because this checks that a complex query compiles to the
;; right ast. Hopefully, other tests will catch any error caught by this test, but this is a bit of a failsafe.
^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
-(deftest complicated-test-2
+(deftest ^:parallel complicated-test-2
(is (= {:used-fields
#{{:column "created_at",
:alias nil,