The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add support for table-functions and selecting tables directly (#65074)
William39ceced98f87
diff --git a/deps.edn b/deps.ednindex 9d77f71d052..b01b9d6f7f2 100644--- a/deps.edn+++ b/deps.edn@@ -90,7 +90,7 @@:git/sha "4998a1fdd9d3713d7034d04509e9212204773bf2":git/url "https://github.com/eerohele/pp"};; TODO: replace back to normal artifact- io.github.metabase/macaw {:mvn/version "0.2.28"} ; Parse native SQL queries+ io.github.metabase/macaw {:mvn/version "0.2.29"} ; Parse native SQL queriesio.github.resilience4j/resilience4j-retry ^:antq/exclude ; 2.x do not support Java 8{:mvn/version "1.7.1"} ; Support for retrying operationsio.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"} ; prometheus jvm collectordiff --git a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.cljindex ebf53f41b05..224e0713e34 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj@@ -87,6 +87,15 @@driver(fake-query mp "select bad from products"))))))))+(deftest ^:parallel validate-table-function-query-test+ (testing "can validate queries using table functions"+ (let [mp (deps.tu/default-metadata-provider)+ driver (:engine (lib.metadata/database mp))]+ (is (= []+ (deps.native-validation/validate-native-query+ driver+ (fake-query mp "select i from my_function(1, 100)")))))))+(deftest ^:parallel validate-native-query-with-subquery-columns-test(testing "validate-native-query should detect invalid columns in subqueries"(let [mp (deps.tu/default-metadata-provider)@@ -348,6 +357,15 @@:base-type :type/Text,:effective-type :type/Text,:semantic-type :type/Category}]))+ (testing "Using a table function"+ (check-result-metadata+ driver mp+ "select ids from my_function(1, 100)"+ [{:name "IDS",+ :lib/desired-column-alias "IDS",+ :display-name "Ids",+ :effective-type :type/*,+ :semantic-type :Semantic/*}]))(testing "Selecting a bad table wildcard"(check-result-metadatadriver mpdiff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.cljindex f6ae81e4aba..c359d708ee8 100644--- a/src/metabase/driver/sql.clj+++ b/src/metabase/driver/sql.clj@@ -225,7 +225,21 @@;; inner query, it can also refer to something in the outer query.;; sql.references/field-references organizes source-cols into a list of lists;; to account for this.- (->> (mapcat (partial resolve-field driver metadata-provider) source-col-set)+ (->> (mapcat (fn [current-col]+ ;; :unknown-columns is a placeholder for "we know there are columns being+ ;; returned, but have no way of knowing what those are -- this is primarily+ ;; used for table-functions like `select * from my_func()`. If we encounter+ ;; something like that, assume that the query is valid and make up a matching+ ;; column to avoid false positives.+ (if (= (:type current-col) :unknown-columns)+ (let [name (:column col-spec)]+ [{:base-type :type/*+ :name name+ :display-name (->> name (u.humanization/name->human-readable-name :simple))+ :effective-type :type/*+ :semantic-type :Semantic/*}])+ (resolve-field driver metadata-provider current-col)))+ source-col-set)(some #(when (= (:name %) (:column col-spec))%))))))](assoc found :lib/desired-column-alias (or alias name))@@ -272,6 +286,10 @@:effective-type (apply lca :type/* (map :effective-type member-fields)):semantic-type (apply lca :Semantic/* (map :semantic-type member-fields))}]))+(defmethod resolve-field :unknown-columns+ [_driver _metadata-provider _col-spec]+ [])+(mu/defmethod driver/native-result-metadata :sql[driver :- :keywordnative-query :- :metabase.lib.schema/native-only-query]diff --git a/src/metabase/driver/sql/references.clj b/src/metabase/driver/sql/references.cljindex e13dcc90018..cf09c05d5c2 100644--- a/src/metabase/driver/sql/references.clj+++ b/src/metabase/driver/sql/references.clj@@ -23,6 +23,10 @@[:database {:optional true} :string][:table-alias {:optional true} :string]]]])+(mr/def ::unknown-columns+ [:map+ [:type [:= :unknown-columns]]])+(mr/def ::custom-field[:map[:type [:= :custom-field]]@@ -47,6 +51,7 @@[:multi {:dispatch :type}[:single-column [:ref ::single-column]][:all-columns [:ref ::all-columns]]+ [:unknown-columns [:ref ::unknown-columns]][:custom-field [:ref ::custom-field]][:composite-field [:ref ::composite-field]][:invalid-table-wildcard [:ref ::invalid-table-wildcard]]])@@ -82,7 +87,7 @@sublist))sources))-(defn- get-column [driver sources raw-col]+(defn- get-column [driver sources raw-col {:keys [return-table-matches?]}](if (and (nil? (:table raw-col))(nil? (:schema raw-col))(nil? (:database raw-col))@@ -100,14 +105,27 @@valid-sources)source-column (some #(when (= column (or (:alias %) (:column %)))%)- (first source-columns))]- [(if source-column- (cond-> source-column- alias (assoc :alias alias))- (-> {:column column- :alias alias- :type :single-column}- (assoc :source-columns source-columns)))])))+ (first source-columns))+ source-matches (when (nil? table)+ (some (fn [inner-sources]+ (some #(when (= column (or (-> % :names :table-alias) (-> % :names :table)))+ (:returned-fields %))+ inner-sources))+ sources))]+ (cond+ ;; we have a direct column match+ source-column [(cond-> source-column+ alias (assoc :alias alias))]+ ;; we have a match to a table+ source-matches (when return-table-matches?+ [{:type :custom-field+ :alias (or alias column)+ :used-fields (set source-matches)}])+ ;; we don't have a direct match, so show the possible columns this could match to+ :else [(-> {:column column+ :alias alias+ :type :single-column}+ (assoc :source-columns source-columns))]))))(defmulti find-used-fields"Finds the fields used in a given sql expression."@@ -122,7 +140,7 @@(defmethod find-used-fields [:sql :macaw.ast/column][driver sources _withs expr]- (get-column driver sources expr))+ (get-column driver sources expr {:return-table-matches? false}))(defmethod find-used-fields [:sql :macaw.ast/unary-expression][driver sources withs expr]@@ -190,7 +208,7 @@(defmethod find-returned-fields [:sql :macaw.ast/column][driver sources _withs expr]- (get-column driver sources expr))+ (get-column driver sources expr {:return-table-matches? true}))(defmethod find-returned-fields [:sql :macaw.ast/wildcard][_driver sources _withs _expr]@@ -300,6 +318,13 @@:names (when-let [alias (:table-alias expr)]{:table-alias (sql.normalize/normalize-name driver alias)})})+(defmethod field-references-impl [:sql :macaw.ast/table-function]+ [driver _outside-sources _withs expr]+ {:used-fields #{}+ :returned-fields [{:type :unknown-columns}]+ :names (when-let [alias (:table-alias expr)]+ {:table-alias (sql.normalize/normalize-name driver alias)})})+(defmethod field-references-impl :default[_driver _outside-sources _outside-withs _expr]{:used-fields #{}diff --git a/test/metabase/driver/sql/references_test.clj b/test/metabase/driver/sql/references_test.cljindex 111559da457..cce2360162b 100644--- a/test/metabase/driver/sql/references_test.clj+++ b/test/metabase/driver/sql/references_test.clj@@ -7,13 +7,13 @@(defn- ->references [query](->> query macaw/parsed-query macaw/->ast (sql.references/field-references :sql)))-(deftest garbage-test+(deftest ^:parallel garbage-test(is (= {:used-fields #{}:returned-fields []:bad-sql true}(->references "nothing"))))-(deftest basic-select-test+(deftest ^:parallel basic-select-test(is (= {:used-fields#{{:column "a",:alias nil,@@ -34,7 +34,7 @@:source-columns [[{:type :all-columns, :table {:table "products"}}]]}]}(->references "select * from (select a, b from products)"))))-(deftest basic-join-test+(deftest ^:parallel basic-join-test(is (= {:used-fields#{{:column "product_id",:alias nil,@@ -56,7 +56,7 @@:source-columns [[{:type :all-columns, :table {:table "orders"}}]]}]}(->references "select products.*, orders.id from products inner join orders on products.id = orders.product_id"))))-(deftest indeterminite-join-test+(deftest ^:parallel indeterminite-join-test(is (= {:used-fields#{{:column "id",:alias nil,@@ -81,7 +81,7 @@{:type :all-columns, :table {:table "products"}}]]}]}(->references "select id from products inner join orders on products.id = orders.product_id"))))-(deftest table-wildcard-join-test+(deftest ^:parallel table-wildcard-join-test(is (= {:used-fields#{{:column "id",:alias nil,@@ -97,7 +97,7 @@[{:type :all-columns, :table {:table "orders"}}]}(->references "select orders.* from products inner join orders on products.id = orders.product_id"))))-(deftest wildcard-join-test+(deftest ^:parallel wildcard-join-test(is (= {:used-fields#{{:column "id",:alias nil,@@ -114,7 +114,7 @@{:type :all-columns, :table {:table "products"}}]}(->references "select * from products inner join orders on products.id = orders.product_id"))))-(deftest basic-alias-test+(deftest ^:parallel basic-alias-test(is (= {:used-fields#{{:column "c",:alias "d",@@ -135,7 +135,7 @@:source-columns [[{:type :all-columns, :table {:table-alias "p", :table "products"}}]]}]}(->references "select p.a as b, p.c as d from products p"))))-(deftest basic-nested-query-test+(deftest ^:parallel basic-nested-query-test(is (= {:used-fields#{{:column "b",:alias nil,@@ -160,7 +160,7 @@:table {:table "products"}}]]}]}(->references "select a, b from (select a, b from products)"))))-(deftest renamed-nested-query-test+(deftest ^:parallel renamed-nested-query-test(is (= {:used-fields#{{:column "a",:alias "b",@@ -180,7 +180,7 @@:table {:table "products"}}]]}]}(->references "select b as c from (select a as b from products)"))))-(deftest broken-nested-query-test+(deftest ^:parallel broken-nested-query-test(is (= {:used-fields#{{:column "b",:alias nil,@@ -206,7 +206,7 @@:source-columns [[{:type :all-columns, :table {:table "products"}}]]}]]}]}(->references "select a from (select b from products)"))))-(deftest different-case-nested-query-test+(deftest ^:parallel different-case-nested-query-test(is (= {:used-fields#{{:column "a",:alias nil,@@ -221,7 +221,7 @@:table {:table "orders"}}]]}]}(->references "select A from (select a from orders)"))))-(deftest wildcard-nested-query-test+(deftest ^:parallel wildcard-nested-query-test(is (= {:used-fields#{{:column "a",:alias nil,@@ -236,7 +236,7 @@:table {:table "orders"}}]]}]}(->references "select * from (select a from orders)"))))-(deftest table-wildcard-nested-query-test+(deftest ^:parallel table-wildcard-nested-query-test(is (= {:used-fields#{{:column "a",:alias nil,@@ -251,7 +251,7 @@:table {:table "orders"}}]]}]}(->references "select o.* from (select a from orders) o"))))-(deftest bad-table-wildcard-nested-query-test+(deftest ^:parallel bad-table-wildcard-nested-query-test(is (= {:used-fields#{{:column "a",:alias nil,@@ -262,7 +262,7 @@[{:type :invalid-table-wildcard, :table "foo"}]}(->references "select foo.* from (select a from orders)"))))-(deftest bad-table-name-test+(deftest ^:parallel bad-table-name-test(is (= {:used-fields#{{:column "a",:alias nil,@@ -275,7 +275,7 @@:source-columns []}]}(->references "select bad.a from products"))))-(deftest basic-where-test+(deftest ^:parallel basic-where-test(is (= {:used-fields#{{:column "category",:alias nil,@@ -286,17 +286,17 @@:table {:table "products"}}]}(->references "select * from products where category = 'hello'"))))-(deftest basic-aggregation-test+(deftest ^:parallel basic-aggregation-test(is (=? {:used-fields #{},:returned-fields [{:alias nil, :type :custom-field, :used-fields #{}}]}(->references "select count(*) from products"))))-(deftest named-aggregation-test+(deftest ^:parallel named-aggregation-test(is (=? {:used-fields #{},:returned-fields [{:alias "count", :type :custom-field, :used-fields #{}}]}(->references "select count(*) as count from products"))))-(deftest extra-names-test+(deftest ^:parallel extra-names-test(is (= {:used-fields#{{:column "col",:alias nil,@@ -317,7 +317,7 @@:table "table"}}]]}]}(->references "select db.schema.table.col from db.schema.table"))))-(deftest basic-grouping-test+(deftest ^:parallel basic-grouping-test(is (= {:used-fields#{{:column "category",:alias nil,@@ -346,7 +346,7 @@:source-columns [[{:type :all-columns, :table {:table "orders"}}]]}}}]}(->references "select sum(total) as sum from orders group by category"))))-(deftest grouping-order-by-test+(deftest ^:parallel grouping-order-by-test(is (= {:used-fields#{{:alias "cards_created",:type :custom-field,@@ -385,7 +385,7 @@:source-columns [[{:type :all-columns, :table {:table "report_card"}}]]}}}]}(->references "SELECT creator_id, COUNT(creator_id) AS cards_created FROM report_card GROUP BY creator_id ORDER BY cards_created DESC"))))-(deftest basic-arg-test+(deftest ^:parallel basic-arg-test(is (= {:used-fields#{{:column "category",:alias nil,@@ -396,7 +396,7 @@:table {:table "products"}}]}(->references "select * from products where category = ?"))))-(deftest basic-case-test+(deftest ^:parallel basic-case-test(is (=? {:used-fields#{{:column "tax",:alias nil,@@ -434,7 +434,7 @@:table {:table "orders"}}]]}}}]}(->references "select case when total < 0 then -subtotal else tax end from orders"))))-(deftest switch-case-test+(deftest ^:parallel switch-case-test(is (=? {:used-fields#{{:column "category",:alias nil,@@ -452,7 +452,7 @@:table {:table "products"}}]]}}}]}(->references "select case category when 'Gizmo' then 'is gizmo' else 'is not gizmo' end from products"))))-(deftest basic-select-subquery-test+(deftest ^:parallel basic-select-subquery-test(is (= {:used-fields#{{:column "product_id",:alias nil,@@ -482,7 +482,7 @@{:type :all-columns, :table {:table "orders"}}]}(->references "select (select category from products where products.id = orders.product_id), * from orders"))))-(deftest named-select-subquery-test+(deftest ^:parallel named-select-subquery-test(is (= {:used-fields#{{:column "product_id",:alias nil,@@ -512,7 +512,7 @@{:type :all-columns, :table {:table "orders"}}]}(->references "select (select category from products where products.id = orders.product_id) as category2, * from orders"))))-(deftest nested-select-subquery-test+(deftest ^:parallel nested-select-subquery-test(is (= {:used-fields#{{:column "product_id",:alias nil,@@ -542,7 +542,7 @@{:type :all-columns, :table {:table "orders"}}]}(->references "select (select (select category from products where products.id = orders.product_id)), * from orders"))))-(deftest select-subquery-with-normal-subquery-test+(deftest ^:parallel select-subquery-with-normal-subquery-test(is (= {:used-fields#{{:column "product_id",:alias nil,@@ -572,7 +572,7 @@{:type :all-columns, :table {:table "orders"}}]}(->references "select (select * from (select category from products where products.id = orders.product_id) sub), * from orders"))))-(deftest nested-select-subquery-with-reference-to-middle-select-test+(deftest ^:parallel nested-select-subquery-with-reference-to-middle-select-test(is (= {:used-fields#{{:column "id",:alias nil,@@ -597,7 +597,7 @@[{:type :all-columns, :table {:table "orders"}}]]}]}(->references "select (select (select category) from products where products.id = orders.product_id) from orders"))))-(deftest nested-select-subquery-with-direct-match-in-outer-query+(deftest ^:parallel nested-select-subquery-with-direct-match-in-outer-query(is (= {:used-fields#{{:column "a",:alias nil,@@ -624,7 +624,7 @@:source-columns [[{:type :all-columns, :table {:table "t2"}}]]}]]}]}(->references "select (select a from t1) from (select a from t2)"))))-(deftest basic-exists-test+(deftest ^:parallel basic-exists-test(is (= {:used-fields#{{:column "name",:alias nil,@@ -661,7 +661,7 @@FROM users uWHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"))))-(deftest basic-not-test+(deftest ^:parallel basic-not-test(is (= {:used-fields#{{:column "category",:alias nil,@@ -672,7 +672,7 @@ WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)")))):table {:table "products"}}]}(->references "select * from products where not (category = 'Gizmo')"))))-(deftest basic-is-null-test+(deftest ^:parallel basic-is-null-test(is (= {:used-fields#{{:column "category",:alias nil,@@ -683,7 +683,7 @@ WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)")))):table {:table "products"}}]}(->references "select * from products where category is null"))))-(deftest negated-is-null-test+(deftest ^:parallel negated-is-null-test(is (= {:used-fields#{{:column "category",:alias nil,@@ -694,7 +694,7 @@ WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)")))):table {:table "products"}}]}(->references "select * from products where category is not null"))))-(deftest basic-between-test+(deftest ^:parallel basic-between-test(is (= {:used-fields#{{:column "created_at",:alias nil,@@ -711,7 +711,27 @@ WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)")))):returned-fields [{:type :all-columns, :table {:table "orders"}}]}(->references "select * from orders where created_at between left and right"))))-(deftest basic-cte-test+(deftest ^:parallel basic-select-table-alias-test+ (is (= {:used-fields #{},+ :returned-fields+ [{:type :custom-field,+ :alias "o",+ :used-fields #{{:type :all-columns, :table {:table "orders"}}}}]}+ (->references "select o from (select * from orders) as o"))))++(deftest ^:parallel select-table-alias-with-alias-test+ (is (= {:used-fields #{},+ :returned-fields+ [{:type :custom-field,+ :alias "o2",+ :used-fields #{{:type :all-columns, :table {:table "orders"}}}}]}+ (->references "select o as o2 from (select * from orders) as o"))))++(deftest ^:parallel table-function-test+ (is (= {:used-fields #{}, :returned-fields [{:type :unknown-columns}]}+ (->references "select i.* from generate_series(1, 500) as i"))))++(deftest ^:parallel basic-cte-test(is (= {:used-fields#{{:column "name",:alias nil,@@ -742,7 +762,7 @@ WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"))))(->references "WITH active_users AS (SELECT id, name FROM users WHERE active = true)SELECT * FROM active_users"))))-(deftest unused-cte-test+(deftest ^:parallel unused-cte-test(is (= {:used-fields#{{:column "name",:alias nil,@@ -764,7 +784,7 @@ SELECT * FROM active_users"))))(->references "WITH active_users AS (SELECT id, name FROM users WHERE active = true)SELECT * FROM products"))))-(deftest shadowed-cte-test+(deftest ^:parallel shadowed-cte-test(is (= {:used-fields#{{:column "y",:alias "x",@@ -823,7 +843,7 @@ SELECTcFROM c;"))))-(deftest recursive-cte-test+(deftest ^:parallel recursive-cte-test(is (= {:used-fields#{{:column "id",:alias nil,@@ -931,7 +951,7 @@ FROM c;")))))SELECT name, level FROM emp_hierarchy"))))-(deftest basic-union-test+(deftest ^:parallel basic-union-test(is (= {:used-fields#{{:column "id",:alias nil,@@ -984,7 +1004,7 @@ SELECT name, level FROM emp_hierarchy"))))UNIONSELECT id, name FROM archived_users"))))-(deftest row-number-test+(deftest ^:parallel row-number-test(is (= {:used-fields#{{:column "salary",:alias nil,@@ -1013,7 +1033,7 @@ SELECT id, name FROM archived_users"))))(->references "SELECT name, ROW_NUMBER() OVER (ORDER BY salary DESC) AS rankFROM employees"))))-(deftest partition-by-test+(deftest ^:parallel partition-by-test(is (= {:used-fields#{{:column "salary",:alias nil,@@ -1047,7 +1067,7 @@ FROM employees"))))RANK() OVER (PARTITION BY department ORDER BY salary DESC) AS dept_rankFROM employees"))))-(deftest week-test+(deftest ^:parallel week-test(is (= {:used-fields#{{:column "created_at",:alias nil,@@ -1089,7 +1109,7 @@ ORDER BY) + INTERVAL '-1 day') ASC"))))-(deftest week-of-year-test+(deftest ^:parallel week-of-year-test(is (= {:used-fields#{{:column "created_at",:alias nil,@@ -1164,7 +1184,7 @@ ORDER BY;; This test is horrifically long and makes cam's eyes bleed because this checks that a complex query compiles to the;; right ast. Hopefully, other tests will catch any error caught by this test, but this is a bit of a failsafe.^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}-(deftest complicated-test-1+(deftest ^:parallel complicated-test-1(is (= {:used-fields#{{:column "title",:alias "title",@@ -1336,7 +1356,7 @@ ORDER BY;; This test is horrifically long and makes cam's eyes bleed because this checks that a complex query compiles to the;; right ast. Hopefully, other tests will catch any error caught by this test, but this is a bit of a failsafe.^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}-(deftest complicated-test-2+(deftest ^:parallel complicated-test-2(is (= {:used-fields#{{:column "created_at",:alias nil,