The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Table tags filters (#70699)
Nicola Mometto88729c52f14c
diff --git a/src/metabase/driver/common/parameters.clj b/src/metabase/driver/common/parameters.cljindex 10f74080711..2e1dc16d923 100644--- a/src/metabase/driver/common/parameters.clj+++ b/src/metabase/driver/common/parameters.clj@@ -62,7 +62,15 @@;; specific column.;;;; `table-id` is the id of the table being referenced-(p.types/defrecord+ ReferencedTableQuery [table-id]+;;+;; `source-filters` is an optional sequence of filter maps applied to the table reference. Each filter map has:+;; :field-id - the ID of the field to filter on+;; :op - the comparison operator, one of :>, :>=, :<, :<=, :=, :!=+;; :value - the value to compare against+;; When present, the table reference is rendered as a filtered subquery:+;; (SELECT * FROM "table" WHERE "col" > ? AND "col" <= ?)+;; This was introduced to support incremental transforms, unused by the frontend.+(p.types/defrecord+ ReferencedTableQuery [table-id source-filters]pretty/PrettyPrintable(pretty [this](list (pretty/qualify-symbol-for-*ns* `map->ReferencedTableQuery) (into {} this))))diff --git a/src/metabase/driver/common/parameters/values.clj b/src/metabase/driver/common/parameters/values.cljindex 41022142e77..89c4b129428 100644--- a/src/metabase/driver/common/parameters/values.clj+++ b/src/metabase/driver/common/parameters/values.clj@@ -258,8 +258,13 @@e))))))(mu/defmethod parse-tag :table :- ReferencedTableQuery- [{:keys [table-id]} _params]- (params/map->ReferencedTableQuery {:table-id table-id}))+ [{:keys [table-id source-filters]} :- ::mbql.s/TemplateTag _params]+ (when (seq source-filters)+ (when-let [op (some #(when-not (lib.schema.template-tag/allowed-source-filter-ops (:op %)) %) source-filters)]+ (throw (ex-info (tru "Invalid source-filter operator: {0}. Allowed operators: {1}"+ (pr-str op) (pr-str lib.schema.template-tag/allowed-source-filter-ops))+ {:op op :allowed-ops lib.schema.template-tag/allowed-source-filter-ops}))))+ (params/->ReferencedTableQuery table-id (not-empty source-filters)))(mu/defmethod parse-tag :snippet :- ReferencedQuerySnippet[{:keys [snippet-name snippet-id], :as tag} :- ::mbql.s/TemplateTagdiff --git a/src/metabase/driver/sql/parameters/substitution.clj b/src/metabase/driver/sql/parameters/substitution.cljindex 3dc0d2c78c0..301f6c5a226 100644--- a/src/metabase/driver/sql/parameters/substitution.clj+++ b/src/metabase/driver/sql/parameters/substitution.clj@@ -6,7 +6,7 @@:replacement-snippet \"= ?\";; ; any prepared statement args (values for `?` placeholders) needed for the replacement snippet:prepared-statement-args [#t \"2017-01-01\"]}"- (:refer-clojure :exclude [not-empty])+ (:refer-clojure :exclude [not-empty mapv])(:require[clojure.string :as str][metabase.driver :as driver]@@ -17,9 +17,10 @@[metabase.driver.sql.query-processor :as sql.qp][metabase.util :as u][metabase.util.date-2 :as u.date]+ [metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu]- [metabase.util.performance :refer [not-empty]])+ [metabase.util.performance :refer [mapv not-empty]])(:import(clojure.lang IPersistentVector Keyword)(java.time.temporal Temporal)@@ -411,11 +412,25 @@(replace-alias driver field alias replacement-snippet-info)))(defmethod ->replacement-snippet-info [:sql ReferencedTableQuery]- [driver {:keys [table-id]}]- (let [mp (driver-api/metadata-provider)- alias (->> (driver-api/table mp table-id)- (sql.qp/->honeysql driver)- (sql.qp/format-honeysql driver)- first)]- {:prepared-statement-args []- :replacement-snippet alias}))+ [driver {:keys [table-id source-filters]}]+ (let [mp (driver-api/metadata-provider)+ table-hsql (sql.qp/->honeysql driver (driver-api/table mp table-id))]+ (if (seq source-filters)+ (let [prepared (mapv (fn [{:keys [field-id op value]}]+ (let [field (driver-api/field mp field-id)]+ {:col (h2x/identifier :field (:name field))+ :op op+ :sub (->prepared-substitution driver value)}))+ source-filters)+ where-clause (into [:and]+ (map (fn [{:keys [col op sub]}]+ [op col [:raw (:sql-string sub)]]))+ prepared)+ hsql {:select [:*] :from [[table-hsql]] :where where-clause}+ [sql] (sql.qp/format-honeysql driver hsql)+ args (into [] (mapcat (comp :param-values :sub)) prepared)]+ {:replacement-snippet (str "(" sql ")")+ :prepared-statement-args args})+ (let [[sql] (sql.qp/format-honeysql driver table-hsql)]+ {:prepared-statement-args []+ :replacement-snippet sql}))))diff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljcindex 33d3cd2ba56..275deadbcd4 100644--- a/src/metabase/legacy_mbql/schema.cljc+++ b/src/metabase/legacy_mbql/schema.cljc@@ -1536,20 +1536,29 @@[:type [:= {:decode/normalize helpers/normalize-keyword} :card]][:card-id ::lib.schema.id/card]]])+(mr/def ::TemplateTag.SourceFilter+ "Schema for a single source-filter applied to a table template tag."+ [:map+ [:field-id ::lib.schema.id/field]+ [:op (into [:enum] lib.schema.template-tag/allowed-source-filter-ops)]+ [:value :any]])+;; Example:;;;; {:id "fc5e14d9-7d14-67af-66b2-b2a6e25afeaf";; :name "#1635";; :display-name "#1635";; :type :table-;; :table-id 2}+;; :table-id 2+;; :source-filters [{:op :> :field-id 3 :value 500}]}(mr/def ::TemplateTag.SourceTable"Schema for a source query template tag."[:merge::TemplateTag.Common[:map[:type [:= {:decode/normalize helpers/normalize-keyword} :table]]- [:table-id ::lib.schema.id/table]]])+ [:table-id ::lib.schema.id/table]+ [:source-filters {:optional true} [:sequential [:ref ::TemplateTag.SourceFilter]]]]])(mr/def ::TemplateTag.Value.Common"Stuff shared between the Field filter and raw value template tag schemas."diff --git a/src/metabase/lib/schema/template_tag.cljc b/src/metabase/lib/schema/template_tag.cljcindex 198a982b102..40e8da8a282 100644--- a/src/metabase/lib/schema/template_tag.cljc+++ b/src/metabase/lib/schema/template_tag.cljc@@ -128,20 +128,33 @@[:card-id ::id/card]]][:ref ::disallow-dimension]])+(def allowed-source-filter-ops+ "Set of allowed source-filter ops"+ #{:> :>= :< :<= := :!=})++(mr/def ::source-filter+ "Schema for a single source-filter applied to a table template tag."+ [:map+ [:field-id ::id/field]+ [:op (into [:enum] allowed-source-filter-ops)]+ [:value :any]])+;; Example:;;;; {:id "fc5e14d9-7d14-67af-66b2-b2a6e25afeaf";; :name "orders";; :display-name "Orders";; :type :table-;; :table-id 2}+;; :table-id 2+;; :source-filters [{:op :> :field-id 3 :value 500}]}(mr/def ::source-table[:and[:merge[:ref ::common][:map[:type [:= :table]]- [:table-id ::id/table]]]+ [:table-id ::id/table]+ [:source-filters {:optional true} [:sequential [:ref ::source-filter]]]]][:ref ::disallow-dimension]])(def raw-value-template-tag-typesdiff --git a/test/metabase/driver/common/parameters/values_test.clj b/test/metabase/driver/common/parameters/values_test.cljindex 5752a127686..afe820bbdb2 100644--- a/test/metabase/driver/common/parameters/values_test.clj+++ b/test/metabase/driver/common/parameters/values_test.clj@@ -24,7 +24,7 @@[toucan2.core :as t2])(:import(clojure.lang ExceptionInfo)- (metabase.driver.common.parameters ReferencedCardQuery)))+ (metabase.driver.common.parameters ReferencedCardQuery ReferencedTableQuery)))(set! *warn-on-reflection* true)@@ -610,6 +610,45 @@(is (= expected(query->params-map query)))))))+(deftest ^:parallel table-tag-test+ (testing "Table template tag produces a ReferencedTableQuery"+ (is (instance? ReferencedTableQuery+ (#'params.values/value-for-tag+ {:name "table-tag-test"+ :display-name "Table tag test"+ :type :table+ :table-id 1}+ [])))+ (is (=? {:table-id 1}+ (value-for-tag {:name "table-tag-test"+ :display-name "Table tag test"+ :type :table+ :table-id 1}+ [])))))++(deftest ^:parallel table-tag-with-source-filters-test+ (testing "Table template tag with source-filters passes them through"+ (let [filters [{:field-id 100 :op :> :value 10}+ {:field-id 100 :op :<= :value 50}]]+ (is (= {:table-id 1+ :source-filters filters}+ (value-for-tag {:name "table-tag-test"+ :display-name "Table tag test"+ :type :table+ :table-id 1+ :source-filters filters}+ [])))))+ (testing "Table template tag with invalid source-filter op throws"+ (is (thrown-with-msg?+ ExceptionInfo+ #"Invalid"+ (value-for-tag {:name "table-tag-test"+ :display-name "Table tag test"+ :type :table+ :table-id 1+ :source-filters [{:field-id 100 :op :LIKE :value "%foo%"}]}+ [])))))+(deftest ^:parallel invalid-param-test(testing "Should throw an Exception if we try to pass with a `:type` we don't understand"(let [query (assoc (mt/native-query {:query "SELECT * FROM table WHERE {{x}}"})diff --git a/test/metabase/driver/sql/parameters/substitution_test.clj b/test/metabase/driver/sql/parameters/substitution_test.cljindex 1829d8ef84f..ccdd3311ce8 100644--- a/test/metabase/driver/sql/parameters/substitution_test.clj+++ b/test/metabase/driver/sql/parameters/substitution_test.clj@@ -129,3 +129,53 @@;; no extra `sql.qp/date` calls due to `nil` returned from the override:replacement-snippet "\"PUBLIC\".\"ORDERS\".\"CREATED_AT\" >= ? AND \"PUBLIC\".\"ORDERS\".\"CREATED_AT\" < ?"}(sql.params.substitution/->replacement-snippet-info ::temporal-unit-alignment-override field-filter))))))))))++(deftest ^:parallel table-query->replacement-snippet-test+ (testing "Basic table reference without source-filters"+ (mt/with-metadata-provider meta/metadata-provider+ (is (= {:replacement-snippet "\"PUBLIC\".\"ORDERS\""+ :prepared-statement-args []}+ (sql.params.substitution/->replacement-snippet-info+ :h2+ (params/map->ReferencedTableQuery+ {:table-id (meta/id :orders)})))))))++(deftest ^:parallel table-query-with-source-filters->replacement-snippet-test+ (testing "Table reference with a single source-filter produces a filtered subquery"+ (mt/with-metadata-provider meta/metadata-provider+ (is (= {:replacement-snippet "(SELECT * FROM \"PUBLIC\".\"ORDERS\" WHERE (\"TOTAL\" > 100))"+ :prepared-statement-args []}+ (sql.params.substitution/->replacement-snippet-info+ :h2+ (params/map->ReferencedTableQuery+ {:table-id (meta/id :orders)+ :source-filters [{:field-id (meta/id :orders :total)+ :op :>+ :value 100}]}))))))+ (testing "Table reference with multiple source-filters joins them with AND"+ (mt/with-metadata-provider meta/metadata-provider+ (is (= {:replacement-snippet "(SELECT * FROM \"PUBLIC\".\"ORDERS\" WHERE (\"TOTAL\" > 100) AND (\"TOTAL\" <= 500))"+ :prepared-statement-args []}+ (sql.params.substitution/->replacement-snippet-info+ :h2+ (params/map->ReferencedTableQuery+ {:table-id (meta/id :orders)+ :source-filters [{:field-id (meta/id :orders :total)+ :op :>+ :value 100}+ {:field-id (meta/id :orders :total)+ :op :<=+ :value 500}]}))))))+ (testing "Table reference with a timestamp source-filter value"+ (driver/with-driver ::temporal-unit-alignment-override+ (mt/with-metadata-provider meta/metadata-provider+ (let [ts (t/offset-date-time 2024 1 15 0 0 0 0 (t/zone-offset 0))]+ (is (= {:replacement-snippet "(SELECT * FROM \"PUBLIC\".\"ORDERS\" WHERE (\"CREATED_AT\" > ?))"+ :prepared-statement-args [ts]}+ (sql.params.substitution/->replacement-snippet-info+ ::temporal-unit-alignment-override+ (params/map->ReferencedTableQuery+ {:table-id (meta/id :orders)+ :source-filters [{:field-id (meta/id :orders :created-at)+ :op :>+ :value ts}]})))))))))