Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Table tags filters (#70699)

Nicola Mometto88729c52f14c
diff --git a/src/metabase/driver/common/parameters.clj b/src/metabase/driver/common/parameters.clj
index 10f74080711..2e1dc16d923 100644
--- a/src/metabase/driver/common/parameters.clj
+++ b/src/metabase/driver/common/parameters.clj
@@ -62,7 +62,15 @@
;; specific column.
;;
;; `table-id` is the id of the table being referenced
-(p.types/defrecord+ ReferencedTableQuery [table-id]
+;;
+;; `source-filters` is an optional sequence of filter maps applied to the table reference. Each filter map has:
+;; :field-id - the ID of the field to filter on
+;; :op - the comparison operator, one of :>, :>=, :<, :<=, :=, :!=
+;; :value - the value to compare against
+;; When present, the table reference is rendered as a filtered subquery:
+;; (SELECT * FROM "table" WHERE "col" > ? AND "col" <= ?)
+;; This was introduced to support incremental transforms, unused by the frontend.
+(p.types/defrecord+ ReferencedTableQuery [table-id source-filters]
pretty/PrettyPrintable
(pretty [this]
(list (pretty/qualify-symbol-for-*ns* `map->ReferencedTableQuery) (into {} this))))
diff --git a/src/metabase/driver/common/parameters/values.clj b/src/metabase/driver/common/parameters/values.clj
index 41022142e77..89c4b129428 100644
--- a/src/metabase/driver/common/parameters/values.clj
+++ b/src/metabase/driver/common/parameters/values.clj
@@ -258,8 +258,13 @@
e))))))
(mu/defmethod parse-tag :table :- ReferencedTableQuery
- [{:keys [table-id]} _params]
- (params/map->ReferencedTableQuery {:table-id table-id}))
+ [{:keys [table-id source-filters]} :- ::mbql.s/TemplateTag _params]
+ (when (seq source-filters)
+ (when-let [op (some #(when-not (lib.schema.template-tag/allowed-source-filter-ops (:op %)) %) source-filters)]
+ (throw (ex-info (tru "Invalid source-filter operator: {0}. Allowed operators: {1}"
+ (pr-str op) (pr-str lib.schema.template-tag/allowed-source-filter-ops))
+ {:op op :allowed-ops lib.schema.template-tag/allowed-source-filter-ops}))))
+ (params/->ReferencedTableQuery table-id (not-empty source-filters)))
(mu/defmethod parse-tag :snippet :- ReferencedQuerySnippet
[{:keys [snippet-name snippet-id], :as tag} :- ::mbql.s/TemplateTag
diff --git a/src/metabase/driver/sql/parameters/substitution.clj b/src/metabase/driver/sql/parameters/substitution.clj
index 3dc0d2c78c0..301f6c5a226 100644
--- a/src/metabase/driver/sql/parameters/substitution.clj
+++ b/src/metabase/driver/sql/parameters/substitution.clj
@@ -6,7 +6,7 @@
:replacement-snippet \"= ?\"
;; ; any prepared statement args (values for `?` placeholders) needed for the replacement snippet
:prepared-statement-args [#t \"2017-01-01\"]}"
- (:refer-clojure :exclude [not-empty])
+ (:refer-clojure :exclude [not-empty mapv])
(:require
[clojure.string :as str]
[metabase.driver :as driver]
@@ -17,9 +17,10 @@
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
+ [metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :refer [tru]]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [not-empty]])
+ [metabase.util.performance :refer [mapv not-empty]])
(:import
(clojure.lang IPersistentVector Keyword)
(java.time.temporal Temporal)
@@ -411,11 +412,25 @@
(replace-alias driver field alias replacement-snippet-info)))
(defmethod ->replacement-snippet-info [:sql ReferencedTableQuery]
- [driver {:keys [table-id]}]
- (let [mp (driver-api/metadata-provider)
- alias (->> (driver-api/table mp table-id)
- (sql.qp/->honeysql driver)
- (sql.qp/format-honeysql driver)
- first)]
- {:prepared-statement-args []
- :replacement-snippet alias}))
+ [driver {:keys [table-id source-filters]}]
+ (let [mp (driver-api/metadata-provider)
+ table-hsql (sql.qp/->honeysql driver (driver-api/table mp table-id))]
+ (if (seq source-filters)
+ (let [prepared (mapv (fn [{:keys [field-id op value]}]
+ (let [field (driver-api/field mp field-id)]
+ {:col (h2x/identifier :field (:name field))
+ :op op
+ :sub (->prepared-substitution driver value)}))
+ source-filters)
+ where-clause (into [:and]
+ (map (fn [{:keys [col op sub]}]
+ [op col [:raw (:sql-string sub)]]))
+ prepared)
+ hsql {:select [:*] :from [[table-hsql]] :where where-clause}
+ [sql] (sql.qp/format-honeysql driver hsql)
+ args (into [] (mapcat (comp :param-values :sub)) prepared)]
+ {:replacement-snippet (str "(" sql ")")
+ :prepared-statement-args args})
+ (let [[sql] (sql.qp/format-honeysql driver table-hsql)]
+ {:prepared-statement-args []
+ :replacement-snippet sql}))))
diff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljc
index 33d3cd2ba56..275deadbcd4 100644
--- a/src/metabase/legacy_mbql/schema.cljc
+++ b/src/metabase/legacy_mbql/schema.cljc
@@ -1536,20 +1536,29 @@
[:type [:= {:decode/normalize helpers/normalize-keyword} :card]]
[:card-id ::lib.schema.id/card]]])
+(mr/def ::TemplateTag.SourceFilter
+ "Schema for a single source-filter applied to a table template tag."
+ [:map
+ [:field-id ::lib.schema.id/field]
+ [:op (into [:enum] lib.schema.template-tag/allowed-source-filter-ops)]
+ [:value :any]])
+
;; Example:
;;
;; {:id "fc5e14d9-7d14-67af-66b2-b2a6e25afeaf"
;; :name "#1635"
;; :display-name "#1635"
;; :type :table
-;; :table-id 2}
+;; :table-id 2
+;; :source-filters [{:op :> :field-id 3 :value 500}]}
(mr/def ::TemplateTag.SourceTable
"Schema for a source query template tag."
[:merge
::TemplateTag.Common
[:map
[:type [:= {:decode/normalize helpers/normalize-keyword} :table]]
- [:table-id ::lib.schema.id/table]]])
+ [:table-id ::lib.schema.id/table]
+ [:source-filters {:optional true} [:sequential [:ref ::TemplateTag.SourceFilter]]]]])
(mr/def ::TemplateTag.Value.Common
"Stuff shared between the Field filter and raw value template tag schemas."
diff --git a/src/metabase/lib/schema/template_tag.cljc b/src/metabase/lib/schema/template_tag.cljc
index 198a982b102..40e8da8a282 100644
--- a/src/metabase/lib/schema/template_tag.cljc
+++ b/src/metabase/lib/schema/template_tag.cljc
@@ -128,20 +128,33 @@
[:card-id ::id/card]]]
[:ref ::disallow-dimension]])
+(def allowed-source-filter-ops
+ "Set of allowed source-filter ops"
+ #{:> :>= :< :<= := :!=})
+
+(mr/def ::source-filter
+ "Schema for a single source-filter applied to a table template tag."
+ [:map
+ [:field-id ::id/field]
+ [:op (into [:enum] allowed-source-filter-ops)]
+ [:value :any]])
+
;; Example:
;;
;; {:id "fc5e14d9-7d14-67af-66b2-b2a6e25afeaf"
;; :name "orders"
;; :display-name "Orders"
;; :type :table
-;; :table-id 2}
+;; :table-id 2
+;; :source-filters [{:op :> :field-id 3 :value 500}]}
(mr/def ::source-table
[:and
[:merge
[:ref ::common]
[:map
[:type [:= :table]]
- [:table-id ::id/table]]]
+ [:table-id ::id/table]
+ [:source-filters {:optional true} [:sequential [:ref ::source-filter]]]]]
[:ref ::disallow-dimension]])
(def raw-value-template-tag-types
diff --git a/test/metabase/driver/common/parameters/values_test.clj b/test/metabase/driver/common/parameters/values_test.clj
index 5752a127686..afe820bbdb2 100644
--- a/test/metabase/driver/common/parameters/values_test.clj
+++ b/test/metabase/driver/common/parameters/values_test.clj
@@ -24,7 +24,7 @@
[toucan2.core :as t2])
(:import
(clojure.lang ExceptionInfo)
- (metabase.driver.common.parameters ReferencedCardQuery)))
+ (metabase.driver.common.parameters ReferencedCardQuery ReferencedTableQuery)))
(set! *warn-on-reflection* true)
@@ -610,6 +610,45 @@
(is (= expected
(query->params-map query)))))))
+(deftest ^:parallel table-tag-test
+ (testing "Table template tag produces a ReferencedTableQuery"
+ (is (instance? ReferencedTableQuery
+ (#'params.values/value-for-tag
+ {:name "table-tag-test"
+ :display-name "Table tag test"
+ :type :table
+ :table-id 1}
+ [])))
+ (is (=? {:table-id 1}
+ (value-for-tag {:name "table-tag-test"
+ :display-name "Table tag test"
+ :type :table
+ :table-id 1}
+ [])))))
+
+(deftest ^:parallel table-tag-with-source-filters-test
+ (testing "Table template tag with source-filters passes them through"
+ (let [filters [{:field-id 100 :op :> :value 10}
+ {:field-id 100 :op :<= :value 50}]]
+ (is (= {:table-id 1
+ :source-filters filters}
+ (value-for-tag {:name "table-tag-test"
+ :display-name "Table tag test"
+ :type :table
+ :table-id 1
+ :source-filters filters}
+ [])))))
+ (testing "Table template tag with invalid source-filter op throws"
+ (is (thrown-with-msg?
+ ExceptionInfo
+ #"Invalid"
+ (value-for-tag {:name "table-tag-test"
+ :display-name "Table tag test"
+ :type :table
+ :table-id 1
+ :source-filters [{:field-id 100 :op :LIKE :value "%foo%"}]}
+ [])))))
+
(deftest ^:parallel invalid-param-test
(testing "Should throw an Exception if we try to pass with a `:type` we don't understand"
(let [query (assoc (mt/native-query {:query "SELECT * FROM table WHERE {{x}}"})
diff --git a/test/metabase/driver/sql/parameters/substitution_test.clj b/test/metabase/driver/sql/parameters/substitution_test.clj
index 1829d8ef84f..ccdd3311ce8 100644
--- a/test/metabase/driver/sql/parameters/substitution_test.clj
+++ b/test/metabase/driver/sql/parameters/substitution_test.clj
@@ -129,3 +129,53 @@
;; no extra `sql.qp/date` calls due to `nil` returned from the override
:replacement-snippet "\"PUBLIC\".\"ORDERS\".\"CREATED_AT\" >= ? AND \"PUBLIC\".\"ORDERS\".\"CREATED_AT\" < ?"}
(sql.params.substitution/->replacement-snippet-info ::temporal-unit-alignment-override field-filter))))))))))
+
+(deftest ^:parallel table-query->replacement-snippet-test
+ (testing "Basic table reference without source-filters"
+ (mt/with-metadata-provider meta/metadata-provider
+ (is (= {:replacement-snippet "\"PUBLIC\".\"ORDERS\""
+ :prepared-statement-args []}
+ (sql.params.substitution/->replacement-snippet-info
+ :h2
+ (params/map->ReferencedTableQuery
+ {:table-id (meta/id :orders)})))))))
+
+(deftest ^:parallel table-query-with-source-filters->replacement-snippet-test
+ (testing "Table reference with a single source-filter produces a filtered subquery"
+ (mt/with-metadata-provider meta/metadata-provider
+ (is (= {:replacement-snippet "(SELECT * FROM \"PUBLIC\".\"ORDERS\" WHERE (\"TOTAL\" > 100))"
+ :prepared-statement-args []}
+ (sql.params.substitution/->replacement-snippet-info
+ :h2
+ (params/map->ReferencedTableQuery
+ {:table-id (meta/id :orders)
+ :source-filters [{:field-id (meta/id :orders :total)
+ :op :>
+ :value 100}]}))))))
+ (testing "Table reference with multiple source-filters joins them with AND"
+ (mt/with-metadata-provider meta/metadata-provider
+ (is (= {:replacement-snippet "(SELECT * FROM \"PUBLIC\".\"ORDERS\" WHERE (\"TOTAL\" > 100) AND (\"TOTAL\" <= 500))"
+ :prepared-statement-args []}
+ (sql.params.substitution/->replacement-snippet-info
+ :h2
+ (params/map->ReferencedTableQuery
+ {:table-id (meta/id :orders)
+ :source-filters [{:field-id (meta/id :orders :total)
+ :op :>
+ :value 100}
+ {:field-id (meta/id :orders :total)
+ :op :<=
+ :value 500}]}))))))
+ (testing "Table reference with a timestamp source-filter value"
+ (driver/with-driver ::temporal-unit-alignment-override
+ (mt/with-metadata-provider meta/metadata-provider
+ (let [ts (t/offset-date-time 2024 1 15 0 0 0 0 (t/zone-offset 0))]
+ (is (= {:replacement-snippet "(SELECT * FROM \"PUBLIC\".\"ORDERS\" WHERE (\"CREATED_AT\" > ?))"
+ :prepared-statement-args [ts]}
+ (sql.params.substitution/->replacement-snippet-info
+ ::temporal-unit-alignment-override
+ (params/map->ReferencedTableQuery
+ {:table-id (meta/id :orders)
+ :source-filters [{:field-id (meta/id :orders :created-at)
+ :op :>
+ :value ts}]})))))))))