Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Finish migrating parameters QP/driver code to MBQL 5 (#74467)

Cam Saul23f1fb438009
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 687a11b5207..0471b712837 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -929,10 +929,7 @@
:deprecated-namespace
{:exclude
- [metabase.driver.common.parameters.dates
- metabase.driver.common.parameters.operators
- metabase.driver.common.parameters.values
- metabase.legacy-mbql.normalize
+ [metabase.legacy-mbql.normalize
metabase.legacy-mbql.schema
metabase.legacy-mbql.schema.helpers
metabase.legacy-mbql.schema.macros
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 7b526fdc871..e24ebd569a5 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -845,7 +845,6 @@
config
driver-api
events
- legacy-mbql
lib
lib-be
premium-features
@@ -1766,6 +1765,7 @@
metabase.query-processor.middleware.permissions
metabase.query-processor.parameters.dates
metabase.query-processor.parameters.operators
+ metabase.query-processor.parameters.values
metabase.query-processor.pipeline
metabase.query-processor.pivot
metabase.query-processor.preprocess
@@ -1776,8 +1776,7 @@
metabase.query-processor.streaming.interface
metabase.query-processor.timezone
metabase.query-processor.util
- metabase.query-processor.util.add-alias-info
- metabase.query-processor.util.persisted-cache}
+ metabase.query-processor.util.add-alias-info}
:uses #{analytics
analytics-interface
analyze
@@ -2677,6 +2676,7 @@
driver
events
lib
+ lib-be
models
query-processor
util
diff --git a/dev/src/dev/deps_graph.clj b/dev/src/dev/deps_graph.clj
index 90269c40d61..2571fa508cd 100644
--- a/dev/src/dev/deps_graph.clj
+++ b/dev/src/dev/deps_graph.clj
@@ -464,7 +464,7 @@
([deps]
(let [kondo-config (kondo-config)]
(-> (ddiff/diff
- (update-vals kondo-config #(dissoc % :team :friends))
+ (update-vals kondo-config #(dissoc % :team :friends :model-imports :model-exports))
(generate-config deps kondo-config))
ddiff/minimize
kondo-config-diff-ignore-any
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index 06ecc7194e7..0ba96fccb0b 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -52,6 +52,18 @@ title: Driver interface changelog
`:db` AST slot for cross-DB consumers); drivers that emit a 3-part `catalog.schema.table`
identifier (SQL Server, BigQuery) override to `[:db :schema]`.
+- The `metabase.driver.common.parameters.values` namespace, deprecated in 0.57.0, has been removed. Use the equivalent
+ QP namespace, `metabase.query-processor.parameters.values`, instead. Note that this namespace operates on MBQL 5
+ rather than MBQL 4.
+
+- `metabase.driver/substitute-native-parameters` has been deprecated; `substitute-native-parameters-in-stage-method`
+ replaces it. The new method operates on MBQL 5 rather than MBQL 4. `substitute-native-parameters` is no longer
+ called in Metabase 0.62.0+; on the off chance that you implemented it, please implement
+ `substitute-native-parameters-in-stage-method` instead as soon as possible.
+
+- `metabase.driver-api.core/wrap-value-literals-in-mbql` has been removed; use
+ `metabase.driver-api.core/wrap-value-literals-in-mbql5` instead.
+
## Metabase 0.61.0
- Added the following driver multimethods to support MBQL5 compilation migration:
diff --git a/enterprise/backend/src/metabase_enterprise/action_v2/data_editing.clj b/enterprise/backend/src/metabase_enterprise/action_v2/data_editing.clj
index 5ba83735550..0310f8ecd4e 100644
--- a/enterprise/backend/src/metabase_enterprise/action_v2/data_editing.clj
+++ b/enterprise/backend/src/metabase_enterprise/action_v2/data_editing.clj
@@ -6,11 +6,10 @@
[metabase-enterprise.action-v2.models.undo :as undo]
[metabase.actions.core :as actions]
[metabase.api.common :as api]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.query-processor.core :as qp]
- ;; legacy usage -- don't do things like this going forward
- ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.util :as u]
[metabase.util.queue :as queue]
[metabase.warehouse-schema.models.field-values :as field-values]
@@ -79,28 +78,27 @@
(let [{:keys [db_id]} (api/check-404 (t2/select-one :model/Table table-id))
row-pks (seq (map (partial get-row-pks pk-fields) rows))]
(assert (every? valid-pks row-pks) "All rows must have valid primary keys")
- (qp.store/with-metadata-provider db_id
- (let [mp (qp.store/metadata-provider)
- query (lib/query mp (lib.metadata/table mp table-id))
- query (lib/filter
- query
- ;; We can optimize the most common case considerably.
- (if (= 1 (count pk-fields))
- (apply lib/in
- (lib.metadata/field mp (:id (first pk-fields)))
- (map (comp val first) row-pks))
- ;; Optimizing this could be done in many cases, but it would be complex.
- (apply* lib/or
- (for [row-pk row-pks]
- (apply* lib/and
- (for [field pk-fields]
- (lib/= (lib.metadata/field mp (:id field))
- (get row-pk (:name field)))))))))]
- (->> query
- qp/userland-query-with-default-constraints
- qp/process-query
- :data
- qp-result->row-map))))))
+ (let [mp (lib-be/application-database-metadata-provider db_id)
+ query (lib/query mp (lib.metadata/table mp table-id))
+ query (lib/filter
+ query
+ ;; We can optimize the most common case considerably.
+ (if (= 1 (count pk-fields))
+ (apply lib/in
+ (lib.metadata/field mp (:id (first pk-fields)))
+ (map (comp val first) row-pks))
+ ;; Optimizing this could be done in many cases, but it would be complex.
+ (apply* lib/or
+ (for [row-pk row-pks]
+ (apply* lib/and
+ (for [field pk-fields]
+ (lib/= (lib.metadata/field mp (:id field))
+ (get row-pk (:name field)))))))))]
+ (->> query
+ qp/userland-query-with-default-constraints
+ qp/process-query
+ :data
+ qp-result->row-map)))))
(defn apply-coercions
"For fields that have a coercion_strategy, apply the coercion function (defined in data-editing.coerce) to the corresponding value in each row.
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.clj
index 8ed475390ab..e7a48458b3a 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj
@@ -523,9 +523,9 @@
(mongo.connection/with-mongo-client [_ (driver-api/database (driver-api/metadata-provider))]
(mongo.execute/execute-reducible-query query respond)))
-(defmethod driver/substitute-native-parameters :mongo
- [driver inner-query]
- (mongo.params/substitute-native-parameters driver inner-query))
+(defmethod driver/substitute-native-parameters-in-stage-method :mongo
+ [driver metadata-providerable stage]
+ (mongo.params/substitute-native-parameters driver metadata-providerable stage))
(defmethod driver/db-start-of-week :mongo
[_]
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj b/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj
index d07969cff92..3fd7af41491 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj
@@ -4,11 +4,13 @@
[clojure.string :as str]
[java-time.api :as t]
[metabase.driver-api.core :as driver-api]
- [metabase.driver.common.parameters.values :as params.values]
[metabase.driver.mongo.query-processor :as mongo.qp]
[metabase.lib.core :as lib]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.query-processor.parameters.dates :as params.dates]
[metabase.query-processor.parameters.operators :as params.ops]
+ [metabase.query-processor.parameters.values :as params.values]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.i18n :refer [tru]]
@@ -229,8 +231,10 @@
(when-not (= x <>)
(log/debugf "Substituted %s -> %s" (pr-str x) (pr-str <>))))))
-(defn substitute-native-parameters
+(mu/defn substitute-native-parameters :- ::lib.schema/stage.native
"Implementation of [[metabase.driver/substitute-native-parameters]] for MongoDB."
- [_driver inner-query]
- (let [param->value (params.values/query->params-map inner-query)]
- (update inner-query :query (partial perf/postwalk (partial parse-and-substitute param->value)))))
+ [_driver :- :keyword
+ metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ stage :- ::lib.schema/stage.native]
+ (let [param->value (params.values/stage->params-map metadata-providerable stage)]
+ (update stage :native (partial perf/postwalk (partial parse-and-substitute param->value)))))
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj
index 7962891ff30..446e94c7583 100644
--- a/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj
+++ b/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj
@@ -9,6 +9,7 @@
[java-time.api :as t]
[metabase.driver.mongo.parameters :as mongo.params]
[metabase.lib.core :as lib]
+ [metabase.lib.test-metadata :as meta]
[metabase.query-processor.test :as qp]
[metabase.test :as mt]
[metabase.util.json :as json]
@@ -142,6 +143,11 @@
(str (to-bson k) ": " (to-bson v))))
"}")))
+(defn- bson-literal [s]
+ (reify ToBSON
+ (to-bson [_this]
+ s)))
+
(defn- bson-fn-call [f & args]
(reify ToBSON
(to-bson [_]
@@ -283,6 +289,29 @@
:content (to-bson {"price" {:$gt 2}})}}
["[{$match: " (lib/parsed-param "snippet: high price") "}]"]))))))
+(deftest ^:parallel substitute-native-parameters-test
+ (let [mp meta/metadata-provider
+ query (lib/query mp {:database (meta/id)
+ :type :native
+ :native {:query (to-bson [{:$match (bson-literal "{{date}}")}
+ {:$sort {:_id 1}}])
+ :collection "checkins"
+ :template-tags {"date" {:name "date"
+ :display-name "Date"
+ :type :dimension
+ :widget-type :date/all-options
+ :dimension [:field (meta/id :checkins :date) nil]}}}
+ :parameters [{:type :date/range
+ :target [:dimension [:template-tag "date"]]
+ :value "2014-03-01~2014-03-02"}]})
+ stage (-> (lib/query-stage query 0)
+ (assoc :parameters (:parameters query)))]
+ (is (=? {:native (to-bson
+ [{:$match {:$and [{"DATE" {:$gte (ISODate "2014-03-01T00:00:00Z")}}
+ {"DATE" {:$lt (ISODate "2014-03-03T00:00:00Z")}}]}}
+ {:$sort {:_id 1}}])}
+ (mongo.params/substitute-native-parameters :mongo mp stage)))))
+
(deftest e2e-field-filter-test
(mt/test-driver :mongo
(mt/with-temporary-setting-values [report-timezone "UTC"]
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index 1c3c58e0b1b..da58da55fec 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -17,6 +17,8 @@
[metabase.classloader.core :as classloader]
[metabase.driver.impl :as driver.impl]
[metabase.driver.settings]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.query-processor.error-type :as qp.error-type]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
@@ -1124,23 +1126,39 @@
nil)
(defmulti substitute-native-parameters
+ "DEPRECATED: Implement [[substitute-native-parameters-method]] going forward."
+ {:added "0.34.0" :arglists '([driver legacy-mbql-native-inner-query]), :deprecated "0.62.0"}
+ dispatch-on-initialized-driver
+ :hierarchy #'hierarchy)
+
+(defmulti substitute-native-parameters-in-stage-method
+ "Implementation for [[substitute-native-parameters-in-stage]]; avoid calling this directly and
+ call [[substitute-native-parameters-in-stage]] instead. Only use this for `defmethod` method implementations."
+ {:added "0.62.0" :arglists '([driver metadata-providerable native-query-stage])}
+ dispatch-on-initialized-driver
+ :hierarchy #'hierarchy)
+
+(mu/defn substitute-native-parameters-in-stage :- ::lib.schema/stage.native
"For drivers that support `:native-parameters`. Substitute parameters in a normalized 'inner' native query.
- {:query \"SELECT count(*) FROM table WHERE id = {{param}}\"
+ {:lib/type :mbql.stage/native
+ :native \"SELECT count(*) FROM table WHERE id = {{param}}\"
:template-tags {:param {:name \"param\", :display-name \"Param\", :type :number}}
:parameters [{:type :number
:target [:variable [:template-tag \"param\"]]
:value 2}]}
->
- {:query \"SELECT count(*) FROM table WHERE id = 2\"}
+ {:native \"SELECT count(*) FROM table WHERE id = 2\", ...}
Much of the implementation for this method is shared across drivers and lives in the
- `metabase.driver.common.parameters.*` namespaces. See the `:sql` and `:mongo` drivers for sample implementations of
- this method.`Driver-agnostic end-to-end native parameter tests live in
+ `metabase.query-processor.parameters.*` namespaces. See the `:sql` and `:mongo` drivers for sample implementations
+ of this method. Driver-agnostic end-to-end native parameter tests live in
[[metabase.query-processor.parameters-test]] and other namespaces."
- {:added "0.34.0" :arglists '([driver inner-native-query])}
- dispatch-on-initialized-driver
- :hierarchy #'hierarchy)
+ {:added "0.62.0"}
+ [driver :- :keyword
+ metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ native-stage :- ::lib.schema/stage.native]
+ (substitute-native-parameters-in-stage-method driver metadata-providerable native-stage))
(defmulti default-field-order
"Return how fields should be sorted by default for this database."
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index e04650cc4fa..dcb53ac6874 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -5,17 +5,19 @@
[clojure.set :as set]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
- ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.values :as params.values]
[metabase.driver.connection :as driver.conn]
- [metabase.driver.sql.normalize :as sql.normalize]
+ [metabase.driver.sql.normalize]
[metabase.driver.sql.parameters.substitute :as sql.params.substitute]
- [metabase.driver.sql.parameters.substitution :as sql.params.substitution]
+ [metabase.driver.sql.parameters.substitution]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.util :as sql.u]
[metabase.driver.util :as driver.u]
[metabase.lib.core :as lib]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.util :as lib.util]
[metabase.query-processor.error-type :as qp.error-type]
+ [metabase.query-processor.parameters.values :as params.values]
[metabase.sql-tools.core :as sql-tools]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
@@ -23,7 +25,8 @@
[metabase.util.performance :refer [mapv]]
[potemkin :as p]))
-(comment sql.params.substitution/keep-me) ; this is so `cljr-clean-ns` and the linter don't remove the `:require`
+(comment metabase.driver.sql.parameters.substitution/keep-me
+ metabase.driver.sql.normalize/keep-me) ; this is so `cljr-clean-ns` and the linter don't remove the `:require`
(driver/register! :sql, :abstract? true)
@@ -70,15 +73,16 @@
[driver native-form]
(sql.u/format-sql-and-fix-params driver native-form))
-(mu/defmethod driver/substitute-native-parameters :sql
- [_driver {:keys [query] :as inner-query} :- [:and [:map-of :keyword :any] [:map {:query driver-api/schema.common.non-blank-string}]]]
- (let [params-map (params.values/query->params-map inner-query)
- referenced-card-ids (params.values/referenced-card-ids params-map)
- [query params] (-> query
- lib/parse-parameters
- (sql.params.substitute/substitute params-map))]
- (cond-> (assoc inner-query
- :query query
+(mu/defmethod driver/substitute-native-parameters-in-stage-method :sql :- ::lib.schema/stage.native
+ [_driver :- :keyword
+ metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ {native-query :native, :as native-stage} :- ::lib.schema/stage.native]
+ (let [params-map (params.values/stage->params-map metadata-providerable native-stage)
+ referenced-card-ids (params.values/referenced-card-ids params-map)
+ parsed-query (lib/parse-parameters native-query)
+ [native-query params] (sql.params.substitute/substitute metadata-providerable parsed-query params-map)]
+ (cond-> (assoc native-stage
+ :native native-query
:params params)
(seq referenced-card-ids)
(update :query-permissions/referenced-card-ids set/union referenced-card-ids))))
@@ -308,5 +312,5 @@
;;; +----------------------------------------------------------------------------------------------------------------+
(p/import-vars
- [sql.params.substitution ->prepared-substitution PreparedStatementSubstitution]
- [sql.normalize default-schema normalize-error normalize-name reserved-literal])
+ [metabase.driver.sql.parameters.substitution ->prepared-substitution PreparedStatementSubstitution]
+ [metabase.driver.sql.normalize default-schema normalize-error normalize-name reserved-literal])
diff --git a/src/metabase/driver/sql/parameters/substitute.clj b/src/metabase/driver/sql/parameters/substitute.clj
index d059998f5d5..b26c2db789f 100644
--- a/src/metabase/driver/sql/parameters/substitute.clj
+++ b/src/metabase/driver/sql/parameters/substitute.clj
@@ -6,6 +6,8 @@
[metabase.driver-api.core :as driver-api]
[metabase.driver.sql.parameters.substitution :as sql.params.substitution]
[metabase.lib.core :as lib]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
+ [metabase.query-processor.store :as qp.store]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
@@ -17,6 +19,10 @@
(declare #^:private substitute*)
+(defn- ->replacement-snippet-info [metadata-providerable x]
+ (qp.store/with-metadata-provider metadata-providerable
+ (sql.params.substitution/->replacement-snippet-info driver/*driver* x)))
+
(mr/def ::acc
[:maybe
[:tuple
@@ -25,7 +31,8 @@
#_missing [:maybe any?]]])
(mu/defn- substitute-field-param :- ::acc
- [[sql args missing] :- ::acc
+ [metadata-providerable
+ [sql args missing] :- ::acc
in-optional?
k
{:keys [_field value], :as v}]
@@ -34,26 +41,35 @@
[sql args (conj missing k)]
;; otherwise no values get replaced with `1 = 1` and other values get replaced normally
(let [{:keys [replacement-snippet prepared-statement-args]}
- (sql.params.substitution/->replacement-snippet-info driver/*driver* v)]
+ (->replacement-snippet-info metadata-providerable v)]
[(str sql replacement-snippet) (concat args prepared-statement-args) missing])))
(mu/defn- substitute-simple-query :- ::acc
- [[sql args missing] :- ::acc
+ [metadata-providerable
+ [sql args missing] :- ::acc
v]
(let [{:keys [replacement-snippet prepared-statement-args]}
- (sql.params.substitution/->replacement-snippet-info driver/*driver* v)]
+ (->replacement-snippet-info metadata-providerable v)]
[(str sql replacement-snippet) (concat args prepared-statement-args) missing]))
(mu/defn- substitute-native-query-snippet :- ::acc
- [param->value [sql args missing] in-optional? v]
- (let [{:keys [replacement-snippet]} (sql.params.substitution/->replacement-snippet-info driver/*driver* v)
- [processed-snippet snippet-args snippet-missing] (substitute* param->value (lib/parse-parameters replacement-snippet) in-optional?)]
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ param->value
+ [sql args missing]
+ in-optional? v]
+ (let [{:keys [replacement-snippet]} (->replacement-snippet-info metadata-providerable v)
+ [processed-snippet snippet-args snippet-missing] (substitute*
+ metadata-providerable
+ param->value
+ (lib/parse-parameters replacement-snippet)
+ in-optional?)]
[(str sql processed-snippet)
(not-empty (concat args snippet-args))
(not-empty (concat missing snippet-missing))]))
(mu/defn- substitute-param :- ::acc
- [param->value
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ param->value
[sql args missing] :- ::acc
in-optional?
{:keys [k]} :- :metabase.lib.parameters.parse.types/param]
@@ -63,36 +79,38 @@
(cond
(or (lib/parsed-field-filter-param? v)
(lib/parsed-temporal-unit-param? v))
- (substitute-field-param [sql args missing] in-optional? k v)
+ (substitute-field-param metadata-providerable [sql args missing] in-optional? k v)
(or (lib/parsed-referenced-card-query-param? v)
(lib/parsed-referenced-table-query-param? v))
- (substitute-simple-query [sql args missing] v)
+ (substitute-simple-query metadata-providerable [sql args missing] v)
(lib/parsed-referenced-query-snippet-param? v)
- (substitute-native-query-snippet param->value [sql args missing] in-optional? v)
+ (substitute-native-query-snippet metadata-providerable param->value [sql args missing] in-optional? v)
(= v lib/parsed-param-no-value-placeholder)
[sql args (conj missing k)]
:else
(let [{:keys [replacement-snippet prepared-statement-args]}
- (sql.params.substitution/->replacement-snippet-info driver/*driver* v)]
+ (->replacement-snippet-info metadata-providerable v)]
[(str sql replacement-snippet) (concat args prepared-statement-args) missing])))))
(mu/defn- substitute-optional :- ::acc
- [param->value
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ param->value
[sql args missing] :- ::acc
{subclauses :args}]
- (let [[opt-sql opt-args opt-missing] (substitute* param->value subclauses true)]
+ (let [[opt-sql opt-args opt-missing] (substitute* metadata-providerable param->value subclauses true)]
(if (seq opt-missing)
[sql args missing]
[(str sql opt-sql) (concat args opt-args) missing])))
(mu/defn- substitute* :- ::acc
"Returns a sequence of `[replaced-sql-string jdbc-args missing-parameters]`."
- [param->value :- [:maybe [:map-of string? any?]]
- parsed
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ param->value :- [:maybe [:map-of string? any?]]
+ parsed :- [:sequential :metabase.lib.parameters.parse/parsed-token]
in-optional?]
(reduce
(fn [[sql args missing] x]
@@ -101,17 +119,17 @@
[(str sql x) args missing]
(lib/parsed-param? x)
- (substitute-param param->value [sql args missing] in-optional? x)
+ (substitute-param metadata-providerable param->value [sql args missing] in-optional? x)
(lib/parsed-optional-param? x)
- (substitute-optional param->value [sql args missing] x)
+ (substitute-optional metadata-providerable param->value [sql args missing] x)
:else
(throw (ex-info (format "Unexpected parsed param ^%s %s" (some-> x class .getCanonicalName) (pr-str x)) {:x x}))))
nil
parsed))
-(defn substitute
+(mu/defn substitute :- [:tuple #_sql :string #_args [:maybe [:sequential :any]]]
"Substitute `Optional` and `Param` objects in a `parsed-query`, a sequence of parsed string fragments and tokens, with
the values from the map `param->value` (using logic from `substitution` to decide what replacement SQL should be
generated).
@@ -119,10 +137,12 @@
(substitute [\"select * from foobars where bird_type = \" (param \"bird_type\")]
{\"bird_type\" \"Steller's Jay\"})
;; -> [\"select * from foobars where bird_type = ?\" [\"Steller's Jay\"]]"
- [parsed-query param->value]
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ parsed-query :- [:sequential :metabase.lib.parameters.parse/parsed-token]
+ param->value :- [:maybe [:map-of string? any?]]]
(log/tracef "Substituting params\n%s\nin query:\n%s" (u/pprint-to-str param->value) (u/pprint-to-str parsed-query))
(let [[sql args missing] (try
- (substitute* param->value parsed-query false)
+ (substitute* metadata-providerable param->value parsed-query false)
(catch Throwable e
(throw (ex-info (tru "Unable to substitute parameters: {0}" (ex-message e))
{:type (or (:type (ex-data e)) driver-api/qp.error-type.qp)
diff --git a/src/metabase/driver/sql/parameters/substitution.clj b/src/metabase/driver/sql/parameters/substitution.clj
index 61decde8cb7..ec477195eb8 100644
--- a/src/metabase/driver/sql/parameters/substitution.clj
+++ b/src/metabase/driver/sql/parameters/substitution.clj
@@ -119,6 +119,7 @@
[:replacement-snippet {:optional true} :string] ; allowed to be blank if this is an optional param
[:prepared-statement-args {:optional true} [:maybe [:sequential :any]]]])
+;; TODO (Cam 2026-05-21) Update this to take an explicit `metadata-providerable`
(defmulti ->replacement-snippet-info
"Return information about how `value` should be converted to SQL, as a map with keys `:replacement-snippet` and
`:prepared-statement-args`.
diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.clj
index 808c09a5625..0ebeeba0587 100644
--- a/src/metabase/driver_api/core.clj
+++ b/src/metabase/driver_api/core.clj
@@ -164,7 +164,6 @@
qp.util.transformations.nest-breakouts/nest-breakouts-in-stages-with-window-aggregation
qp.util/default-query->remark
qp.util/query->remark
- qp.wrap-value-literals/wrap-value-literals-in-mbql
qp.wrap-value-literals/wrap-value-literals-in-mbql5
qp.writeback/execute-write-sql!
qp/process-query
@@ -340,7 +339,7 @@
(def MBQLQuery
"Schema for a legacy MBQL inner query."
- ::mbql.s/MBQLQuery)
+ ::mbql.s/MBQLInnerQuery)
(def Join
"Schema for a legacy MBQL join."
diff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljc
index 56280e1400f..375555a1446 100644
--- a/src/metabase/legacy_mbql/schema.cljc
+++ b/src/metabase/legacy_mbql/schema.cljc
@@ -1690,15 +1690,7 @@
Map of template tag name -> template tag definition"
[:and
- [:map-of
- {:decode/normalize (fn [m]
- (when (and (map? m)
- (seq m))
- (update-keys m (fn [k]
- (cond-> k
- (keyword? k) u/qualified-name)))))}
- ::lib.schema.common/non-blank-string
- [:ref ::TemplateTag]]
+ [:map-of ::lib.schema.common/non-blank-string [:ref ::TemplateTag]]
[:ref ::lib.schema.template-tag/template-tag-map.validate-names]])
(defn- remove-empty-keys [m {:keys [non-empty-keys non-nil-keys]}]
@@ -1732,33 +1724,44 @@
:source-table ":source-table is only allowed in MBQL inner queries."
:fields ":fields is only allowed in MBQL inner queries."})])
-(mr/def ::NativeQuery
+(mr/def ::TopLevelNativeInnerQuery
"Schema for a valid, normalized native [inner] query."
- [:merge
- {:decode/normalize #'remove-empty-keys-from-native-inner-query}
- ::NativeQuery.Common
- [:map
- [:query :some]]])
+ [:and
+ [:merge
+ {:decode/normalize #'remove-empty-keys-from-native-inner-query}
+ ::NativeQuery.Common
+ [:map
+ [:query :some]]]
+ (lib.schema.common/disallowed-keys
+ {:native "A top-level native inner query should have the :query key, not :native"})])
(mr/def ::NativeSourceQuery
- [:merge
- {:decode/normalize #'remove-empty-keys-from-native-inner-query}
- ::NativeQuery.Common
- [:map
- [:native :some]]])
+ [:and
+ [:merge
+ {:decode/normalize #'remove-empty-keys-from-native-inner-query}
+ ::NativeQuery.Common
+ [:map
+ [:native :some]]]
+ (lib.schema.common/disallowed-keys
+ {:query "A top-level native inner query should have the :native key, not :query"})])
(mr/def ::SourceQuery
"Schema for a valid value for a `:source-query`."
- [:multi
- {:dispatch (fn [x]
- (if ((every-pred map? :native) x)
- :native
- :mbql))}
- ;; when using native queries as source queries the schema is exactly the same except use `:native` in place of
- ;; `:query` for reasons I do not fully remember (perhaps to make it easier to differentiate them from MBQL source
- ;; queries).
- [:native [:ref ::NativeSourceQuery]]
- [:mbql [:ref ::MBQLQuery]]])
+ [:and
+ ;; normalize the keys in the map first so we can check for the presence of `:native` versus `:mbql` in the `:multi`
+ ;; schema below
+ [:map
+ {:decode/normalize lib.schema.common/normalize-map}]
+ [:multi
+ {:dispatch (fn [x]
+ (if ((every-pred map? :native) x)
+ :native
+ :mbql))}
+ ;; when using native queries as source queries the schema is exactly the same except use `:native` in place of
+ ;; `:query` for reasons I do not fully remember (perhaps to make it easier to differentiate them from MBQL source
+ ;; queries).
+ [:native [:ref ::NativeSourceQuery]]
+ [:mbql [:ref ::MBQLInnerQuery]]]])
(defn- normalize-legacy-column
"Normalize legacy column metadata when using [[metabase.lib.normalize/normalize]]."
@@ -2051,7 +2054,7 @@
(into #{} (map without-temporal-unit) breakout)
(into #{} (map without-temporal-unit) fields))))]))
-(mr/def ::MBQLQuery
+(mr/def ::MBQLInnerQuery
[:and
[:map
{:decode/normalize lib.schema.common/normalize-map}
@@ -2095,7 +2098,9 @@
:type "An inner query must not include :type, this will cause us to mix it up with an outer query"
:aggregation-idents ":aggregation-idents is deprecated and should not be used"
:breakout-idents ":breakout-idents is deprecated and should not be used"
- :expression-idents ":expression-idents is deprecated and should not be used"})])
+ :expression-idents ":expression-idents is deprecated and should not be used"
+ :query "An inner query should not itself contain :query -- something must have been nested improperly"
+ :native "An inner MBQL query should not have :native -- this is only for inner native queries"})])
(mr/def ::WidgetType
"Schema for valid values of `:widget-type` for a `::TemplateTag.FieldFilter`."
@@ -2198,8 +2203,8 @@
{:decode/normalize helpers/normalize-keyword
:description "Type of query. `:query` = MBQL; `:native` = native."}
:query :native]]
- [:native {:optional true} [:ref ::NativeQuery]]
- [:query {:optional true} [:ref ::MBQLQuery]]
+ [:native {:optional true} [:ref ::TopLevelNativeInnerQuery]]
+ [:query {:optional true} [:ref ::MBQLInnerQuery]]
[:parameters {:optional true} [:maybe [:ref ::lib.schema.parameter/parameters]]]
;;
;; OPTIONS
diff --git a/src/metabase/lib/convert.cljc b/src/metabase/lib/convert.cljc
index c11a1eac55a..4eecd03060a 100644
--- a/src/metabase/lib/convert.cljc
+++ b/src/metabase/lib/convert.cljc
@@ -456,10 +456,12 @@
"Convert a legacy 'inner query' to a full legacy 'outer query' so you can pass it to stuff
like [[metabase.legacy-mbql.normalize/normalize]], and then probably to [[->mbql5]]."
[database-id inner-query]
- (merge {:database database-id, :type :query}
+ (merge {:database database-id}
(if (:native inner-query)
- {:native (set/rename-keys inner-query {:native :query})}
- {:query inner-query})))
+ {:native (set/rename-keys inner-query {:native :query})
+ :type :native}
+ {:query inner-query
+ :type :query})))
(defmulti ->legacy-MBQL
"Coerce something to legacy MBQL (the version of MBQL understood by the query processor and Metabase Lib v1) if it's
diff --git a/src/metabase/lib/native.cljc b/src/metabase/lib/native.cljc
index 7d5fe9e9378..a5fc78c21e2 100644
--- a/src/metabase/lib/native.cljc
+++ b/src/metabase/lib/native.cljc
@@ -6,6 +6,7 @@
[clojure.string :as str]
[medley.core :as m]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.normalize :as lib.normalize]
[metabase.lib.parameters.parse :as lib.params.parse]
[metabase.lib.parameters.parse.types :as lib.params.parse.types]
[metabase.lib.parse :as lib.parse]
@@ -24,7 +25,7 @@
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.match :as match]
- [metabase.util.performance :refer [every? mapv select-keys some empty? not-empty]]))
+ [metabase.util.performance :refer [empty? every? mapv not-empty select-keys some]]))
(defn- finish-tag [{tag-name :name :as tag}]
(merge tag
@@ -153,7 +154,7 @@
;; Otherwise just an empty map, no tags.
{}))))
-(defn- assert-native-query [stage]
+(defn- assert-native-stage [stage]
(assert (= (:lib/type stage) :mbql.stage/native) (i18n/tru "Must be a native query")))
(def ^:private all-native-extra-keys
@@ -184,7 +185,7 @@
stage-without-old-extras (apply dissoc stage extras-to-remove)
result (merge stage-without-old-extras (select-keys native-extras required-extras))
missing-keys (set/difference required-extras (set (keys native-extras)))]
- (assert-native-query (lib.util/query-stage query 0))
+ (assert-native-stage (lib.util/query-stage query 0))
(assert (empty? missing-keys)
(i18n/tru "Missing extra, required keys for native query: {0}"
(pr-str missing-keys)))
@@ -215,7 +216,7 @@
Native extras must be provided if the new database requires it."
[query :- ::lib.schema/query
metadata-provider :- ::lib.schema.metadata/metadata-providerable]
- (assert-native-query (lib.util/query-stage query 0))
+ (assert-native-stage (lib.util/query-stage query 0))
(let [stages-without-fields (->> (:stages query)
(mapv (fn [stage]
(update stage :template-tags update-vals #(dissoc % :dimension)))))]
@@ -234,7 +235,7 @@
(lib.util/update-query-stage
query 0
(fn [{existing-tags :template-tags :as stage}]
- (assert-native-query stage)
+ (assert-native-stage stage)
(assoc stage
:native inner-query
:template-tags (extract-template-tags query inner-query existing-tags)))))
@@ -242,7 +243,10 @@
;;; TODO (Cam 7/16/25) -- this really doesn't seem to do what I'd expect, maybe we should rename it something like
;;; `with-replaced-template-tags`. It only replaces tags you specify rather then completely setting a new list
(mu/defn with-template-tags :- ::lib.schema/query
- "Updates the native query's template tags."
+ "Updates the native query's template tags.
+
+ Note that this only updates existing tags, and will not blindly set them to `template-tags`; however, initializing a
+ query with [[native-query]] should populate them automatically by way of [[extract-template-tags]]."
[query :- ::lib.schema/query
updated-tags :- ::lib.schema.template-tag/template-tag-map]
(letfn [(update-template-tags [existing-tags]
@@ -251,22 +255,25 @@
;; https://metaboat.slack.com/archives/C0645JP1W81/p1759975007383889?thread_ts=1759289751.539169&cid=C0645JP1W81
;;
;; first, filter out the tags in `updated-tags` not in existing tags, preserving the original order.
- (let [tags (reduce-kv
- (fn [m k v]
- (cond-> m
- (contains? existing-tags k) (assoc k v)))
- {}
- updated-tags)]
+ (let [updates (reduce-kv
+ (fn [m updated-k updated-v]
+ (let [updated-k (lib.params.parse/match-and-normalize-tag-name updated-k)]
+ (cond-> m
+ (contains? existing-tags updated-k) (assoc updated-k updated-v))))
+ {}
+ updated-tags)]
;; merge in old values that weren't in the `updated-tags` map
(reduce-kv
- (fn [m k v]
- (cond-> m
- (not (contains? m k)) (assoc k v)))
- tags
+ (fn [updates existing-k existing-v]
+ (cond-> updates
+ (not (contains? updates existing-k)) (assoc existing-k existing-v)))
+ updates
existing-tags)))
(update-stage [stage]
- (assert-native-query stage)
- (update stage :template-tags update-template-tags))]
+ (assert-native-stage stage)
+ (-> stage
+ (update :template-tags update-template-tags)
+ (->> (lib.normalize/normalize ::lib.schema/stage.native))))]
(lib.util/update-query-stage query 0 update-stage)))
(mu/defn raw-native-query :- some?
@@ -312,7 +319,7 @@
This is only filled in by [[metabase.warehouses-rest.api/add-native-perms-info]]
and added to metadata when pulling a database from the list of dbs in js."
[query :- ::lib.schema/query]
- (assert-native-query (lib.util/query-stage query 0))
+ (assert-native-stage (lib.util/query-stage query 0))
(= :write (:native-permissions (lib.metadata/database query))))
(mu/defn- validate-template-tag :- [:sequential [:map [:error/message :string] [:tag-name :string]]]
@@ -349,7 +356,7 @@
"Returns the database engine.
Must be a native query"
[query :- ::lib.schema/query]
- (assert-native-query (lib.util/query-stage query 0))
+ (assert-native-stage (lib.util/query-stage query 0))
(:engine (lib.metadata/database query)))
(defn- get-parameter-value
diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljc
index a04a81b5b7d..5ce41f32426 100644
--- a/src/metabase/lib/schema.cljc
+++ b/src/metabase/lib/schema.cljc
@@ -363,7 +363,8 @@
(common/disallowed-keys
{:source-metadata "A query stage should not have :source-metadata, the prior stage should have :lib/stage-metadata instead"
:source-query ":source-query is not allowed in MBQL 5 queries."
- :type ":type is not allowed in a query stage in any version of MBQL"})])
+ :type ":type is not allowed in a query stage in any version of MBQL"
+ :database ":database is not allowed in a query stage, only at the top level of a query."})])
(mr/def ::stage.initial
[:multi {:dispatch lib-type
diff --git a/src/metabase/lib/schema/common.cljc b/src/metabase/lib/schema/common.cljc
index 12f4c51ebd1..f4484b9b451 100644
--- a/src/metabase/lib/schema/common.cljc
+++ b/src/metabase/lib/schema/common.cljc
@@ -93,7 +93,9 @@
[:and
{:error/message "non-blank string"
:json-schema {:type "string" :minLength 1}}
- [:string {:min 1}]
+ [:string {:min 1, :decode/normalize (fn [x]
+ (cond-> x
+ (keyword? x) u/qualified-name))}]
[:fn
{:error/message "non-blank string"}
(complement str/blank?)]])
diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljc
index ad99f98ce5e..2c6a8ace06d 100644
--- a/src/metabase/lib/schema/metadata.cljc
+++ b/src/metabase/lib/schema/metadata.cljc
@@ -617,6 +617,9 @@
[:legacy
[:ref :metabase.legacy-mbql.schema/legacy-column-metadata]]])
+;;; TODO (Cam 2026-05-26) check whether we can change `::card.query` to point to `:metabase.lib.schema/query` now that
+;;; Cards should always have MBQL 5
+
(defn- normalize-card-query [query]
(when query
(let [query (lib.schema.common/normalize-map query)]
diff --git a/src/metabase/lib/schema/template_tag.cljc b/src/metabase/lib/schema/template_tag.cljc
index d072d2263fd..b562fec0c96 100644
--- a/src/metabase/lib/schema/template_tag.cljc
+++ b/src/metabase/lib/schema/template_tag.cljc
@@ -107,6 +107,8 @@
[:map
[:type [:= :snippet]]
[:snippet-name ::common/non-blank-string]
+ ;; TODO (Cam 2026-05-28) why isn't this required? Parameter substitution fails if it's not
+ ;; present (see [[metabase.query-processor.parameters.values-test/snippet-validation-test]]
[:snippet-id {:optional true} ::id/snippet]
;; database to which this Snippet belongs. Doesn't always seem to be specified.
[:database {:optional true} ::id/database]]]
diff --git a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj
index f4e87fa0175..aa309c0ff0c 100644
--- a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj
+++ b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj
@@ -33,7 +33,7 @@
(lib/query-from-legacy-inner-query
(qp.store/metadata-provider)
(:id (lib.metadata/database (qp.store/metadata-provider)))
- (mbql.normalize/normalize ::mbql.s/MBQLQuery inner-query))
+ (mbql.normalize/normalize ::mbql.s/MBQLInnerQuery inner-query))
(catch Throwable e
(throw (ex-info (tru "Error converting query to MBQL 5: {0}" (ex-message e))
{:inner-query inner-query, :type qp.error-type/qp}
diff --git a/src/metabase/query_processor/middleware/parameters/native.clj b/src/metabase/query_processor/middleware/parameters/native.clj
index 4ce565ea151..27302b18027 100644
--- a/src/metabase/query_processor/middleware/parameters/native.clj
+++ b/src/metabase/query_processor/middleware/parameters/native.clj
@@ -12,9 +12,8 @@
replaced as usual and the rest of the clause (`AND ...`) is included in the query as-is
Native parameter parsing and substution logic shared by multiple drivers lives in
- `metabase.driver.common.parameters.*` (for legacy MBQL) and `metabase.query-processor.parameters.*` (for MBQL 5).
- Driver-specific parsing/substitution logic is implemented in `metabase.driver.sql.parameters.*` (for SQL drivers) or
- similar namespaces for others.
+ `metabase.query-processor.parameters.*` (for MBQL 5). Driver-specific parsing/substitution logic is implemented in
+ `metabase.driver.sql.parameters.*` (for SQL drivers) or similar namespaces for others.
The different steps of this process, are similar between existing driver implementations, and are as follows:
@@ -25,30 +24,13 @@
3. `substitute` (and the related namespace `substitution`) replace optional and param objects with appropriate SQL
snippets and prepared statement args, and combine the sequence of fragments back into a single SQL string."
- (:refer-clojure :exclude [mapv])
(:require
- [clojure.set :as set]
- [medley.core :as m]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
- [metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.metadata :as lib.schema.metadata]
- ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
- [metabase.util.malli :as mu]
- [metabase.util.performance :refer [mapv]]))
-
-(defn- substitute-native-parameters* [stage]
- (-> stage
- (set/rename-keys {:native :query})
- (m/update-existing :parameters (fn [parameters]
- (mapv lib/->legacy-MBQL parameters)))
- (m/update-existing :template-tags update-vals lib/->legacy-MBQL)
- (dissoc :lib/type)
- (->> (driver/substitute-native-parameters driver/*driver*))
- (set/rename-keys {:query :native})
- (assoc :lib/type (:lib/type stage))))
+ [metabase.util.malli :as mu]))
(mu/defn expand-stage :- ::lib.schema/stage.native
"Expand parameters inside an *inner* native `query`. Not recursive -- recursive transformations are handled in
@@ -57,10 +39,7 @@
stage :- ::lib.schema/stage.native]
(if-not (driver.u/supports? driver/*driver* :native-parameters (lib.metadata/database metadata-providerable))
stage
- (let [substituted-stage (if (qp.store/initialized?)
- (substitute-native-parameters* stage)
- (qp.store/with-metadata-provider (lib.metadata/->metadata-provider metadata-providerable)
- (substitute-native-parameters* stage)))]
+ (let [substituted-stage (driver/substitute-native-parameters-in-stage driver/*driver* metadata-providerable stage)]
(->
substituted-stage
(dissoc :parameters :template-tags)))))
diff --git a/src/metabase/query_processor/middleware/wrap_value_literals.clj b/src/metabase/query_processor/middleware/wrap_value_literals.clj
index faa8cea0fa1..68a18eaad72 100644
--- a/src/metabase/query_processor/middleware/wrap_value_literals.clj
+++ b/src/metabase/query_processor/middleware/wrap_value_literals.clj
@@ -212,14 +212,14 @@
(lib/absolute-datetime t target-unit)))
(defmethod add-type-info String
- [s {:keys [unit], :as info} & {:keys [parse-datetime-strings?]
- :or {parse-datetime-strings? true}}]
- (if (and (or unit (when info (lib.types.isa/temporal? info)))
+ [s {:keys [unit], :as col} & {:keys [parse-datetime-strings?]
+ :or {parse-datetime-strings? true}}]
+ (if (and (or unit (when col (lib.types.isa/temporal? col)))
parse-datetime-strings?
(seq s))
- (let [effective-type ((some-fn :effective-type :base-type) info)]
+ (let [effective-type ((some-fn :effective-type :base-type) col)]
(parse-temporal-string-literal effective-type s (or unit :default)))
- (value info s)))
+ (value col s)))
;;; -------------------------------------------- wrap-literals-in-clause ---------------------------------------------
@@ -318,11 +318,3 @@
(fn [clause]
(wrap-value-literals-in-clause nil nil clause)))
first)))))
-
-(mu/defn wrap-value-literals-in-mbql :- [:cat :keyword [:* :any]]
- "Wrapper around `wrap-value-literals-in-mbql5` that converts the clause back to legacy MBQL.
- DEPRECATED: This is for legacy compatibility and should not be used in new code."
- {:deprecated "0.57.0"}
- [mbql :- [:cat :keyword [:* :any]]]
- #_{:clj-kondo/ignore [:deprecated-var]}
- (lib/->legacy-MBQL (wrap-value-literals-in-mbql5 mbql)))
diff --git a/src/metabase/driver/common/parameters/values.clj b/src/metabase/query_processor/parameters/values.clj
similarity index 84%
rename from src/metabase/driver/common/parameters/values.clj
rename to src/metabase/query_processor/parameters/values.clj
index 22518c5af40..1ebc7aa444c 100644
--- a/src/metabase/driver/common/parameters/values.clj
+++ b/src/metabase/query_processor/parameters/values.clj
@@ -1,37 +1,28 @@
-(ns metabase.driver.common.parameters.values
+(ns metabase.query-processor.parameters.values
"These functions build a map of information about the types and values of the params used in a query. (These functions
don't parse the query itself, but instead look at the values of `:template-tags` and `:parameters` passed along with
the query.)
- (query->params-map some-inner-query)
+ (stage->params-map metadata-providerable query-stage)
;; -> {\"checkin_date\" {:field {:name \"date\", :parent_id nil, :table_id 1375}
:param {:type \"date/range\"
:target [\"dimension\" [\"template-tag\" \"checkin_date\"]]
- :value \"2015-01-01~2016-09-01\"}}}
-
-
- DEPRECATED: `driver.common.parameters.*` namespaces deal with legacy MBQL queries. Migrate to MBQL-5-friendly
- replacement namespaces. The replacement for this namespace is [[metabase.query-processor.parameters.values]].
-
- TODO (Cam 10/3/25) -- that namespace was introduced in #61158, but then I removed it in a subsequent PR to prune
- unused namespaces. We can't migrate to it if it's gone... please restore it when we start migrating usages of it
- over."
- {:deprecated "0.57.0"}
+ :value \"2015-01-01~2016-09-01\"}}}"
(:refer-clojure :exclude [every? some mapv not-empty get-in])
(:require
[clojure.string :as str]
- [metabase.legacy-mbql.schema :as mbql.s]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
- [metabase.lib.metadata.protocols :as lib.metadata.protocols]
+ [metabase.lib.parameters.parse.types :as params.types]
+ [metabase.lib.schema :as lib.schema]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.schema.parameter :as lib.schema.parameter]
[metabase.lib.schema.template-tag :as lib.schema.template-tag]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.core :as qp]
[metabase.query-processor.error-type :as qp.error-type]
- ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.query-processor.util.persisted-cache :as qp.persistence]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
@@ -46,14 +37,13 @@
(set! *warn-on-reflection* true)
(defmulti ^:private parse-tag
- "Parse a tag by its `:type`, returning an appropriate record type such as
- `metabase.driver.common.parameters.FieldFilter`."
- {:arglists '([tag params])}
- (fn [{tag-type :type} _]
+ "Parse a tag by its `:type`, returning one of the parsed types in [[metabase.lib.parameters.parse.types]]."
+ {:arglists '([metadata-providerable tag params])}
+ (fn [_metadata-providerable {tag-type :type, :as _tag} _params]
(keyword tag-type)))
(defmethod parse-tag :default
- [{tag-type :type, :as tag} _]
+ [_metadata-providerable {tag-type :type, :as tag} _params]
(throw (ex-info (tru "Don''t know how to parse parameter of type {0}" (pr-str tag-type))
{:tag tag})))
@@ -69,8 +59,8 @@
(mr/def ::single-value
"Schema for a valid *single* value for a param."
[:or
- :metabase.lib.parameters.parse.types/field-filter
- :metabase.lib.parameters.parse.types/date
+ ::params.types/field-filter
+ ::params.types/date
number?
:string
:boolean])
@@ -80,7 +70,7 @@
[:maybe
[:or
{:error/message "Valid param value(s)"}
- [:= lib/parsed-param-no-value-placeholder]
+ ::params.types/no-value
::single-value
[:sequential ::single-value]
:map]])
@@ -97,7 +87,7 @@
Targeting template tags by ID is preferable (as of version 44) but targeting by name is supported for backwards
compatibility."
- [tag :- ::mbql.s/TemplateTag]
+ [tag :- ::lib.schema.template-tag/template-tag]
(let [target-type (case (:type tag)
(:dimension :temporal-unit) :dimension
:variable)]
@@ -121,7 +111,7 @@
(mu/defn- tag-params
"Return params from the provided `params` list targeting the provided `tag`."
- [tag :- ::mbql.s/TemplateTag
+ [tag :- ::lib.schema.template-tag/template-tag
params :- [:maybe [:sequential ::lib.schema.parameter/parameter]]]
(let [tag-target? (tag-target-pred tag)]
(seq (for [param params
@@ -144,14 +134,14 @@
param-display-name)
{:type qp.error-type/multiple-conflicting-default-values}))
-(mu/defn- dimension->field-id :- ::lib.schema.id/field
- [dimension]
- (second dimension))
+(mu/defn- field-ref->field-id :- ::lib.schema.id/field
+ [field-ref :- :mbql.clause/field]
+ (last field-ref))
(mu/defn- field-filter-value
"Get parameter value(s) for a Field filter. Returns map if there is a normal single value, or a vector of maps for
multiple values."
- [tag :- ::mbql.s/TemplateTag
+ [tag :- ::lib.schema.template-tag/template-tag
params :- [:maybe [:sequential ::lib.schema.parameter/parameter]]]
(let [matching-params (tag-params tag params)
tag-opts (:options tag)
@@ -215,22 +205,25 @@
:else
lib/parsed-param-no-value-placeholder)))
-(mu/defmethod parse-tag :dimension :- [:maybe :metabase.lib.parameters.parse.types/field-filter]
- [{:keys [dimension alias], :as tag} :- ::mbql.s/TemplateTag
+(mu/defmethod parse-tag :dimension :- ::params.types/field-filter
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ {:keys [dimension alias], :as tag} :- ::lib.schema.template-tag/template-tag
params :- [:maybe [:sequential ::lib.schema.parameter/parameter]]]
- (let [field-id (dimension->field-id dimension)
- field (or (lib.metadata/field (qp.store/metadata-provider) field-id)
+ (let [field-id (field-ref->field-id dimension)
+ field (or (lib.metadata/field metadata-providerable field-id)
(throw (ex-info (tru "Can''t find field with ID: {0}" field-id)
{:field-id field-id, :type qp.error-type/invalid-parameter})))
value (field-filter-value tag params)]
(lib/parsed-field-filter-param field value alias)))
-(mu/defmethod parse-tag :card :- :metabase.lib.parameters.parse.types/referenced-card-query
- [{:keys [card-id], :as tag} :- ::mbql.s/TemplateTag _params]
+(mu/defmethod parse-tag :card :- ::params.types/referenced-card-query
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ {:keys [card-id], :as tag} :- ::lib.schema.template-tag/template-tag
+ _params]
(when-not card-id
(throw (ex-info (tru "Invalid :card parameter: missing `:card-id`")
{:tag tag, :type qp.error-type/invalid-parameter})))
- (let [card (lib.metadata.protocols/card (qp.store/metadata-provider) card-id)
+ (let [card (lib.metadata/card metadata-providerable card-id)
persisted-info (when (= (:type card) :model)
(:lib/persisted-info card))
query (or (:dataset-query card)
@@ -241,7 +234,7 @@
(let [query (assoc query :info {:card-id card-id})
{:keys [query params]} (or (when (qp.persistence/can-substitute? card persisted-info)
{:query (qp.persistence/persisted-info-native-query
- (u/the-id (lib.metadata/database (qp.store/metadata-provider)))
+ (u/the-id (lib.metadata/database metadata-providerable))
persisted-info)})
(qp.compile/compile (qp/disable-max-results query)))]
(lib/parsed-referenced-card-query-param card-id query params))
@@ -255,8 +248,10 @@
:type qp.error-type/invalid-parameter}
e))))))
-(mu/defmethod parse-tag :table :- :metabase.lib.parameters.parse.types/referenced-table-query
- [{:keys [table-id source-filters emit-alias name]} :- ::mbql.s/TemplateTag _params]
+(mu/defmethod parse-tag :table :- ::params.types/referenced-table-query
+ [_metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ {:keys [table-id source-filters emit-alias name]} :- ::lib.schema.template-tag/template-tag
+ _params]
(when (seq source-filters)
(when-let [op (some #(when-not (lib.schema.template-tag/allowed-source-filter-ops (:op %)) %) source-filters)]
(throw (ex-info (tru "Invalid source-filter operator: {0}. Allowed operators: {1}"
@@ -267,13 +262,14 @@
(not-empty source-filters)
(when emit-alias name)))
-(mu/defmethod parse-tag :snippet :- :metabase.lib.parameters.parse.types/referenced-query-snippet
- [{:keys [snippet-name snippet-id], :as tag} :- ::mbql.s/TemplateTag
+(mu/defmethod parse-tag :snippet :- ::params.types/referenced-query-snippet
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ {:keys [snippet-name snippet-id], :as tag} :- ::lib.schema.template-tag/template-tag
_params]
(let [snippet-id (or snippet-id
(throw (ex-info (tru "Unable to resolve Snippet: missing `:snippet-id`")
{:tag tag, :type qp.error-type/invalid-parameter})))
- snippet (or (lib.metadata/native-query-snippet (qp.store/metadata-provider) snippet-id)
+ snippet (or (lib.metadata/native-query-snippet metadata-providerable snippet-id)
(throw (ex-info (tru "Snippet {0} {1} not found." snippet-id (pr-str snippet-name))
{:snippet-id snippet-id
:snippet-name snippet-name
@@ -281,8 +277,10 @@
:type qp.error-type/invalid-parameter})))]
(lib/parsed-referenced-query-snippet-param (:id snippet) (:content snippet))))
-(defmethod parse-tag :temporal-unit
- [{:keys [required dimension alias] :as tag} params]
+(mu/defmethod parse-tag :temporal-unit :- ::params.types/temporal-unit
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ {:keys [required dimension alias] :as tag} :- ::lib.schema.template-tag/template-tag
+ params :- [:maybe [:sequential ::lib.schema.parameter/parameter]]]
(let [matching-param (when-let [matching-params (not-empty (tag-params tag params))]
(when (> (count matching-params) 1)
(throw (ex-info (tru "Error: multiple values specified for parameter; non-Field Filter parameters can only have one value.")
@@ -301,8 +299,8 @@
(throw (ex-info (tru "Error: invalid value specified for temporal-unit parameter.")
{:value param-value
:expected valid-temporal-units})))
- (let [field (let [field-id (dimension->field-id dimension)]
- (or (lib.metadata/field (qp.store/metadata-provider) field-id)
+ (let [field (let [field-id (field-ref->field-id dimension)]
+ (or (lib.metadata/field metadata-providerable field-id)
(throw (ex-info (tru "Can''t find field with ID: {0}" field-id)
{:field-id field-id, :type qp.error-type/invalid-parameter}))))
value (or (:value matching-param)
@@ -318,7 +316,7 @@
(mu/defn- param-value-for-raw-value-tag
"Get the value that should be used for a raw value (i.e., non-Field filter) template tag from `params`."
- [tag :- ::mbql.s/TemplateTag
+ [tag :- ::lib.schema.template-tag/template-tag
params :- [:maybe [:sequential ::lib.schema.parameter/parameter]]]
(let [matching-params (tag-params tag params)
real-values (into #{} (keep :value) matching-params)
@@ -346,21 +344,10 @@
;; Fall back to no value.
:else lib/parsed-param-no-value-placeholder)))
-(defmethod parse-tag :number
- [tag params]
- (param-value-for-raw-value-tag tag params))
-
-(defmethod parse-tag :text
- [tag params]
- (param-value-for-raw-value-tag tag params))
-
-(defmethod parse-tag :date
- [tag params]
- (param-value-for-raw-value-tag tag params))
-
-(defmethod parse-tag :boolean
- [tag params]
- (param-value-for-raw-value-tag tag params))
+(doseq [tag-type [:number :text :date :boolean]]
+ (defmethod parse-tag tag-type
+ [_metadata-providerable tag params]
+ (param-value-for-raw-value-tag tag params)))
;;; Parsing Values
@@ -411,7 +398,7 @@
(mu/defn- update-filter-for-field-type :- ::parsed-param-value
"Update a Field Filter with a textual, or sequence of textual, values. The base type and semantic type of the field
are used to determine what 'semantic' type interpretation is required (e.g. for UUID fields)."
- [{field :field, {value :value} :value, :as field-filter} :- :metabase.lib.parameters.parse.types/field-filter]
+ [{field :field, {value :value} :value, :as field-filter} :- ::params.types/field-filter]
(let [effective-type ((some-fn :effective-type :base-type) field)
new-value (cond
(string? value)
@@ -462,10 +449,11 @@
(mu/defn- value-for-tag :- ::parsed-param-value
"Given a map `tag` (a value in the `:template-tags` dictionary) return the corresponding value from the `params`
sequence. The `value` is something that can be compiled to SQL via `->replacement-snippet-info`."
- [tag :- ::mbql.s/TemplateTag
- params :- [:maybe [:sequential ::lib.schema.parameter/parameter]]]
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ tag :- ::lib.schema.template-tag/template-tag
+ params :- [:maybe [:sequential ::lib.schema.parameter/parameter]]]
(try
- (parse-value-for-type (:type tag) (parse-tag tag params))
+ (parse-value-for-type (:type tag) (parse-tag metadata-providerable tag params))
(catch Throwable e
(throw (ex-info (tru "Error determining value for parameter {0}: {1}"
(pr-str (:name tag))
@@ -475,21 +463,22 @@
:params params}
e)))))
-(mu/defn query->params-map :- [:map-of ::lib.schema.common/non-blank-string ::parsed-param-value]
+(mu/defn stage->params-map :- [:map-of ::lib.schema.common/non-blank-string ::parsed-param-value]
"Extract parameters info from `query`. Return a map of parameter name -> value.
- (query->params-map some-inner-query)
+ (stage->params-map metadata-provider stage)
->
- {:checkin_date #t \"2019-09-19T23:30:42.233-07:00\"}"
- [{tags :template-tags, params :parameters, :as _inner-query} :- [:and
- [:map]
- [:fn
- {:error/message "should be a legacy inner query"}
- (complement :lib/type)]]]
+ {:checkin_date #t \"2019-09-19T23:30:42.233-07:00\"}
+
+ Note that this takes a `metadata-providerable` plus `stage` rather than the more typical `query` plus `stage-number`
+ because it ultimately called for all stages of a query (even ones inside joins) by
+ [[metabase.query-processor.middleware.parameters/expand-stage]]. "
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ {tags :template-tags, params :parameters, :as _stage} :- ::lib.schema/stage]
(log/tracef "Building params map out of tags\n%s\nand params\n%s\n" (u/pprint-to-str tags) (u/pprint-to-str params))
(try
(into {} (for [[k tag] tags
- :let [v (value-for-tag tag params)]]
+ :let [v (value-for-tag metadata-providerable tag params)]]
(do
(log/tracef "Value for tag %s\n%s\n->\n%s" (pr-str k) (u/pprint-to-str tag) (u/pprint-to-str v))
[(or (lib/match-and-normalize-tag-name k) k) v])))
diff --git a/src/metabase/query_processor/timezone.clj b/src/metabase/query_processor/timezone.clj
index b7964959c1b..6a23c7a00aa 100644
--- a/src/metabase/query_processor/timezone.clj
+++ b/src/metabase/query_processor/timezone.clj
@@ -10,6 +10,7 @@
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.util.malli.schema :as ms])
(:import
(java.time ZonedDateTime)))
@@ -41,6 +42,12 @@
;;; | Public Interface |
;;; +----------------------------------------------------------------------------------------------------------------+
+(mr/def ::database
+ [:or
+ ::lib.schema.metadata/database
+ (ms/InstanceOf :model/Database)
+ [:= ::db-from-store]])
+
(mu/defn report-timezone-id-if-supported
"Timezone ID for the report timezone, if the current driver and database supports it. (If the current driver supports it, this is
bound by the `bind-effective-timezone` middleware.)"
@@ -48,10 +55,7 @@
(report-timezone-id-if-supported driver/*driver* (lib.metadata/database (qp.store/metadata-provider))))
(^String [driver :- :keyword
- database :- [:or
- ::lib.schema.metadata/database
- (ms/InstanceOf :model/Database)
- [:= ::db-from-store]]]
+ database :- ::database]
(when-let [database (if (= database ::db-from-store)
(when (qp.store/initialized?)
(lib.metadata/database (qp.store/metadata-provider)))
@@ -85,7 +89,7 @@
^String []
(valid-timezone-id (report-timezone-id*)))
-(defn results-timezone-id
+(mu/defn results-timezone-id :- :string
"The timezone that a query is actually ran in ­ report timezone, if set and supported by the current driver;
otherwise the timezone of the database (if known), otherwise the system timezone. Guaranteed to always return a
timezone ID,­ never returns `nil`."
@@ -95,8 +99,10 @@
(^String [database]
(results-timezone-id (:engine database) database))
- (^String [driver database & {:keys [use-report-timezone-id-if-unsupported?]
- :or {use-report-timezone-id-if-unsupported? false}}]
+ (^String [driver :- :keyword
+ database :- ::database
+ & {:keys [use-report-timezone-id-if-unsupported?]
+ :or {use-report-timezone-id-if-unsupported? false}}]
(valid-timezone-id
(or *results-timezone-id-override*
(if use-report-timezone-id-if-unsupported?
diff --git a/test/metabase/app_db/custom_migrations/metrics_v2_test.clj b/test/metabase/app_db/custom_migrations/metrics_v2_test.clj
index 1450820ffb1..5566ffc0f85 100644
--- a/test/metabase/app_db/custom_migrations/metrics_v2_test.clj
+++ b/test/metabase/app_db/custom_migrations/metrics_v2_test.clj
@@ -148,10 +148,10 @@
json/decode+kw)))))
(def query-validator
- (mr/validator ::mbql.s/MBQLQuery))
+ (mr/validator ::mbql.s/MBQLInnerQuery))
(def query-explainer
- (mr/explainer ::mbql.s/MBQLQuery))
+ (mr/explainer ::mbql.s/MBQLInnerQuery))
(deftest migrate-metrics-to-v2-test
(impl/test-migrations ["v51.2024-05-13T15:30:57" "v51.2024-05-13T16:00:00"] [migrate!]
diff --git a/test/metabase/driver/sql/parameters/substitute_test.clj b/test/metabase/driver/sql/parameters/substitute_test.clj
index 4193b3ed26e..258b9b01527 100644
--- a/test/metabase/driver/sql/parameters/substitute_test.clj
+++ b/test/metabase/driver/sql/parameters/substitute_test.clj
@@ -25,8 +25,7 @@
(defn- substitute [parsed param->value]
(driver/with-driver :h2
- (mt/with-metadata-provider meta/metadata-provider
- (sql.params.substitute/substitute parsed param->value))))
+ (sql.params.substitute/substitute meta/metadata-provider parsed param->value)))
;; as with the MBQL parameters tests Redshift fail for unknown reasons; disable their tests for now
;; TIMEZONE FIXME
@@ -162,8 +161,7 @@
field-metadata
{:type :string/=
:value ["banana"]})
- [sql _args] (mt/with-metadata-provider metadata-provider
- (sql.params.substitute/substitute query {"tag" field-filter}))]
+ [sql _args] (sql.params.substitute/substitute metadata-provider query {"tag" field-filter})]
(testing "The SQL identifier should include the parent field 'result' before 'tag_name'"
(let [[exp-identifier] (sql.qp/format-honeysql driver/*driver*
(h2x/identifier :field "result" "tag_name"))]
@@ -390,8 +388,7 @@
(defn- substitute-e2e [sql params]
(let [[query params] (driver/with-driver :h2
- (mt/with-metadata-provider meta/metadata-provider
- (#'sql.params.substitute/substitute (lib/parse-parameters sql) (into {} params))))]
+ (#'sql.params.substitute/substitute meta/metadata-provider (lib/parse-parameters sql) (into {} params)))]
{:query query, :params (vec params)}))
(deftest ^:parallel basic-substitution-test
@@ -978,7 +975,7 @@
(deftest ^:parallel e2e-relative-dates-test
(mt/test-drivers (sql-parameters-engines)
(testing (str "test that relative dates work correctly. It should be enough to try just one type of relative date "
- "here, since handling them gets delegated to the functions in `metabase.driver.common.parameters.dates`, "
+ "here, since handling them gets delegated to the functions in `metabase.query-processor.parameters.dates`, "
"which is fully-tested :D")
(is (= [0]
(mt/first-row
@@ -1008,7 +1005,7 @@
(deftest ^:parallel e2e-exclude-date-parts-test
(mt/test-drivers (mt/normal-drivers-with-feature ::e2e-exclude-date-parts-test)
(testing (str "test that excluding date parts work correctly. It should be enough to try just one type of exclusion "
- "here, since handling them gets delegated to the functions in `metabase.driver.common.parameters.dates`, "
+ "here, since handling them gets delegated to the functions in `metabase.query-processor.parameters.dates`, "
"which is fully-tested :D")
(doseq [[exclusion-string expected] {"exclude-months-Jan" 14
"exclude-months-Jan-Feb" 13
@@ -1284,6 +1281,7 @@
(is (= ["SELECT * FROM (SELECT * FROM table WHERE x LIKE ?)"
["G%"]]
(sql.params.substitute/substitute
+ meta/metadata-provider
["SELECT * FROM " (lib/parsed-param "#1")]
{"#1"
(lib/parsed-referenced-card-query-param 1 "SELECT * FROM table WHERE x LIKE ?" ["G%"])}))))))
diff --git a/test/metabase/legacy_mbql/normalize_test.cljc b/test/metabase/legacy_mbql/normalize_test.cljc
index be8e26e1e21..77ede2f7c7f 100644
--- a/test/metabase/legacy_mbql/normalize_test.cljc
+++ b/test/metabase/legacy_mbql/normalize_test.cljc
@@ -41,28 +41,28 @@
(deftest ^:parallel normalize-test-3
(testing "METRICS shouldn't get normalized in some kind of wacky way"
(is (= {:aggregation [[:+ [:metric 10] 1]]}
- (mbql.normalize/normalize ::mbql.s/MBQLQuery {:aggregation ["+" ["METRIC" 10] 1]})))))
+ (mbql.normalize/normalize ::mbql.s/MBQLInnerQuery {:aggregation ["+" ["METRIC" 10] 1]})))))
;; TODO (Tamas 2026-01-05): Remove this test once FE tests switch to using MBQL5
(deftest ^:parallel normalize-measure-test
(testing "MEASURES shouldn't get normalized in some kind of wacky way"
(is (= {:aggregation [[:+ [:measure 10] 1]]}
- (mbql.normalize/normalize ::mbql.s/MBQLQuery {:aggregation ["+" ["MEASURE" 10] 1]})))))
+ (mbql.normalize/normalize ::mbql.s/MBQLInnerQuery {:aggregation ["+" ["MEASURE" 10] 1]})))))
(deftest ^:parallel normalize-test-4
(testing "Nor should SEGMENTS"
(is (= {:filter [:= [:+ [:segment 10] 1] 10]}
- (mbql.normalize/normalize ::mbql.s/MBQLQuery {:filter ["=" ["+" ["SEGMENT" 10] 1] 10]})))))
+ (mbql.normalize/normalize ::mbql.s/MBQLInnerQuery {:filter ["=" ["+" ["SEGMENT" 10] 1] 10]})))))
(deftest ^:parallel normalize-test-5
(testing "field literals should be exempt too"
(is (= {:order-by [[:desc [:field "SALES_TAX" {:base-type :type/Number}]]]}
- (mbql.normalize/normalize ::mbql.s/MBQLQuery {:order-by [[:desc [:field-literal "SALES_TAX" :type/Number]]]})))))
+ (mbql.normalize/normalize ::mbql.s/MBQLInnerQuery {:order-by [[:desc [:field-literal "SALES_TAX" :type/Number]]]})))))
(deftest ^:parallel normalize-test-6
(testing "... but they should be converted to strings if passed in as a KW for some reason"
(is (= {:order-by [[:desc [:field "SALES/TAX" {:base-type :type/Number}]]]}
- (mbql.normalize/normalize ::mbql.s/MBQLQuery {:order-by [[:desc ["field_literal" :SALES/TAX "type/Number"]]]})))))
+ (mbql.normalize/normalize ::mbql.s/MBQLInnerQuery {:order-by [[:desc ["field_literal" :SALES/TAX "type/Number"]]]})))))
(deftest ^:parallel normalize-test-7
(testing "modern :field clauses should get normalized"
diff --git a/test/metabase/legacy_mbql/schema_test.cljc b/test/metabase/legacy_mbql/schema_test.cljc
index d94bcfa408b..20b1ce65101 100644
--- a/test/metabase/legacy_mbql/schema_test.cljc
+++ b/test/metabase/legacy_mbql/schema_test.cljc
@@ -162,7 +162,7 @@
[:or ::mbql.s/absolute-datetime ::mbql.s/value])))
(deftest ^:parallel expression-value-wrapped-literals-test
- (are [value] (not (me/humanize (mr/explain ::mbql.s/MBQLQuery
+ (are [value] (not (me/humanize (mr/explain ::mbql.s/MBQLInnerQuery
{:source-table 1, :expressions {"expr" [:value value nil]}})))
""
"192.168.1.1"
@@ -176,7 +176,7 @@
;; Allowed in #67203 and above
(deftest ^:parallel expression-unwrapped-literals-test
- (are [value] (mr/validate ::mbql.s/MBQLQuery {:source-table 1, :expressions {"expr" value}})
+ (are [value] (mr/validate ::mbql.s/MBQLInnerQuery {:source-table 1, :expressions {"expr" value}})
""
"192.168.1.1"
"2025-03-11"
diff --git a/test/metabase/lib/convert_test.cljc b/test/metabase/lib/convert_test.cljc
index 4d65b206899..d86d54d6d5f 100644
--- a/test/metabase/lib/convert_test.cljc
+++ b/test/metabase/lib/convert_test.cljc
@@ -1662,3 +1662,17 @@
["case"
[[["<" ["aggregation" 0 {"base-type" "type/Float"}] 0.591] "60%"]]]
{"name" "A", "display-name" "B"}]))))
+
+(deftest ^:parallel legacy-query-from-inner-query-test
+ (is (=? {:database 1
+ :native {:query "SELECT * FROM table WHERE {{checkin_date}};"
+ :template-tags {}}
+ :type :native}
+ (lib.convert/legacy-query-from-inner-query
+ 1
+ {:native "SELECT * FROM table WHERE {{checkin_date}};"
+ :template-tags {"checkin_date" {:name "checkin_date"
+ :display-name "Checkin Date"
+ :type :dimension
+ :widget-type :date/all-options
+ :dimension [:field 2 nil]}}}))))
diff --git a/test/metabase/lib/native_test.cljc b/test/metabase/lib/native_test.cljc
index 247ea92fa36..b8e46038581 100644
--- a/test/metabase/lib/native_test.cljc
+++ b/test/metabase/lib/native_test.cljc
@@ -174,8 +174,8 @@
(let [query (lib/native-query meta/metadata-provider "select * from venues where id = {{myid}}")]
(testing "Updating query keeps template tags in sync"
(is (=? ["select * from venues where id = {{myid}}"
- {"myid" {:type :text,
- :name "myid",
+ {"myid" {:type :text
+ :name "myid"
:id string?
:display-name "Myid"}}]
((juxt lib/raw-native-query lib/template-tags) query)))
@@ -467,22 +467,22 @@
(lib/returned-columns query)))))
(deftest ^:parallel add-parameters-text-tag-test
- (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7",
- :type :string/=,
- :value ["foo"],
+ (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7"
+ :type :string/=
+ :value ["foo"]
:target ["variable" ["template-tag" "mytag"]]}]
(-> (lib/native-query meta/metadata-provider "select * from venues where {{mytag}}")
- (lib/with-template-tags {"mytag" {:type :text,
- :name "mytag",
+ (lib/with-template-tags {"mytag" {:type :text
+ :name "mytag"
:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7"
:display-name "My Tag"}})
lib/add-parameters-for-template-tags
:parameters))))
(deftest ^:parallel add-parameters-number-tag-test
- (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7",
- :type :number/=,
- :value ["0"],
+ (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7"
+ :type :number/=
+ :value ["0"]
:target ["variable" ["template-tag" "mytag"]]}]
(-> (lib/native-query meta/metadata-provider "select * from venues where {{mytag}}")
(lib/with-template-tags {"mytag"
@@ -494,9 +494,9 @@
:parameters))))
(deftest ^:parallel add-parameters-date-tag-test
- (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7",
- :type :date/single,
- :value "1970-01-01",
+ (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7"
+ :type :date/single
+ :value "1970-01-01"
:target ["variable" ["template-tag" "mytag"]]}]
(-> (lib/native-query meta/metadata-provider "select * from venues where {{mytag}}")
(lib/with-template-tags {"mytag"
@@ -508,9 +508,9 @@
:parameters))))
(deftest ^:parallel add-parameters-boolean-tag-test
- (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7",
- :type :boolean/=,
- :value [false],
+ (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7"
+ :type :boolean/=
+ :value [false]
:target ["variable" ["template-tag" "mytag"]]}]
(-> (lib/native-query meta/metadata-provider "select * from venues where {{mytag}}")
(lib/with-template-tags {"mytag"
@@ -522,9 +522,9 @@
:parameters))))
(deftest ^:parallel add-parameters-string-dimension-tag-test
- (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7",
- :type :string/=,
- :value ["foo"],
+ (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7"
+ :type :string/=
+ :value ["foo"]
:target ["dimension" ["template-tag" "mytag"]]}]
(-> (lib/native-query meta/metadata-provider "select * from venues where {{mytag}}")
(lib/with-template-tags {"mytag"
@@ -540,9 +540,9 @@
:parameters))))
(deftest ^:parallel add-parameters-number-dimension-tag-test
- (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7",
- :type :number/=,
- :value ["0"],
+ (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7"
+ :type :number/=
+ :value ["0"]
:target ["dimension" ["template-tag" "mytag"]]}]
(-> (lib/native-query meta/metadata-provider "select * from venues where {{mytag}}")
(lib/with-template-tags {"mytag"
@@ -558,9 +558,9 @@
:parameters))))
(deftest ^:parallel add-parameters-date-dimension-tag-test
- (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7",
- :type :date/single,
- :value "2025-01-01",
+ (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7"
+ :type :date/single
+ :value "2025-01-01"
:target ["dimension" ["template-tag" "mytag"]]}]
(-> (lib/native-query meta/metadata-provider "select * from venues where {{mytag}}")
(lib/with-template-tags {"mytag"
@@ -576,9 +576,9 @@
:parameters))))
(deftest ^:parallel add-parameters-temporal-unit-tag-test
- (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7",
- :type :temporal-unit,
- :value "week",
+ (is (= [{:id "9ae1ea5e-ac33-4574-bc95-ff595b0ac1a7"
+ :type :temporal-unit
+ :value "week"
:target ["dimension" ["template-tag" "mytag"]]}]
(-> (lib/native-query meta/metadata-provider "select * from venues where {{mytag}}")
(lib/with-template-tags {"mytag"
@@ -652,6 +652,27 @@
(lib.native/with-template-tags base-query
{"table" (merge template-tag {:type :table} template-tag-overrides)})))
+(deftest ^:parallel with-template-tags-normalize-template-tag-names-test
+ (testing "`with-template-tags` should normalize tag names"
+ ;; `snippet:expensive_venues` should be normalized to `snippet: expensive_venues`
+ (let [mp (lib.tu/mock-metadata-provider
+ meta/metadata-provider
+ {:native-query-snippets [{:id 1
+ :name "expensive_venues"
+ :content "venues WHERE price = 4"}]})]
+ (is (=? {:stages [{:native "SELECT * FROM {{snippet:expensive_venues}}"
+ :template-tags {"snippet: expensive_venues" {:name "snippet: expensive_venues"
+ :snippet-id 1
+ :snippet-name "expensive_venues"
+ :type :snippet
+ :display-name "(New display name)"}}}]}
+ (-> (lib/native-query mp "SELECT * FROM {{snippet:expensive_venues}}")
+ (lib/with-template-tags {"snippet:expensive_venues" {:type :snippet
+ :name "snippet:expensive_venues"
+ :display-name "(New display name)"
+ :snippet-name "expensive_venues"
+ :snippet-id 1}})))))))
+
(deftest ^:parallel basic-native-query-table-references-test
(testing "should find id-based native query table references"
(is (= #{{:table (meta/id :orders)}}
diff --git a/test/metabase/lib/query_test.cljc b/test/metabase/lib/query_test.cljc
index c2e099a7ba6..1617a728877 100644
--- a/test/metabase/lib/query_test.cljc
+++ b/test/metabase/lib/query_test.cljc
@@ -604,9 +604,30 @@
"query" {"source-table" 2
"expressions" {"booking" ["sum" ["field" 3 {"base-type" "type/BigInteger"}]]}}}
mp (lib.tu/mock-metadata-provider {})]
- (is (= {:lib/type :mbql/query,
+ (is (= {:lib/type :mbql/query
:stages [{:lib/type :mbql.stage/mbql, :source-table 2}]
:database 1
:lib.convert/converted? true
:lib/metadata (lib.metadata.cached-provider/cached-metadata-provider mp)}
(lib.query/query mp query)))))))
+
+(deftest ^:parallel query-from-legacy-inner-query-test
+ (is (=? {:lib/type :mbql/query
+ :database 1
+ :stages [{:lib/type :mbql.stage/native
+ :native "SELECT * FROM table WHERE {{checkin_date}};"
+ :template-tags {"checkin_date"
+ {:dimension [:field {} 2]
+ :display-name "Checkin Date"
+ :name "checkin_date"
+ :type :dimension
+ :widget-type :date/all-options}}}]}
+ (lib.query/query-from-legacy-inner-query
+ meta/metadata-provider
+ 1
+ {:native "SELECT * FROM table WHERE {{checkin_date}};"
+ :template-tags {"checkin_date" {:name "checkin_date"
+ :display-name "Checkin Date"
+ :type :dimension
+ :widget-type :date/all-options
+ :dimension [:field 2 nil]}}}))))
diff --git a/test/metabase/lib/test_util/macros.clj b/test/metabase/lib/test_util/macros.clj
index 53e92493d33..1b7fdfe86c3 100644
--- a/test/metabase/lib/test_util/macros.clj
+++ b/test/metabase/lib/test_util/macros.clj
@@ -35,6 +35,7 @@
([table-name inner-query]
{:pre [(map? inner-query)]}
+ (assert (not (:type inner-query)) "Should be an inner-query, not an outer query")
(do-with-bindings
#(as-> inner-query <>
(mbql-query-impl/parse-tokens table-name <>)
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index 455866a1fcd..60589953301 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -11,12 +11,15 @@
[metabase.metabot.self.openrouter :as openrouter]
[metabase.metabot.test-util :as test-util]
[metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
[metabase.util.json :as json]
[metabase.util.log.capture :as log.capture]
[ring.adapter.jetty :as jetty]))
(set! *warn-on-reflection* true)
+(use-fixtures :once (fixtures/initialize :db))
+
;;; provider resolution tests
(deftest ^:parallel parse-provider-model-test
diff --git a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj b/test/metabase/query_processor/middleware/wrap_value_literals_test.clj
index f684fada457..d254ad4ea13 100644
--- a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj
+++ b/test/metabase/query_processor/middleware/wrap_value_literals_test.clj
@@ -26,12 +26,13 @@
(letfn [(f [query]
(mt/with-results-timezone-id timezone-id
(qp.wrap-value-literals/wrap-value-literals query)))]
- (if (:lib/type query)
- (f query)
- (let [mp meta/metadata-provider]
- (-> (lib/query mp query)
- f
- lib/->legacy-MBQL))))))
+ (driver/with-driver ::tz-driver
+ (if (:lib/type query)
+ (f query)
+ (let [mp meta/metadata-provider]
+ (-> (lib/query mp query)
+ f
+ lib/->legacy-MBQL)))))))
(deftest ^:parallel wrap-integers-test
(is (= (lib.tu.macros/mbql-query venues
@@ -240,24 +241,27 @@
(deftest ^:parallel base-type-test
(testing "Make sure base-type from `:field` w/ name is picked up correctly"
- (is (= [:not [:starts-with
- [:field "A" {:base-type :type/Text}]
- [:value "f" {:base_type :type/Text}]]]
- #_{:clj-kondo/ignore [:deprecated-var]}
- (qp.wrap-value-literals/wrap-value-literals-in-mbql
- [:not [:starts-with [:field "A" {:base-type :type/Text}] "f"]])))))
+ (is (=? [:not {}
+ [:starts-with {}
+ [:field {:base-type :type/Text} "A"]
+ [:value {:base-type :type/Text} "f"]]]
+ (qp.wrap-value-literals/wrap-value-literals-in-mbql5
+ (lib/not (lib/starts-with (-> (meta/field-metadata :venues :name)
+ (assoc :name "A")
+ (dissoc :id))
+ "f")))))))
(deftest ^:parallel parse-temporal-string-literals-based-on-column-effective-type-test
(testing "Temporal string literals should be parsed to different things based on the effective type of the target column (#39769)"
(driver/with-driver :h2
;; I'm wrapping the `#t` literals in `(t/...)` functions below to make the types we expect SUPER EXPLICIT.
(doseq [[column-type expected]
- {:type/Date [:absolute-datetime (t/local-date #t "2024-03-20") :default]
- :type/DateTime [:absolute-datetime (t/local-date-time #t "2024-03-20T15:24:00") :default]
- :type/DateTimeWithTZ [:absolute-datetime (t/offset-date-time #t "2024-03-20T15:24:00-07:00") :default]
- :type/DateTimeWithZoneID [:absolute-datetime (t/zoned-date-time #t "2024-03-20T15:24:00-07:00[US/Pacific]") :default]
- :type/Time [:time (t/local-time #t "15:24:00") :default]
- :type/TimeWithTZ [:time (t/offset-time #t "15:24:00-07:00") :default]}]
+ {:type/Date [:absolute-datetime {} (t/local-date #t "2024-03-20") :default]
+ :type/DateTime [:absolute-datetime {} (t/local-date-time #t "2024-03-20T15:24:00") :default]
+ :type/DateTimeWithTZ [:absolute-datetime {} (t/offset-date-time #t "2024-03-20T15:24:00-07:00") :default]
+ :type/DateTimeWithZoneID [:absolute-datetime {} (t/zoned-date-time #t "2024-03-20T15:24:00-07:00[US/Pacific]") :default]
+ :type/Time [:time {} (t/local-time #t "15:24:00") :default]
+ :type/TimeWithTZ [:time {} (t/offset-time #t "15:24:00-07:00") :default]}]
(testing column-type
(qp.store/with-metadata-provider (lib.tu/merged-mock-metadata-provider
meta/metadata-provider
@@ -266,14 +270,14 @@
:base-type :type/Text
:coercion-strategy :Coercion/UNIXSeconds->DateTime
:effective-type column-type}]})
- (is (= [:=
- [:field (meta/id :checkins :date) {:base-type :type/Text, :effective-type column-type}]
- expected]
- #_{:clj-kondo/ignore [:deprecated-var]}
- (qp.wrap-value-literals/wrap-value-literals-in-mbql
- [:=
- [:field (meta/id :checkins :date) {:base-type :type/Text, :effective-type column-type}]
- "2024-03-20T15:24:00-07:00[US/Pacific]"])))))))))
+ (is (=? [:=
+ {}
+ [:field {:base-type :type/Text, :effective-type column-type} (meta/id :checkins :date)]
+ expected]
+ (qp.wrap-value-literals/wrap-value-literals-in-mbql5
+ (lib/= (-> (meta/field-metadata :checkins :date)
+ (assoc :base-type :type/Text, :effective-type column-type))
+ "2024-03-20T15:24:00-07:00[US/Pacific]"))))))))))
(deftest ^:parallel expression-test
(testing "Value literals compared to :expression refs should get wrapped. Should give date literal strings :day bucketing (#17807)"
@@ -368,11 +372,13 @@
wrap-value-literals)))))
(deftest ^:parallel wrap-value-literals-in-mbql-test
- (is (= [:!=
- [:field 1 {:base-type :type/Date, :temporal-unit :month-of-year}]
- [:absolute-datetime #t "2016-01-01" :month-of-year]]
- #_{:clj-kondo/ignore [:deprecated-var]}
- (#'qp.wrap-value-literals/wrap-value-literals-in-mbql
- [:!=
- [:field 1 {:base-type :type/Date, :temporal-unit :month-of-year}]
- "2016-01-01"]))))
+ (driver/with-driver :h2
+ (is (=? [:!=
+ {}
+ [:field {:base-type :type/Date, :temporal-unit :month} pos-int?]
+ [:absolute-datetime {} #t "2016-01-01" :month]]
+ (#'qp.wrap-value-literals/wrap-value-literals-in-mbql5
+ (lib/!= (-> (meta/field-metadata :checkins :date)
+ (assoc :base-type :type/Date, :effective-type :type/Date)
+ (lib/with-temporal-bucket :month))
+ "2016-01-01"))))))
diff --git a/test/metabase/driver/common/parameters/values_test.clj b/test/metabase/query_processor/parameters/values_test.clj
similarity index 51%
rename from test/metabase/driver/common/parameters/values_test.clj
rename to test/metabase/query_processor/parameters/values_test.clj
index 8ce0ef74953..538409c8e13 100644
--- a/test/metabase/driver/common/parameters/values_test.clj
+++ b/test/metabase/query_processor/parameters/values_test.clj
@@ -1,15 +1,16 @@
-(ns metabase.driver.common.parameters.values-test
- {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.driver.common.parameters.values-test]}}}}}}
+(ns metabase.query-processor.parameters.values-test
(:require
[clojure.java.jdbc :as jdbc]
[clojure.string :as str]
[clojure.test :refer :all]
[metabase.driver :as driver]
- ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.values :as params.values]
[metabase.driver.ddl.interface :as ddl.i]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+ [metabase.legacy-mbql.schema :as mbql.s]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
+ [metabase.lib.schema.template-tag :as lib.schema.template-tag]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.macros :as lib.tu.macros]
@@ -17,11 +18,12 @@
[metabase.permissions.models.permissions :as perms]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.query-processor.middleware.permissions :as qp.perms]
- ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
+ [metabase.query-processor.parameters.values :as params.values]
[metabase.query-processor.test :as qp]
[metabase.system.core :as system]
[metabase.test :as mt]
[metabase.util :as u]
+ [metabase.util.malli :as mu]
[toucan2.core :as t2])
(:import
(clojure.lang ExceptionInfo)))
@@ -30,54 +32,83 @@
(def ^:private test-uuid (str (random-uuid)))
+(defn- value-for-tag
+ "Call the private function and de-recordize the field"
+ ([tag params]
+ (value-for-tag meta/metadata-provider tag params))
+ ([metadata-providerable tag params]
+ (#'params.values/value-for-tag
+ metadata-providerable
+ (lib/normalize ::lib.schema.template-tag/template-tag tag)
+ params)))
+
(deftest ^:parallel variable-value-test
(testing "Specified value, targeted by name"
(is (= "2"
- (#'params.values/value-for-tag
+ (value-for-tag
{:name "id", :display-name "ID", :type :text, :required true, :default "100"}
- [{:type :category, :target [:variable [:template-tag "id"]], :value "2"}]))))
+ [{:type :category, :target [:variable [:template-tag "id"]], :value "2"}])))))
+
+(deftest ^:parallel variable-value-test-2
(testing "Specified value, targeted by ID"
(is (= "2"
- (#'params.values/value-for-tag
+ (value-for-tag
{:name "id", :id test-uuid, :display-name "ID", :type :text, :required true, :default "100"}
- [{:type :category, :target [:variable [:template-tag {:id test-uuid}]], :value "2"}]))))
+ [{:type :category, :target [:variable [:template-tag {:id test-uuid}]], :value "2"}])))))
+
+(deftest ^:parallel variable-value-test-3
(testing "Multiple values with new operators"
(is (= 20
- (#'params.values/value-for-tag
+ (value-for-tag
{:name "number_filter", :display-name "ID", :type :number, :required true, :default "100"}
[{:type :number/=, :value ["20"], :target [:variable [:template-tag "number_filter"]]}])))
(is (= [20 40]
- (#'params.values/value-for-tag
+ (value-for-tag
{:name "number_filter", :display-name "ID", :type :number, :required true, :default "100"}
- [{:type :number/=, :value ["20" "40"], :target [:variable [:template-tag "number_filter"]]}]))))
+ [{:type :number/=, :value ["20" "40"], :target [:variable [:template-tag "number_filter"]]}])))))
+
+(deftest ^:parallel variable-value-test-4
(testing "Unspecified value"
(is (= lib/parsed-param-no-value-placeholder
- (#'params.values/value-for-tag {:name "id", :display-name "ID", :type :text} nil))))
+ (value-for-tag {:name "id", :display-name "ID", :type :text} nil)))))
+
+(deftest ^:parallel variable-value-test-5
(testing "Unspecified value when required"
- (is (thrown? Exception
- (#'params.values/value-for-tag {:name "id", :display-name "ID", :required true, :type :text} nil))))
+ (is (thrown?
+ Exception
+ (value-for-tag {:name "id", :display-name "ID", :required true, :type :text} nil)))))
+
+(deftest ^:parallel variable-value-test-6
(testing "Empty value when required"
(is (thrown? Exception
- (#'params.values/value-for-tag
+ (value-for-tag
{:name "id", :id test-uuid, :display-name "ID", :required true, :type :text}
- [{:type :category, :target [:variable [:template-tag {:id test-uuid}]], :value nil}]))))
+ [{:type :category, :target [:variable [:template-tag {:id test-uuid}]], :value nil}])))))
+
+(deftest ^:parallel variable-value-test-7
(testing "Default used with unspecified value"
(is (= "100"
- (#'params.values/value-for-tag
- {:name "id", :display-name "ID", :type :text, :required true, :default "100"} nil))))
+ (value-for-tag
+ {:name "id", :display-name "ID", :type :text, :required true, :default "100"} nil)))))
+
+(deftest ^:parallel variable-value-test-8
(testing "Default not used with empty value"
(is (= lib/parsed-param-no-value-placeholder
- (#'params.values/value-for-tag
+ (value-for-tag
{:name "id", :id test-uuid, :display-name "ID", :type :text, :default "100"}
- [{:type :category, :target [:variable [:template-tag {:id test-uuid}]], :value nil}]))))
+ [{:type :category, :target [:variable [:template-tag {:id test-uuid}]], :value nil}])))))
+
+(deftest ^:parallel variable-value-test-9
(testing "Default used with empty value when required"
(is (= "100"
- (#'params.values/value-for-tag
+ (value-for-tag
{:name "id", :id test-uuid, :display-name "ID", :type :text, :required true, :default "100"}
- [{:type :category, :target [:variable [:template-tag {:id test-uuid}]], :value nil}]))))
+ [{:type :category, :target [:variable [:template-tag {:id test-uuid}]], :value nil}])))))
+
+(deftest ^:parallel variable-value-test-10
(testing "BigInteger value"
(is (= 9223372036854775808
- (#'params.values/value-for-tag
+ (value-for-tag
{:name "id", :id test-uuid, :display-name "ID", :type :number}
[{:type :category, :target [:variable [:template-tag {:id test-uuid}]], :value "9223372036854775808"}])))))
@@ -85,14 +116,14 @@
(testing "Allows multiple bindings of the same tag"
(testing "if only one has a value set"
(is (= "2"
- (#'params.values/value-for-tag
+ (value-for-tag
{:name "id", :display-name "ID", :type :text, :required true, :default "100"}
[{:type :category, :target [:variable [:template-tag "id"]], :value "2"}
{:type :category, :target [:variable [:template-tag "id"]], :value nil}
{:type :category, :target [:variable [:template-tag "id"]], :value nil}]))))
(testing "if all values are equal"
(is (= "2"
- (#'params.values/value-for-tag
+ (value-for-tag
{:name "id", :display-name "ID", :type :text, :required true, :default "100"}
[{:type :category, :target [:variable [:template-tag "id"]], :value "2"}
{:type :category, :target [:variable [:template-tag "id"]], :value "2"}
@@ -100,65 +131,46 @@
(testing "if no values are given"
(testing "required tags use their defaults"
(is (= "100"
- (#'params.values/value-for-tag
+ (value-for-tag
{:name "id", :display-name "ID", :type :text, :required true, :default "100"}
[{:type :category, :target [:variable [:template-tag "id"]], :value nil}
{:type :category, :target [:variable [:template-tag "id"]], :value nil}]))))
(testing "optional tags get no value"
(is (= lib/parsed-param-no-value-placeholder
- (#'params.values/value-for-tag
+ (value-for-tag
{:name "id", :display-name "ID", :type :text, :required false, :default "100"}
[{:type :category, :target [:variable [:template-tag "id"]], :value nil}
- {:type :category, :target [:variable [:template-tag "id"]], :value nil}]))))))
+ {:type :category, :target [:variable [:template-tag "id"]], :value nil}])))))))
+
+(deftest ^:parallel variable-multiple-values-test-2
(testing "Throws if multiple real values are set"
- (is (thrown-with-msg? Exception #"Multiple conflicting values"
- (#'params.values/value-for-tag
- {:name "id", :display-name "ID", :type :text, :required true, :default "100"}
- [{:type :category, :target [:variable [:template-tag "id"]], :value "2"}
- {:type :category, :target [:variable [:template-tag "id"]], :value "8"}])))))
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Multiple conflicting values"
+ (value-for-tag
+ {:name "id", :display-name "ID", :type :text, :required true, :default "100"}
+ [{:type :category, :target [:variable [:template-tag "id"]], :value "2"}
+ {:type :category, :target [:variable [:template-tag "id"]], :value "8"}])))))
-(defn- value-for-tag
- "Call the private function and de-recordize the field"
- [field-info info]
- (letfn [(thunk []
- (mt/derecordize (#'params.values/value-for-tag field-info info)))]
- (if (qp.store/initialized?)
- (thunk)
- (mt/with-metadata-provider (mt/id)
- (thunk)))))
-
-(defn- extra-field-info
- "Add extra field information like `:coercion-strategy`, `:semantic-type`, and `:effective-type`."
- [field]
- (letfn [(field-metadata* []
- (lib.metadata/field (qp.store/metadata-provider) (u/the-id field)))
- (field-metadata []
- (if (qp.store/initialized?)
- (field-metadata*)
- (mt/with-metadata-provider (mt/id)
- (field-metadata*))))]
- (merge (field-metadata) field)))
-
-(defn parse-tag
- [field-info info]
- (mt/derecordize (#'params.values/parse-tag field-info info)))
+(defn- parse-tag
+ [tag params]
+ (#'params.values/parse-tag meta/metadata-provider (lib/normalize ::lib.schema.template-tag/template-tag tag) params))
(deftest ^:parallel field-filter-date-range-targeted-by-name-test
(testing "date range for a normal :type/Temporal field, targeted by name"
- (is (=? {:field (extra-field-info
- {:id (mt/id :checkins :date)
- :name "DATE"
- :parent-id nil
- :table-id (mt/id :checkins)
- :base-type :type/Date
- :semantic-type nil})
+ (is (=? {:field {:id (meta/id :checkins :date)
+ :name "DATE"
+ :parent-id nil
+ :table-id (meta/id :checkins)
+ :base-type :type/Date
+ :semantic-type nil}
:value {:type :date/range
:value "2015-04-01~2015-05-01"}}
(value-for-tag
{:name "checkin_date"
:display-name "Checkin Date"
:type :dimension
- :dimension [:field (mt/id :checkins :date) nil]
+ :dimension [:field (meta/id :checkins :date) nil]
:widget-type :date/all-options}
[{:type :date/range
:target [:dimension [:template-tag "checkin_date"]]
@@ -166,13 +178,12 @@
(deftest ^:parallel field-filter-date-range-targeted-by-id-test
(testing "date range for a normal :type/Temporal field, targeted by id"
- (is (=? {:field (extra-field-info
- {:id (mt/id :checkins :date)
- :name "DATE"
- :parent-id nil
- :table-id (mt/id :checkins)
- :base-type :type/Date
- :semantic-type nil})
+ (is (=? {:field {:id (meta/id :checkins :date)
+ :name "DATE"
+ :parent-id nil
+ :table-id (meta/id :checkins)
+ :base-type :type/Date
+ :semantic-type nil}
:value {:type :date/range
:value "2015-04-01~2015-05-01"}}
(value-for-tag
@@ -180,7 +191,7 @@
:id test-uuid
:display-name "Checkin Date"
:type :dimension
- :dimension [:field (mt/id :checkins :date) nil]
+ :dimension [:field (meta/id :checkins :date) nil]
:widget-type :date/all-options}
[{:type :date/range
:target [:dimension [:template-tag {:id test-uuid}]]
@@ -188,55 +199,58 @@
(deftest ^:parallel field-filter-date-range-for-unix-timestamp-test
(testing "date range for a UNIX timestamp field should work just like a :type/Temporal field (#11934)"
- (mt/dataset tupac-sightings
- (mt/$ids sightings
- (is (=? {:field (extra-field-info
- {:id %timestamp
- :name "TIMESTAMP"
- :parent-id nil
- :table-id $$sightings
- :base-type :type/BigInteger
- :effective-type :type/Instant
- :coercion-strategy :Coercion/UNIXSeconds->DateTime})
- :value {:type :date/range
- :value "2020-02-01~2020-02-29"}}
- (value-for-tag
- {:name "timestamp"
- :display-name "Sighting Timestamp"
- :type :dimension
- :dimension $timestamp
- :widget-type :date/range}
- [{:type :date/range
- :target [:dimension [:template-tag "timestamp"]]
- :value "2020-02-01~2020-02-29"}])))))))
+ (let [mp (lib.tu/merged-mock-metadata-provider
+ meta/metadata-provider
+ {:fields [{:id (meta/id :checkins :date)
+ :name "TIMESTAMP"
+ :base-type :type/BigInteger
+ :effective-type :type/Instant
+ :coercion-strategy :Coercion/UNIXSeconds->DateTime}]})]
+ (is (=? {:field {:id (meta/id :checkins :date)
+ :name "TIMESTAMP"
+ :parent-id nil
+ :table-id (meta/id :checkins)
+ :base-type :type/BigInteger
+ :effective-type :type/Instant
+ :coercion-strategy :Coercion/UNIXSeconds->DateTime}
+ :value {:type :date/range
+ :value "2020-02-01~2020-02-29"}}
+ (value-for-tag
+ mp
+ {:name "timestamp"
+ :display-name "Sighting Timestamp"
+ :type :dimension
+ :dimension [:field (meta/id :checkins :date) nil]
+ :widget-type :date/range}
+ [{:type :date/range
+ :target [:dimension [:template-tag "timestamp"]]
+ :value "2020-02-01~2020-02-29"}]))))))
(deftest ^:parallel field-filter-with-unspecified-value-test
(testing "unspecified"
- (is (=? {:field (extra-field-info
- {:id (mt/id :checkins :date)
- :name "DATE"
- :parent-id nil
- :table-id (mt/id :checkins)
- :base-type :type/Date
- :semantic-type nil})
+ (is (=? {:field {:id (meta/id :checkins :date)
+ :name "DATE"
+ :parent-id nil
+ :table-id (meta/id :checkins)
+ :base-type :type/Date
+ :semantic-type nil}
:value lib/parsed-param-no-value-placeholder}
(value-for-tag
{:name "checkin_date"
:display-name "Checkin Date"
:type :dimension
:widget-type :date/all-options
- :dimension [:field (mt/id :checkins :date) nil]}
+ :dimension [:field (meta/id :checkins :date) nil]}
nil)))))
(deftest ^:parallel field-filter-id-requiring-parsing-test
(testing "id requiring parsing"
- (is (=? {:field (extra-field-info
- {:id (mt/id :checkins :id)
- :name "ID"
- :parent-id nil
- :table-id (mt/id :checkins)
- :base-type :type/BigInteger
- :semantic-type :type/PK})
+ (is (=? {:field {:id (meta/id :checkins :id)
+ :name "ID"
+ :parent-id nil
+ :table-id (meta/id :checkins)
+ :base-type :type/BigInteger
+ :semantic-type :type/PK}
:value {:type :id
:value 5}}
(value-for-tag
@@ -244,30 +258,31 @@
:display-name "ID"
:type :dimension
:widget-type :number
- :dimension [:field (mt/id :checkins :id) nil]}
+ :dimension [:field (meta/id :checkins :id) nil]}
[{:type :id, :target [:dimension [:template-tag "id"]], :value "5"}])))))
(deftest ^:parallel field-filter-with-required-but-no-value-test
(testing "required but unspecified"
- (is (thrown? Exception
- (value-for-tag
- {:name "checkin_date"
- :display-name "Checkin Date"
- :type :dimension
- :widget-type :date/all-options
- :required true
- :dimension [:field (mt/id :checkins :date) nil]}
- nil)))))
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"\QYou'll need to pick a value for 'Checkin Date' before this query can run.\E"
+ (value-for-tag
+ {:name "checkin_date"
+ :display-name "Checkin Date"
+ :type :dimension
+ :widget-type :date/all-options
+ :required true
+ :dimension [:field (meta/id :checkins :date) nil]}
+ nil)))))
(deftest ^:parallel field-filter-with-required-and-default-test
(testing "required and default specified"
- (is (=? {:field (extra-field-info
- {:id (mt/id :checkins :date)
- :name "DATE"
- :parent-id nil
- :table-id (mt/id :checkins)
- :base-type :type/Date
- :semantic-type nil})
+ (is (=? {:field {:id (meta/id :checkins :date)
+ :name "DATE"
+ :parent-id nil
+ :table-id (meta/id :checkins)
+ :base-type :type/Date
+ :semantic-type nil}
:value {:type :date/range
:value "2015-04-01~2015-05-01"}}
(value-for-tag
@@ -277,18 +292,17 @@
:widget-type :date/range
:required true
:default "2015-04-01~2015-05-01"
- :dimension [:field (mt/id :checkins :date) nil]}
+ :dimension [:field (meta/id :checkins :date) nil]}
nil)))))
(deftest ^:parallel field-filter-multiple-values-for-same-tag-test
(testing "multiple values for the same tag should return a vector with multiple params instead of a single param"
- (is (=? {:field (extra-field-info
- {:id (mt/id :checkins :date)
- :name "DATE"
- :parent-id nil
- :table-id (mt/id :checkins)
- :base-type :type/Date
- :semantic-type nil})
+ (is (=? {:field {:id (meta/id :checkins :date)
+ :name "DATE"
+ :parent-id nil
+ :table-id (meta/id :checkins)
+ :base-type :type/Date
+ :semantic-type nil}
:value [{:type :date/range
:value "2015-01-01~2016-09-01"}
{:type :date/single
@@ -298,7 +312,7 @@
:display-name "Checkin Date"
:type :dimension
:widget-type :date/all-options
- :dimension [:field (mt/id :checkins :date) nil]}
+ :dimension [:field (meta/id :checkins :date) nil]}
[{:type :date/range
:target [:dimension [:template-tag "checkin_date"]]
:value "2015-01-01~2016-09-01"}
@@ -309,13 +323,12 @@
(deftest ^:parallel field-filter-default-values-test
(mt/with-metadata-provider meta/metadata-provider
(testing "Make sure defaults values get picked up for field filter clauses"
- (is (=? {:field (extra-field-info
- {:id (meta/id :checkins :date)
- :name "DATE"
- :parent-id nil
- :table-id (meta/id :checkins)
- :base-type :type/Date
- :semantic-type nil})
+ (is (=? {:field {:id (meta/id :checkins :date)
+ :name "DATE"
+ :parent-id nil
+ :table-id (meta/id :checkins)
+ :base-type :type/Date
+ :semantic-type nil}
:value {:type :date/all-options
:value "past5days"}}
(parse-tag
@@ -330,13 +343,12 @@
(deftest ^:parallel field-filter-nil-values-test
(mt/with-metadata-provider meta/metadata-provider
(testing "Make sure nil values result in no value"
- (is (=? {:field (extra-field-info
- {:id (meta/id :checkins :date)
- :name "DATE"
- :parent-id nil
- :table-id (meta/id :checkins)
- :base-type :type/Date
- :effective-type :type/Date})
+ (is (=? {:field {:id (meta/id :checkins :date)
+ :name "DATE"
+ :parent-id nil
+ :table-id (meta/id :checkins)
+ :base-type :type/Date
+ :effective-type :type/Date}
:value lib/parsed-param-no-value-placeholder}
(parse-tag
{:name "checkin_date"
@@ -346,11 +358,13 @@
:widget-type :date/all-options}
nil))))))
-(defn- query->params-map [inner-query]
- (if (qp.store/initialized?)
- (params.values/query->params-map inner-query)
- (qp.store/with-metadata-provider (mt/id)
- (params.values/query->params-map inner-query))))
+(mu/defn- query->params-map
+ ([inner-query]
+ (query->params-map meta/metadata-provider inner-query))
+ ([metadata-provider :- ::lib.schema.metadata/metadata-provider
+ inner-query :- ::mbql.s/SourceQuery]
+ (let [query (lib/query-from-legacy-inner-query metadata-provider (:id (lib.metadata/database metadata-provider)) inner-query)]
+ (params.values/stage->params-map query (lib/query-stage query -1)))))
(deftest ^:parallel field-filter-errors-test
(testing "error conditions for field filter (:dimension) parameters"
@@ -367,19 +381,20 @@
(deftest ^:parallel card-query-test
(mt/with-test-user :rasta
(testing "Card query template tag gets card's native query"
- (let [test-query "SELECT 1"]
- (qp.store/with-metadata-provider (lib.tu/metadata-provider-with-cards-for-queries
- meta/metadata-provider
- [{:database (meta/id)
- :type "native"
- :native {:query test-query}}])
- (is (=? {:card-id 1, :query test-query, :parameters nil}
- (value-for-tag
- {:name "card-template-tag-test"
- :display-name "Card template tag test"
- :type :card
- :card-id 1}
- []))))))))
+ (let [test-query "SELECT 1"
+ mp (lib.tu/metadata-provider-with-cards-for-queries
+ meta/metadata-provider
+ [{:database (meta/id)
+ :type "native"
+ :native {:query test-query}}])]
+ (is (=? {:card-id 1, :query test-query, :parameters nil}
+ (value-for-tag
+ mp
+ {:name "card-template-tag-test"
+ :display-name "Card template tag test"
+ :type :card
+ :card-id 1}
+ [])))))))
(deftest ^:parallel card-query-test-2
(mt/with-test-user :rasta
@@ -396,127 +411,138 @@
"\"PUBLIC\".\"VENUES\".\"LONGITUDE\" AS \"LONGITUDE\", "
"\"PUBLIC\".\"VENUES\".\"PRICE\" AS \"PRICE\" "
"FROM \"PUBLIC\".\"VENUES\" "
- "WHERE \"PUBLIC\".\"VENUES\".\"PRICE\" < 3")]
- (qp.store/with-metadata-provider (lib.tu/metadata-provider-with-cards-for-queries
- meta/metadata-provider
- [mbql-query])
- (is (= {:lib/type :metabase.lib.parameters.parse.types/referenced-card-query
- :card-id 1
- :query expected-sql
- :parameters nil}
- (value-for-tag
- {:name "card-template-tag-test"
- :display-name "Card template tag test"
- :type :card
- :card-id 1}
- [])))))))))
-
-(deftest card-query-test-3
+ "WHERE \"PUBLIC\".\"VENUES\".\"PRICE\" < 3")
+ mp (lib.tu/metadata-provider-with-cards-for-queries
+ meta/metadata-provider
+ [mbql-query])]
+ (is (= {:lib/type :metabase.lib.parameters.parse.types/referenced-card-query
+ :card-id 1
+ :query expected-sql
+ :parameters nil}
+ (value-for-tag
+ mp
+ {:name "card-template-tag-test"
+ :display-name "Card template tag test"
+ :type :card
+ :card-id 1}
+ []))))))))
+
+(deftest ^:synchronized card-query-test-3
(mt/with-test-user :rasta
(testing "Persisted Models are substituted"
+ ;; legacy test -- don't hardcode driver names in new tests going forward.
+ #_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}
(mt/test-driver :postgres
- (mt/dataset test-data
- (mt/with-persistence-enabled! [persist-models!]
- (let [mbql-query (mt/mbql-query categories)]
- (mt/with-temp [:model/Card model {:name "model"
- :type :model
- :dataset_query mbql-query
- :database_id (mt/id)}]
- (persist-models!)
- (testing "tag uses persisted table"
- (let [pi (t2/select-one 'PersistedInfo :card_id (u/the-id model))]
- (is (= "persisted" (:state pi)))
- (is (re-matches #"select \* from \"metabase_cache_[a-z0-9]+_[0-9]+\".\"model_[0-9]+_model\""
- (:query
- (value-for-tag
- {:name "card-template-tag-test"
- :display-name "Card template tag test"
- :type :card
- :card-id (:id model)}
- []))))
- (testing "query hits persisted table"
- (let [persisted-schema (ddl.i/schema-name {:id (mt/id)}
- (system/site-uuid))
- update-query (format "update %s.%s set name = name || ' from cached table'"
- persisted-schema (:table_name pi))
- model-query (format "select c_orig.name, c_cached.name
+ ;; TODO (Cam 7/16/25) -- rework this to use metadata providers -- we support model persisted info directly from
+ ;; the metadata provider
+ (mt/with-persistence-enabled! [persist-models!]
+ (let [mp (mt/metadata-provider)
+ mbql-query (lib/query mp (lib.metadata/table mp (mt/id :categories)))]
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (mt/with-temp [:model/Card model {:name "model"
+ :type :model
+ :dataset_query mbql-query
+ :database_id (mt/id)}]
+ (persist-models!)
+ (testing "tag uses persisted table"
+ (let [pi (t2/select-one :model/PersistedInfo :card_id (u/the-id model))]
+ (is (= "persisted" (:state pi)))
+ (is (re-matches #"select \* from \"metabase_cache_[a-z0-9]+_[0-9]+\".\"model_[0-9]+_model\""
+ (:query
+ (value-for-tag
+ (mt/metadata-provider)
+ {:name "card-template-tag-test"
+ :display-name "Card template tag test"
+ :type :card
+ :card-id (:id model)}
+ []))))
+ (testing "query hits persisted table"
+ (let [persisted-schema (ddl.i/schema-name {:id (mt/id)}
+ (system/site-uuid))
+ update-query (format "update %s.%s set name = name || ' from cached table'"
+ persisted-schema (:table_name pi))
+ model-query (format "select c_orig.name, c_cached.name
from categories c_orig
left join {{#%d}} c_cached
on c_orig.id = c_cached.id
order by c_orig.id desc limit 3"
- (u/the-id model))
- tag-name (format "#%d" (u/the-id model))]
- (jdbc/execute! (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
- [update-query])
- (is (= [["Winery" "Winery from cached table"]
- ["Wine Bar" "Wine Bar from cached table"]
- ["Vegetarian / Vegan" "Vegetarian / Vegan from cached table"]]
- (mt/rows (qp/process-query
- {:database (mt/id)
- :type :native
- :native {:query model-query
+ (u/the-id model))
+ tag-name (format "#%d" (u/the-id model))]
+ (jdbc/execute! (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+ [update-query])
+ (is (= [["Winery" "Winery from cached table"]
+ ["Wine Bar" "Wine Bar from cached table"]
+ ["Vegetarian / Vegan" "Vegetarian / Vegan from cached table"]]
+ (mt/rows (qp/process-query
+ {:database (mt/id)
+ :type :native
+ :native {:query model-query
:template-tags
{(keyword tag-name)
- {:id "c6558da4-95b0-d829-edb6-45be1ee10d3c"
- :name tag-name
+ {:id "c6558da4-95b0-d829-edb6-45be1ee10d3c"
+ :name tag-name
:display-name tag-name
- :type "card"
- :card-id (u/the-id model)}}}}))))))))))))))))
+ :type "card"
+ :card-id (u/the-id model)}}}})))))))))))))))
(deftest ^:parallel card-query-test-4
(mt/with-test-user :rasta
(testing "Card query template tag wraps error in tag details"
- (qp.store/with-metadata-provider (lib.tu/metadata-provider-with-cards-for-queries
- meta/metadata-provider
- [{:type :native
- :native {:query "SELECT {{x}}"
- :template-tags {"x" {:id "x-tag"
- :name "x"
- :display-name "Number x"
- :type :number
- :required false}}}
- :database (meta/id)}
- {:type :native
- :native {:query "SELECT * FROM {{#1}} AS y"
- :template-tags {"#1" {:id "#1"
- :name "#1"
- :display-name "#1"
- :type "card"
- :card-id 1}}}
- :database (meta/id)}])
- (let [tag {:name "card-template-tag-test"
- :display-name "Card template tag test"
- :type :card
- :card-id 1}
- e (try
- (value-for-tag tag [])
- (catch ExceptionInfo e
- e))
- exc-data (some (fn [e]
- (when (:card-query-error? (ex-data e))
- (ex-data e)))
- (take-while some? (iterate ex-cause e)))]
- (testing "should be a card Query error"
- (is (true?
- (boolean (:card-query-error? exc-data)))))
- (testing "card-id"
- (is (= 1
- (:card-id exc-data))))
- (testing "tag"
- (is (= tag
- (:tag exc-data)))))))))
-
-(deftest card-query-permissions-test
+ (let [mp (lib.tu/metadata-provider-with-cards-for-queries
+ meta/metadata-provider
+ [{:type :native
+ :native {:query "SELECT {{x}}"
+ :template-tags {"x" {:id "x-tag"
+ :name "x"
+ :display-name "Number x"
+ :type :number
+ :required false}}}
+ :database (meta/id)}
+ {:type :native
+ :native {:query "SELECT * FROM {{#1}} AS y"
+ :template-tags {"#1" {:id "#1"
+ :name "#1"
+ :display-name "#1"
+ :type "card"
+ :card-id 1}}}
+ :database (meta/id)}])
+ tag {:name "card-template-tag-test"
+ :display-name "Card template tag test"
+ :type :card
+ :card-id 1}
+ e (try
+ (value-for-tag mp tag [])
+ (catch ExceptionInfo e
+ e))
+ exc-data (some (fn [e]
+ (when (:card-query-error? (ex-data e))
+ (ex-data e)))
+ (take-while some? (iterate ex-cause e)))]
+ (testing "should be a card Query error"
+ (is (true?
+ (boolean (:card-query-error? exc-data)))))
+ (testing "card-id"
+ (is (= 1
+ (:card-id exc-data))))
+ (testing "tag"
+ (is (= tag
+ (:tag exc-data))))))))
+
+(deftest ^:synchronized card-query-permissions-test
(testing "We should be able to run a query referenced via a template tag if we have perms for the Card in question (#12354)"
(mt/with-non-admin-groups-no-root-collection-perms
(mt/with-temp-copy-of-db
(mt/with-no-data-perms-for-all-users!
(data-perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/view-data :unrestricted)
(data-perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/create-queries :no)
+ ;; `with-temp` needed here because this tests permissions
+ #_{:clj-kondo/ignore [:discouraged-var]}
(mt/with-temp [:model/Collection collection {}
:model/Card {card-1-id :id} {:collection_id (u/the-id collection)
- :dataset_query (mt/mbql-query venues
- {:order-by [[:asc $id]] :limit 2})}
+ :dataset_query (let [mp (mt/metadata-provider)]
+ (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))
+ (lib/order-by (lib.metadata/field mp (mt/id :venues :id)) :asc)
+ (lib/limit 2)))}
:model/Card card-2 {:collection_id (u/the-id collection)
:dataset_query (mt/native-query
{:query "SELECT * FROM {{card}}"
@@ -544,67 +570,67 @@
clojure.lang.ExceptionInfo
(query->params-map query)))))))
-(defn- query-with-snippet [& {:as snippet-properties}]
- (assoc (mt/native-query {:query "SELECT * FROM {{expensive-venues}}"})
- :template-tags {"expensive-venues" (merge
- {:type :snippet
- :name "expensive-venues"
- :display-name "Expensive Venues"
- :snippet-name "expensive-venues"}
- snippet-properties)}))
+(defn- native-query-with-snippet [metadata-provider & {:as snippet-properties}]
+ (-> (lib/native-query metadata-provider "SELECT * FROM {{expensive_venues}}")
+ (lib/with-template-tags {"expensive_venues" (merge {:type :snippet
+ :name "expensive_venues"
+ :display-name "Expensive Venues"
+ :snippet-name "expensive_venues"}
+ snippet-properties)})))
(deftest ^:parallel snippet-validation-test
(testing "`:snippet-id` should be required"
- (is (thrown?
- clojure.lang.ExceptionInfo
- (query->params-map (query-with-snippet)))))
+ (let [query (native-query-with-snippet meta/metadata-provider)]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"\QUnable to resolve Snippet: missing `:snippet-id`\E"
+ (params.values/stage->params-map query (lib/query-stage query -1)))))))
+
+(deftest ^:parallel snippet-validation-test-2
(testing "If no such Snippet exists, it should throw an Exception"
- (is (thrown?
- clojure.lang.ExceptionInfo
- (query->params-map (query-with-snippet :snippet-id Integer/MAX_VALUE))))))
+ (let [query (native-query-with-snippet meta/metadata-provider :snippet-id Integer/MAX_VALUE)]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Snippet [\d,]+ \"expensive_venues\" not found\."
+ (params.values/stage->params-map query (lib/query-stage query -1)))))))
(deftest ^:parallel snippet-happy-path-test
(testing "Snippet parsing should work correctly for a valid Snippet"
- (mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "expensive-venues"
- :content "venues WHERE price = 4"}]
- (let [expected {"expensive-venues" (lib/parsed-referenced-query-snippet-param snippet-id "venues WHERE price = 4")}]
- (is (= expected
- (query->params-map (query-with-snippet :snippet-id snippet-id))))
- (testing "`:snippet-name` property in query shouldn't have to match `:name` of Snippet in DB"
- (is (= expected
- (query->params-map (query-with-snippet :snippet-id snippet-id, :snippet-name "Old Name")))))))))
-
-(deftest ^:parallel snippet-happy-path-mock-metadata-provider-test
- (testing "Snippet parsing should work correctly for a valid Snippet"
- (qp.store/with-metadata-provider (lib.tu/mock-metadata-provider
- meta/metadata-provider
- {:native-query-snippets [{:id 1
- :content "venues WHERE price = 4"}]})
- (let [expected {"expensive-venues" (lib/parsed-referenced-query-snippet-param 1 "venues WHERE price = 4")}]
- (is (= expected
- (query->params-map (query-with-snippet :snippet-id 1))))
- (testing "`:snippet-name` property in query shouldn't have to match `:name` of Snippet in DB"
+ (let [mp (lib.tu/mock-metadata-provider
+ meta/metadata-provider
+ {:native-query-snippets [{:id 1
+ :name "expensive_venues"
+ :content "venues WHERE price = 4"}]})
+ expected {"expensive_venues" (lib/parsed-referenced-query-snippet-param 1 "venues WHERE price = 4")}
+ query (native-query-with-snippet mp :snippet-id 1)]
+ (is (= expected
+ (#'params.values/stage->params-map query (lib/query-stage query -1))))
+ (testing "`:snippet-name` property in query shouldn't have to match `:name` of Snippet in DB"
+ (let [query (native-query-with-snippet mp :snippet-id 1, :snippet-name "Old Name")]
(is (= expected
- (query->params-map (query-with-snippet :snippet-id 1, :snippet-name "Old Name")))))))))
+ (#'params.values/stage->params-map query (lib/query-stage query -1)))))))))
(deftest ^:parallel unnormalized-snippet-test
(testing "Snippet parsing should normalize snippet names when parsing"
- (mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "expensive-venues"
- :content "venues WHERE price = 4"}]
- (let [expected {"snippet: expensive-venues" (lib/parsed-referenced-query-snippet-param snippet-id "venues WHERE price = 4")}
- query (assoc (mt/native-query {:query "SELECT * FROM {{snippet:expensive-venues}}"})
- :template-tags {"snippet:expensive-venues" {:type :snippet
- :name "expensive-venues"
- :display-name "Expensive Venues"
- :snippet-name "expensive-venues"
- :snippet-id snippet-id}})]
- (is (= expected
- (query->params-map query)))))))
+ (let [mp (lib.tu/mock-metadata-provider
+ meta/metadata-provider
+ {:native-query-snippets [{:id 1
+ :name "expensive_venues"
+ :content "venues WHERE price = 4"}]})
+ query (-> (lib/native-query mp "SELECT * FROM {{snippet:expensive_venues}}")
+ ;; force unnormalized snippet names
+ (lib/update-query-stage 0 assoc :template-tags {"snippet:expensive_venues" {:type :snippet
+ :name "snippet:expensive_venues"
+ :display-name "Expensive Venues"
+ :snippet-name "expensive_venues"
+ :snippet-id 1}}))]
+ (is (= {"snippet: expensive_venues" (lib/parsed-referenced-query-snippet-param 1 "venues WHERE price = 4")}
+ (params.values/stage->params-map query (lib/query-stage query -1)))))))
(deftest ^:parallel table-tag-test
(testing "Table template tag produces a ReferencedTableQuery"
(is (=? {:lib/type :metabase.lib.parameters.parse.types/referenced-table-query}
- (#'params.values/value-for-tag
+ (value-for-tag
{:name "table-tag-test"
:display-name "Table tag test"
:type :table
@@ -615,7 +641,9 @@
:display-name "Table tag test"
:type :table
:table-id 1}
- []))))
+ [])))))
+
+(deftest ^:parallel table-tag-test-2
(testing "Table template tag with emit-alias resolves alias from name"
(is (=? {:table-id 1
:alias "my_table"}
@@ -624,7 +652,9 @@
:type :table
:table-id 1
:emit-alias true}
- []))))
+ [])))))
+
+(deftest ^:parallel table-tag-test-3
(testing "Table template tag without emit-alias has nil alias"
(is (=? {:table-id 1
:alias nil}
@@ -696,20 +726,21 @@
(deftest ^:parallel parse-card-include-parameters-test
(testing "Parsing a Card reference should return a `ReferencedCardQuery` record that includes its parameters (#12236)"
- (qp.store/with-metadata-provider (lib.tu/mock-metadata-provider
- meta/metadata-provider
- {:cards [(assoc (:orders (lib.tu/mock-cards))
- :id 1
- :dataset-query (lib.tu.macros/mbql-query orders
- {:filter [:between $total 30 60]
- :aggregation [[:aggregation-options
- [:count-where [:starts-with $product-id->products.category "G"]]
- {:name "G Monies", :display-name "G Monies"}]]
- :breakout [!month.created-at]}))]})
+ (let [mp (lib.tu/mock-metadata-provider
+ meta/metadata-provider
+ {:cards [(assoc (:orders (lib.tu/mock-cards))
+ :id 1
+ :dataset-query (lib.tu.macros/mbql-query orders
+ {:filter [:between $total 30 60]
+ :aggregation [[:aggregation-options
+ [:count-where [:starts-with $product-id->products.category "G"]]
+ {:name "G Monies", :display-name "G Monies"}]]
+ :breakout [!month.created-at]}))]})]
(is (=? {:card-id 1
:query (every-pred string? (complement str/blank?))
:parameters ["G%"]}
(#'params.values/parse-tag
+ mp
{:id "5aa37572-058f-14f6-179d-a158ad6c029d"
:name "#1"
:display-name "#1"
@@ -723,12 +754,13 @@
(testing "Field filters"
(is (=? {"filter" {:value {:value ["Gizmo" "Gadget"]}}}
(query->params-map
- {:template-tags {"filter"
+ {:native "SELECT * FROM table WHERE {{filter}};"
+ :template-tags {"filter"
{:id "xyz456"
:name "filter"
:display-name "Filter"
:type :dimension
- :dimension [:field (mt/id :products :category) nil]
+ :dimension [:field (meta/id :products :category) nil]
:widget-type :category
:default ["Gizmo" "Gadget"]
:required true}}
@@ -742,7 +774,8 @@
(testing "Raw value template tags"
(is (= {"filter" "Foo"}
(query->params-map
- {:template-tags {"filter"
+ {:native "SELECT * FROM table WHERE {{filter}};"
+ :template-tags {"filter"
{:id "f0774ef5-a14a-e181-f557-2d4bb1fc94ae"
:name "filter"
:display-name "Filter"
@@ -760,12 +793,13 @@
(testing "Field filters"
(is (=? {"filter" {:value lib/parsed-param-no-value-placeholder}}
(query->params-map
- {:template-tags {"filter"
+ {:native "SELECT * FROM table WHERE {{filter}};"
+ :template-tags {"filter"
{:id "xyz456"
:name "filter"
:display-name "Filter"
:type :dimension
- :dimension [:field (mt/id :products :category) nil]
+ :dimension [:field (meta/id :products :category) nil]
:widget-type :category
:default ["Gizmo" "Gadget"]}}
:parameters [{:type :string/=
@@ -778,7 +812,8 @@
(testing "Raw value template tags"
(is (= {"filter" lib/parsed-param-no-value-placeholder}
(query->params-map
- {:template-tags {"filter"
+ {:native "SELECT * FROM table WHERE {{filter}};"
+ :template-tags {"filter"
{:id "f0774ef5-a14a-e181-f557-2d4bb1fc94ae"
:name "filter"
:display-name "Filter"
@@ -794,11 +829,12 @@
(is (=? {"checkin_date" {:value [{:type :date/range, :value "2015-01-01~2016-09-01"}
{:type :date/single, :value "2015-07-01"}]}}
(query->params-map
- {:template-tags {"checkin_date" {:name "checkin_date"
+ {:native "SELECT * FROM table WHERE {{checkin_date}};"
+ :template-tags {"checkin_date" {:name "checkin_date"
:display-name "Checkin Date"
:type :dimension
:widget-type :date/all-options
- :dimension [:field (mt/id :checkins :date) nil]}}
+ :dimension [:field (meta/id :checkins :date) nil]}}
:parameters [{:type :date/range
:target [:dimension [:template-tag "checkin_date"]]
:value "2015-01-01~2016-09-01"}
@@ -812,12 +848,13 @@
(testing "Field filters"
(is (=? {"filter" {:value lib/parsed-param-no-value-placeholder}}
(query->params-map
- {:template-tags {"filter"
+ {:native "SELECT * FROM table WHERE {{filter}};"
+ :template-tags {"filter"
{:id "xyz456"
:name "filter"
:display-name "Filter"
:type :dimension
- :dimension [:field (mt/id :products :category) nil]
+ :dimension [:field (meta/id :products :category) nil]
:widget-type :category}}
:parameters [{:type :string/=
:id "abc123"
@@ -830,7 +867,8 @@
(testing "Raw value template tags"
(is (= {"filter" lib/parsed-param-no-value-placeholder}
(query->params-map
- {:template-tags {"filter"
+ {:native "SELECT * FROM table WHERE {{filter}};"
+ :template-tags {"filter"
{:id "f0774ef5-a14a-e181-f557-2d4bb1fc94ae"
:name "filter"
:display-name "Filter"
@@ -853,105 +891,113 @@
(deftest ^:parallel handle-referenced-card-parameter-mixed-with-other-parameters-test
(testing "Should be able to handle for Card ref params regardless of whether other params are passed in (#21246)\n"
- (mt/dataset test-data
- (qp.store/with-metadata-provider (lib.tu/metadata-provider-with-cards-for-queries
- meta/metadata-provider
- [(lib.tu.macros/mbql-query products)])
- (let [param-name "#1"
- template-tags {param-name {:type :card
- :card-id 1
- :display-name param-name
- :id "__source__"
- :name param-name}}]
- (testing "With no parameters passed in"
- (is (=? {param-name {:lib/type :metabase.lib.parameters.parse.types/referenced-card-query}}
- (query->params-map {:template-tags template-tags}))))
- (testing "WITH parameters passed in"
- (let [parameters [{:type :date/all-options
- :value "2022-04-20"
- :target [:dimension [:template-tag "created_at"]]}]]
- (is (=? {param-name {:lib/type :metabase.lib.parameters.parse.types/referenced-card-query}}
- (query->params-map {:template-tags template-tags
- :parameters parameters}))))))))))
+ (let [mp (lib.tu/metadata-provider-with-cards-for-queries
+ meta/metadata-provider
+ [(lib.tu.macros/mbql-query products)])
+ param-name "#1"
+ template-tags {param-name {:type :card
+ :card-id 1
+ :display-name param-name
+ :id "__source__"
+ :name param-name}}]
+ (testing "With no parameters passed in"
+ (is (=? {param-name {:lib/type :metabase.lib.parameters.parse.types/referenced-card-query}}
+ (query->params-map mp {:native "SELECT * FROM {{__source__}};"
+ :template-tags template-tags}))))
+ (testing "WITH parameters passed in"
+ (let [parameters [{:type :date/all-options
+ :value "2022-04-20"
+ :target [:dimension [:template-tag "created_at"]]}]]
+ (is (=? {param-name {:lib/type :metabase.lib.parameters.parse.types/referenced-card-query}}
+ (query->params-map mp {:native "SELECT * FROM {{__source__}};"
+ :template-tags template-tags
+ :parameters parameters}))))))))
(deftest ^:parallel handle-dashboard-parameters-without-values-test
(testing "dash params for a template tag may have no :value or :default (#38012)"
- (mt/dataset test-data
- (qp.store/with-metadata-provider meta/metadata-provider
- (let [template-tags {"createdAt" {:type :dimension
- :dimension [:field (meta/id :orders :created-at) nil]
- :name "createdAt"
- :id "4636d745-1467-4a70-ba20-2a08069d77ff"
- :display-name "CreatedAt"
- :widget-type :date/all-options}}]
- (testing "with no parameters given, no value"
- (is (=? {"createdAt" {:field {:lib/type :metadata/column}
- :value lib/parsed-param-no-value-placeholder}}
- (query->params-map {:template-tags template-tags}))))
- (testing "with parameters given but blank, no value"
- (is (=? {"createdAt" {:field {:lib/type :metadata/column}
- :value lib/parsed-param-no-value-placeholder}}
- (query->params-map {:template-tags template-tags
- :parameters [{:type :date/relative
- :value nil
- :target [:dimension [:template-tag "createdAt"]]}
- {:type :date/month-year
- :value nil
- :target [:dimension [:template-tag "createdAt"]]}]}))))
- (testing "with only the relative date parameter set, use it"
- (is (=? {"createdAt" {:field {:lib/type :metadata/column}
- :value {:type :date/relative
- :value "past30days"}}}
- (query->params-map {:template-tags template-tags
- :parameters [{:type :date/relative
- :value "past30days"
- :target [:dimension [:template-tag "createdAt"]]}
- {:type :date/month-year
- :value nil
- :target [:dimension [:template-tag "createdAt"]]}]}))))
- (testing "with only the month-year parameter set, use it"
- (is (=? {"createdAt" {:field {:lib/type :metadata/column}
- :value {:type :date/month-year
- :value "2023-01"}}}
- (query->params-map {:template-tags template-tags
- :parameters [{:type :date/relative
- :value nil
- :target [:dimension [:template-tag "createdAt"]]}
- {:type :date/month-year
- :value "2023-01"
- :target [:dimension [:template-tag "createdAt"]]}]}))))
- (testing "with both parameters set, use both"
- (is (=? {"createdAt" {:field {:lib/type :metadata/column}
- :value [{:type :date/relative
- :value "past30days"}
- {:type :date/month-year
- :value "2023-01"}]}}
- (query->params-map {:template-tags template-tags
- :parameters [{:type :date/relative
- :value "past30days"
- :target [:dimension [:template-tag "createdAt"]]}
- {:type :date/month-year
- :value "2023-01"
- :target [:dimension [:template-tag "createdAt"]]}]})))))))))
+ (let [template-tags {"createdAt" {:type :dimension
+ :dimension [:field (meta/id :orders :created-at) nil]
+ :name "createdAt"
+ :id "4636d745-1467-4a70-ba20-2a08069d77ff"
+ :display-name "CreatedAt"
+ :widget-type :date/all-options}}]
+ (testing "with no parameters given, no value"
+ (is (=? {"createdAt" {:field {:lib/type :metadata/column}
+ :value lib/parsed-param-no-value-placeholder}}
+ (query->params-map {:native "SELECT * FROM table WHERE {{createdAt}};"
+ :template-tags template-tags}))))
+ (testing "with parameters given but blank, no value"
+ (is (=? {"createdAt" {:field {:lib/type :metadata/column}
+ :value lib/parsed-param-no-value-placeholder}}
+ (query->params-map {:native "SELECT * FROM table WHERE {{createdAt}};"
+ :template-tags template-tags
+ :parameters [{:type :date/relative
+ :value nil
+ :target [:dimension [:template-tag "createdAt"]]}
+ {:type :date/month-year
+ :value nil
+ :target [:dimension [:template-tag "createdAt"]]}]}))))
+ (testing "with only the relative date parameter set, use it"
+ (is (=? {"createdAt" {:field {:lib/type :metadata/column}
+ :value {:type :date/relative
+ :value "past30days"}}}
+ (query->params-map {:native "SELECT * FROM table WHERE {{createdAt}};"
+ :template-tags template-tags
+ :parameters [{:type :date/relative
+ :value "past30days"
+ :target [:dimension [:template-tag "createdAt"]]}
+ {:type :date/month-year
+ :value nil
+ :target [:dimension [:template-tag "createdAt"]]}]}))))
+ (testing "with only the month-year parameter set, use it"
+ (is (=? {"createdAt" {:field {:lib/type :metadata/column}
+ :value {:type :date/month-year
+ :value "2023-01"}}}
+ (query->params-map {:native "SELECT * FROM table WHERE {{createdAt}};"
+ :template-tags template-tags
+ :parameters [{:type :date/relative
+ :value nil
+ :target [:dimension [:template-tag "createdAt"]]}
+ {:type :date/month-year
+ :value "2023-01"
+ :target [:dimension [:template-tag "createdAt"]]}]}))))
+ (testing "with both parameters set, use both"
+ (is (=? {"createdAt" {:field {:lib/type :metadata/column}
+ :value [{:type :date/relative
+ :value "past30days"}
+ {:type :date/month-year
+ :value "2023-01"}]}}
+ (query->params-map {:native "SELECT * FROM table WHERE {{createdAt}};"
+ :template-tags template-tags
+ :parameters [{:type :date/relative
+ :value "past30days"
+ :target [:dimension [:template-tag "createdAt"]]}
+ {:type :date/month-year
+ :value "2023-01"
+ :target [:dimension [:template-tag "createdAt"]]}]})))))))
(deftest ^:parallel referenced-card-ids-test
- (mt/with-temp [:model/Card {card-1-id :id} {:collection_id nil
- :dataset_query (mt/mbql-query venues {:limit 2})}
- :model/Card {card-2-id :id} {:collection_id nil
- :dataset_query (mt/native-query
- {:query "SELECT * FROM {{card}}"
- :template-tags {"card" {:name "card"
- :display-name "card"
- :type :card
- :card-id card-1-id}}})}]
+ (let [mp (lib.tu/mock-metadata-provider
+ meta/metadata-provider
+ {:cards [{:id 1
+ :dataset-query (lib.tu.macros/mbql-query venues {:limit 2})}
+ {:id 2
+ :dataset-query {:database (meta/id)
+ :type :native
+ :native {:query "SELECT * FROM {{card}}"
+ :template-tags {"card" {:name "card"
+ :display-name "card"
+ :type :card
+ :card-id 1}}}}}]})
+ query (lib/query
+ mp
+ {:type :native
+ :native {:query "SELECT * FROM {{card}}"
+ :template-tags {"card" {:name "card"
+ :display-name "card"
+ :type :card
+ :card-id 2}}}})]
;; even tho Card 2 references Card 1, we don't want to include it in the set of referenced Card IDs, since you
;; should only need permissions for Card 2 to be able to run the query (see #15131)
- (testing (format "Card 1 ID = %d, Card 2 ID = %d" card-1-id card-2-id)
- (mt/with-metadata-provider (mt/id)
- (is (=? #{card-2-id}
- (params.values/referenced-card-ids (params.values/query->params-map
- {:query "SELECT * FROM {{card}}"
- :template-tags {"card" {:name "card"
- :display-name "card"
- :type :card
- :card-id card-2-id}}}))))))))
+ (is (=? #{2}
+ (params.values/referenced-card-ids (params.values/stage->params-map query (lib/query-stage query -1)))))))
diff --git a/test/metabase/test/data.clj b/test/metabase/test/data.clj
index d17c73151ad..13a0d2c9358 100644
--- a/test/metabase/test/data.clj
+++ b/test/metabase/test/data.clj
@@ -158,6 +158,7 @@
([table-name inner-query]
{:pre [(map? inner-query)]}
+ (assert (not (:type inner-query)) "Should be an inner-query, not an outer query")
(as-> inner-query <>
(mbql-query-impl/parse-tokens table-name <>)
(mbql-query-impl/maybe-add-source-table <> table-name)
@@ -192,7 +193,7 @@
#_{:clj-kondo/ignore [:deprecated-var]}
{:database (id)
:type :native
- :native (mbql.normalize/normalize ::mbql.s/NativeQuery inner-native-query)})
+ :native (mbql.normalize/normalize ::mbql.s/TopLevelNativeInnerQuery inner-native-query)})
(defn run-mbql-query* [query]
;; catch the Exception and rethrow with the query itself so we can have a little extra info for debugging if it fails.