Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Misc Lib improvements to support wider Lib usage in the backend (#63911)

Cam Saul3903af487fb3
diff --git a/enterprise/backend/src/metabase_enterprise/permission_debug/impl.clj b/enterprise/backend/src/metabase_enterprise/permission_debug/impl.clj
index ba46697096a..332e4a3a466 100644
--- a/enterprise/backend/src/metabase_enterprise/permission_debug/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/permission_debug/impl.clj
@@ -178,7 +178,7 @@
[user-id card permissions-blocking permissions-granting]
;; legacy usage -- don't do things like this going forward
(let [query (-> card :dataset_query qp.preprocess/preprocess #_{:clj-kondo/ignore [:discouraged-var]} lib/->legacy-MBQL)
- query-tables (-> query :query lib.util/collect-source-tables)
+ query-tables (-> query :query #_{:clj-kondo/ignore [:deprecated-var]} lib.util/collect-source-tables)
native? (boolean (lib.util.match/match-one query (m :guard (every-pred map? :native))))]
(->>
(cond
diff --git a/src/metabase/lib/card.cljc b/src/metabase/lib/card.cljc
index d57a0b048b7..5f359195f5c 100644
--- a/src/metabase/lib/card.cljc
+++ b/src/metabase/lib/card.cljc
@@ -55,8 +55,8 @@
(mu/defn- infer-returned-columns :- [:maybe [:sequential ::lib.schema.metadata/column]]
[metadata-providerable :- ::lib.schema.metadata/metadata-providerable
- {card-query :dataset-query :as _card} :- :map]
- (when (some? card-query)
+ {card-query :dataset-query :as _card} :- ::lib.schema.metadata/card]
+ (when (seq card-query)
(lib.metadata.calculation/returned-columns (lib.query/query metadata-providerable card-query))))
(mu/defn- ->card-metadata-column :- ::lib.schema.metadata/column
@@ -170,7 +170,7 @@
"If `card` itself has a source card that is a Model, return that Model's columns."
[metadata-providerable :- ::lib.schema.metadata/metadata-providerable
card :- ::lib.schema.metadata/card]
- (when-let [card-query (some->> (:dataset-query card) (lib.query/query metadata-providerable))]
+ (when-let [card-query (some->> (:dataset-query card) not-empty (lib.query/query metadata-providerable))]
(when-let [source-card-id (lib.util/source-card-id card-query)]
(when-not (= source-card-id (:id card))
(let [source-card (lib.metadata/card metadata-providerable source-card-id)]
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index 4ee5fdb65ac..6597fcf4da9 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -34,6 +34,7 @@
[metabase.lib.metadata]
[metabase.lib.metadata.calculation :as lib.metadata.calculation]
[metabase.lib.metadata.composed-provider :as lib.metadata.composed-provider]
+ [metabase.lib.metadata.protocols]
[metabase.lib.metric :as lib.metric]
[metabase.lib.native :as lib.native]
[metabase.lib.normalize :as lib.normalize]
@@ -44,6 +45,7 @@
[metabase.lib.query :as lib.query]
[metabase.lib.ref :as lib.ref]
[metabase.lib.remove-replace :as lib.remove-replace]
+ [metabase.lib.schema]
[metabase.lib.schema.util]
[metabase.lib.segment :as lib.segment]
[metabase.lib.serialize]
@@ -86,6 +88,7 @@
metabase.lib.metadata/keep-me
lib.metadata.calculation/keep-me
lib.metadata.composed-provider/keep-me
+ metabase.lib.metadata.protocols/keep-me
lib.metric/keep-me
lib.native/keep-me
lib.normalize/keep-me
@@ -95,6 +98,7 @@
lib.query/keep-me
lib.ref/keep-me
lib.remove-replace/keep-me
+ metabase.lib.schema/keep-me
metabase.lib.schema.util/keep-me
lib.segment/keep-me
metabase.lib.serialize/keep-me
@@ -336,6 +340,7 @@
limit
max-rows-limit]
[metabase.lib.metadata
+ ->metadata-provider
general-cached-value]
[lib.metadata.calculation
column-name
@@ -350,6 +355,10 @@
visible-columns]
[lib.metadata.composed-provider
composed-metadata-provider]
+ [metabase.lib.metadata.protocols
+ cached-metadata-provider-with-cache?
+ metadata-provider?
+ metadata-providerable?]
[lib.native
engine
extract-template-tags
@@ -408,6 +417,8 @@
rename-join
replace-clause
replace-join]
+ [metabase.lib.schema
+ native-only-query?]
[metabase.lib.schema.util]
[lib.segment
available-segments]
@@ -437,16 +448,23 @@
clause?
clause-of-type?
fresh-uuids
+ mbql-stage?
native-stage?
normalized-query-type
+ normalized-mbql-version
previous-stage
previous-stage-number
query-stage
source-table-id
+ source-card-id
update-query-stage]
[metabase.lib.walk.util
all-field-ids
all-source-card-ids
all-source-table-ids
+ all-template-tag-field-ids
+ all-template-tag-snippet-ids
all-template-tags
+ all-template-tags-map
+ all-template-tags-id->field-ids
any-native-stage?])
diff --git a/src/metabase/lib/metadata.cljc b/src/metabase/lib/metadata.cljc
index 47d12242bf1..9533a96f010 100644
--- a/src/metabase/lib/metadata.cljc
+++ b/src/metabase/lib/metadata.cljc
@@ -22,10 +22,26 @@
(mu/defn ->metadata-provider :- ::lib.schema.metadata/metadata-provider
"Get a MetadataProvider from something that can provide one."
- [metadata-providerable :- ::lib.schema.metadata/metadata-providerable]
- (if (lib.metadata.protocols/metadata-provider? metadata-providerable)
- metadata-providerable
- (some-> metadata-providerable :lib/metadata ->metadata-provider)))
+ ([metadata-providerable]
+ (->metadata-provider metadata-providerable nil))
+
+ ([metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ database-id :- [:maybe
+ [:or
+ ::lib.schema.id/database
+ ::lib.schema.id/saved-questions-virtual-database]]]
+ (cond
+ (lib.metadata.protocols/metadata-provider? metadata-providerable)
+ metadata-providerable
+
+ (map? metadata-providerable)
+ (some-> metadata-providerable :lib/metadata ->metadata-provider)
+
+ ((some-fn fn? var?) metadata-providerable)
+ (if (pos-int? database-id)
+ (metadata-providerable database-id)
+ (throw (ex-info "Cannot initialize new metadata provider without a Database ID"
+ {:f metadata-providerable}))))))
(mu/defn database :- ::lib.schema.metadata/database
"Get metadata about the Database we're querying."
diff --git a/src/metabase/lib/metadata/cached_provider.cljc b/src/metabase/lib/metadata/cached_provider.cljc
index dc1cabe0304..9cf229ac622 100644
--- a/src/metabase/lib/metadata/cached_provider.cljc
+++ b/src/metabase/lib/metadata/cached_provider.cljc
@@ -1,6 +1,7 @@
(ns metabase.lib.metadata.cached-provider
(:require
- #?@(:clj ([pretty.core :as pretty]))
+ #?@(:clj ([metabase.util.json :as json]
+ [pretty.core :as pretty]))
[clojure.set :as set]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
[metabase.lib.schema.metadata :as lib.schema.metadata]
@@ -158,3 +159,10 @@
^CachedProxyMetadataProvider [metadata-provider]
(log/debugf "Wrapping %s in CachedProxyMetadataProvider" (pr-str metadata-provider))
(->CachedProxyMetadataProvider (atom {}) metadata-provider))
+
+#?(:clj
+ ;; do not encode MetadataProviders to JSON, just generate `nil` instead.
+ (json/add-encoder
+ CachedProxyMetadataProvider
+ (fn [_mp json-generator]
+ (json/generate-nil nil json-generator))))
diff --git a/src/metabase/lib/metadata/invocation_tracker.cljc b/src/metabase/lib/metadata/invocation_tracker.cljc
index eba50703506..ca664f6e9a6 100644
--- a/src/metabase/lib/metadata/invocation_tracker.cljc
+++ b/src/metabase/lib/metadata/invocation_tracker.cljc
@@ -1,6 +1,8 @@
(ns metabase.lib.metadata.invocation-tracker
(:require
- #?(:clj [pretty.core :as pretty])
+ #?@(:clj
+ ([metabase.util.json :as json]
+ [pretty.core :as pretty]))
[metabase.lib.metadata.protocols :as lib.metadata.protocols]))
(def ^:private ^:dynamic *to-track-metadata-types*
@@ -89,3 +91,10 @@
"Wraps `metadata-provider` with a provider that records all invoked ids of [[lib.metadata.protocols/MetadataProvider]] methods."
[metadata-provider]
(->InvocationTracker (atom {}) metadata-provider))
+
+#?(:clj
+ ;; do not encode MetadataProviders to JSON, just generate `nil` instead.
+ (json/add-encoder
+ InvocationTracker
+ (fn [_mp json-generator]
+ (json/generate-nil nil json-generator))))
diff --git a/src/metabase/lib/metadata/protocols.cljc b/src/metabase/lib/metadata/protocols.cljc
index e81714b3fe4..fc15b25f32f 100644
--- a/src/metabase/lib/metadata/protocols.cljc
+++ b/src/metabase/lib/metadata/protocols.cljc
@@ -150,7 +150,12 @@
"Whether `x` is a [[metadata-provider?]], or has one attached at `:lib/metadata` (i.e., a query)."
[x]
(or (metadata-provider? x)
- (some-> x :lib/metadata metadata-providerable?)))
+ (some-> x :lib/metadata metadata-providerable?)
+ ;; function with the signature
+ ;;
+ ;; (f database-id) => metadata-provider
+ (fn? x)
+ (var? x)))
(mr/def ::metadata-providerable
"Something that can be used to get a MetadataProvider. Either a MetadataProvider, or a map with a MetadataProvider in
@@ -337,6 +342,8 @@
;;; single call to the underlying ApplicationDatabaseMetadataProvider... Fetch stuff already present in the cache
;;; without needing a lock, but if we need to fetch something from the parent provider wait for a lock to do it.
;;; -- Cam
+;;;
+;;; TODO (Cam 9/22/25) -- if we have this, do we still need [[store-metadata!]] and [[store-metadatas!]]?
(mu/defn warm-cache
"Convenience for warming a `CachedMetadataProvider` for side-effects. Checks whether the provider is a cached
metadata provider, and, if it is, calls [[metadatas]] to fetch the objects in question and warm the cache."
diff --git a/src/metabase/lib/query.cljc b/src/metabase/lib/query.cljc
index d8fcbc57eea..9d70e0ebdc7 100644
--- a/src/metabase/lib/query.cljc
+++ b/src/metabase/lib/query.cljc
@@ -2,6 +2,8 @@
(:refer-clojure :exclude [remove])
(:require
[medley.core :as m]
+ ;; allowed since this is needed to convert legacy queries to MBQL 5
+ ^{:clj-kondo/ignore [:discouraged-namespace]}
[metabase.legacy-mbql.normalize :as mbql.normalize]
[metabase.lib.convert :as lib.convert]
[metabase.lib.dispatch :as lib.dispatch]
@@ -120,10 +122,10 @@
[query :- ::lib.schema/query]
(can-run query "question"))
-(defn add-types-to-fields
+(mu/defn add-types-to-fields
"Add `:base-type` and `:effective-type` to options of fields in `x` using `metadata-provider`. Works on pmbql fields.
`:effective-type` is required for coerced fields to pass schema checks."
- [x metadata-provider]
+ [x metadata-provider :- ::lib.schema.metadata/metadata-provider]
(if-let [field-ids (lib.util.match/match x
[:field
(_options :guard (every-pred map? (complement (every-pred :base-type :effective-type))))
@@ -170,12 +172,13 @@
(defn- query-from-legacy-query
[metadata-providerable legacy-query]
(try
- (let [pmbql-query (-> (binding [lib.schema.expression/*suppress-expression-type-check?* true]
- (lib.convert/->pMBQL (mbql.normalize/normalize-or-throw legacy-query)))
- (add-types-to-fields metadata-providerable))]
+ (let [mbql5-query (binding [lib.schema.expression/*suppress-expression-type-check?* true]
+ (lib.convert/->pMBQL (mbql.normalize/normalize-or-throw legacy-query)))
+ mp (lib.metadata/->metadata-provider metadata-providerable (:database mbql5-query))
+ mbql5-query (add-types-to-fields mbql5-query mp)]
(merge
- pmbql-query
- (query-with-stages metadata-providerable (:stages pmbql-query))))
+ mbql5-query
+ (query-with-stages mp (:stages mbql5-query))))
(catch #?(:clj Throwable :cljs :default) e
(throw (ex-info (i18n/tru "Error creating query from legacy query: {0}" (ex-message e))
{:legacy-query legacy-query}
@@ -207,12 +210,12 @@
;; - fill in top expression refs with metadata
(defmethod query-method :mbql/query
[metadata-providerable {converted? :lib.convert/converted? :as query}]
- (let [metadata-provider (lib.metadata/->metadata-provider metadata-providerable)
- query (-> query
- (assoc :lib/metadata metadata-provider)
- (dissoc :lib.convert/converted?)
-
- lib.normalize/normalize)
+ (let [database-id (some #(get query %) [:database "database"])
+ metadata-provider (lib.metadata/->metadata-provider metadata-providerable database-id)
+ query (-> query
+ (assoc :lib/metadata metadata-provider)
+ (dissoc :lib.convert/converted?)
+ lib.normalize/normalize)
stages (:stages query)]
(cond-> query
converted?
@@ -232,9 +235,9 @@
second
(select-keys [:base-type :effective-type])))
(catch #?(:clj Exception :cljs :default) _
- ;; This currently does not find expressions defined in join stages
+ ;; This currently does not find expressions defined in join stages
nil))]
- ;; Fallback if metadata is missing
+ ;; Fallback if metadata is missing
[:expression (merge found-ref opts) expression-name]))))
(m/indexed stages))))))
diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljc
index c422301ce01..3f5d036f5b9 100644
--- a/src/metabase/lib/schema.cljc
+++ b/src/metabase/lib/schema.cljc
@@ -393,7 +393,8 @@
(mr/def ::query
[:and
[:map
- {:decode/normalize common/normalize-map
+ {:description "Valid MBQL 5 query."
+ :decode/normalize common/normalize-map
:encode/serialize serialize-query}
[:lib/type [:=
{:decode/normalize common/normalize-keyword, :default :mbql/query}
@@ -437,3 +438,17 @@
:expressions ":expressions is not allowed in the top level of a query, only in MBQL stages"
:filters ":filters is not allowed in the top level of a query, only in MBQL stages"
:source-table ":source-table is not allowed in the top level of a query, only in MBQL stages"})])
+
+(defn native-only-query?
+ "Whether MBQL 5 `query` only has a single native stage (and is thus pure-native). This is the equivalent of the old
+ `:type :native` queries in MBQL <= 4."
+ [query]
+ (and (map? query)
+ (= (count (:stages query)) 1)
+ (= (get-in query [:stages 0 :lib/type]) :mbql.stage/native)))
+
+(mr/def ::native-only-query
+ "Schema for a pure-native query with one single native stage."
+ [:and
+ [:ref ::query]
+ [:fn {:error/message "native-only query"} native-only-query?]])
diff --git a/src/metabase/lib/schema/binning.cljc b/src/metabase/lib/schema/binning.cljc
index 0e69d5eca70..99bdcd0be36 100644
--- a/src/metabase/lib/schema/binning.cljc
+++ b/src/metabase/lib/schema/binning.cljc
@@ -25,7 +25,6 @@
(mr/def ::binning
"Schema for `:binning` options passed to a `:field` clause."
[:and
- {:doc/title "`:binning` options"}
[:map
{:decode/normalize lib.schema.common/normalize-map}
[:strategy [:ref ::strategy]]]
diff --git a/src/metabase/lib/schema/common.cljc b/src/metabase/lib/schema/common.cljc
index c6591f0532e..6bde2f85ab8 100644
--- a/src/metabase/lib/schema/common.cljc
+++ b/src/metabase/lib/schema/common.cljc
@@ -154,7 +154,7 @@
;;; only support fixing really broken types like `:type/creationtime` to `:type/CreationTime` in prod... Malli checks
;;; will throw in dev. See [[metabase.lib.schema.common-test/normalize-base-type-test]] for more info
-(mu/defn- normalize-base-type* :- [:ref ::base-type]
+(mu/defn- normalize-base-type* :- [:maybe [:ref ::base-type]]
[x]
(normalize-keyword x))
diff --git a/src/metabase/lib/schema/id.cljc b/src/metabase/lib/schema/id.cljc
index 397c9ad1667..68d076e4a47 100644
--- a/src/metabase/lib/schema/id.cljc
+++ b/src/metabase/lib/schema/id.cljc
@@ -6,7 +6,8 @@
;;; them, e.g. when we start working on the generative stuff
(mr/def ::database
- [:schema {:doc/title "Valid Database ID"} pos-int?])
+ "Valid Database ID"
+ pos-int?)
(def saved-questions-virtual-database-id
"The ID used to signify that a database is 'virtual' rather than physical.
@@ -27,45 +28,57 @@
;;; with that Card. The QP will resolve this to the correct Database later.
(mr/def ::saved-questions-virtual-database
[:=
- {:doc/title "Saved Questions Virtual Database ID"
- :description (:doc (meta #'saved-questions-virtual-database-id))}
+ {:description (:doc (meta #'saved-questions-virtual-database-id))}
saved-questions-virtual-database-id])
(mr/def ::table
- [:schema {:doc/title "Valid Table ID"} pos-int?])
+ "Valid Table ID"
+ pos-int?)
(mr/def ::field
- [:schema {:doc/title "Valid Field ID"} pos-int?])
+ "Valid Field ID"
+ pos-int?)
(mr/def ::card
- [:schema {:doc/title "Valid Card ID"} pos-int?])
+ "Valid Card ID"
+ pos-int?)
(mr/def ::segment
- [:schema {:doc/title "Valid legacy Segment ID"} pos-int?])
+ "Valid legacy Segment ID"
+ pos-int?)
(mr/def ::snippet
- [:schema {:doc/title "Valid Snippet ID"} pos-int?])
+ "Valid Snippet ID"
+ pos-int?)
(mr/def ::dimension
- [:schema {:doc/title "Valid Dimension ID"} pos-int?])
+ "Valid Dimension ID"
+ pos-int?)
(mr/def ::action
- [:schema {:doc/title "Valid Action ID"} pos-int?])
+ "Valid Action ID"
+ pos-int?)
(mr/def ::dashboard
- [:schema {:doc/title "Valid Dashboard ID"} pos-int?])
+ "Valid Dashboard ID"
+ pos-int?)
(mr/def ::dashcard
- [:schema {:doc/title "Valid DashboardCard ID"} pos-int?])
+ "Valid DashboardCard ID"
+ pos-int?)
(mr/def ::user
- [:schema {:doc/title "Valid User ID"} pos-int?])
+ "Valid User ID"
+ pos-int?)
(mr/def ::pulse
- [:schema {:doc/title "Valid Pulse ID"} pos-int?])
+ "Valid Pulse ID"
+ pos-int?)
(mr/def ::native-query-snippet
- [:schema {:doc/title "Valid Native Query Snippet ID"} pos-int?])
+ "Valid Native Query Snippet ID"
+ pos-int?)
(mr/def ::collection
- [:schema {:doc/title "Valid Collection ID"} pos-int?])
+ "Valid Collection ID"
+ pos-int?)
diff --git a/src/metabase/lib/schema/literal.cljc b/src/metabase/lib/schema/literal.cljc
index a4245a66efd..374711e171d 100644
--- a/src/metabase/lib/schema/literal.cljc
+++ b/src/metabase/lib/schema/literal.cljc
@@ -91,8 +91,8 @@
:cljs ::string.date))
(mr/def ::time
+ "time literal"
#?(:clj [:or
- {:doc/title "time literal"}
::string.time
[:time/local-time
{:error/message "instance of java.time.LocalTime"
diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljc
index 97b682c52aa..5ca63d59af1 100644
--- a/src/metabase/lib/schema/metadata.cljc
+++ b/src/metabase/lib/schema/metadata.cljc
@@ -1,5 +1,7 @@
(ns metabase.lib.schema.metadata
(:require
+ #?@(:clj
+ ([metabase.util.regex :as u.regex]))
[clojure.string :as str]
[medley.core :as m]
[metabase.lib.schema.binning :as lib.schema.binning]
@@ -536,11 +538,26 @@
[:definition {:optional true} [:maybe [:ref ::persisted-info.definition]]]
[:query-hash {:optional true} [:maybe ::lib.schema.common/non-blank-string]]])
+(def card-types
+ "Valid Card `:type`s."
+ #{:question :model :metric})
+
(mr/def ::card.type
- [:enum
- :question
- :model
- :metric])
+ "All acceptable card types.
+
+ Previously (< 49), we only had 2 card types: question and model, which were differentiated using the boolean
+ `dataset` column. Soon we'll have more card types (e.g: metric) and we will longer be able to use a boolean column
+ to differentiate between all types. So we've added a new `type` column for this purpose.
+
+ Migrating all the code to use `report_card.type` will be quite an effort, we decided that we'll migrate it
+ gradually."
+ (into [:enum
+ (merge
+ {:decode/json lib.schema.common/normalize-keyword
+ :decode/normalize lib.schema.common/normalize-keyword}
+ #?(:clj
+ {:api/regex (u.regex/re-or (map name card-types))}))]
+ card-types))
(mr/def ::type
"TODO -- not convinced we need a separate `:metadata/metric` anymore, it made sense back when Legacy/V1 Metrics were a
diff --git a/src/metabase/lib/schema/parameter.cljc b/src/metabase/lib/schema/parameter.cljc
index 7642f9744d2..acffae442a7 100644
--- a/src/metabase/lib/schema/parameter.cljc
+++ b/src/metabase/lib/schema/parameter.cljc
@@ -24,6 +24,7 @@
(:require
#?@(:clj
([flatland.ordered.map :as ordered-map]))
+ [malli.core :as mc]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.util.malli.registry :as mr]))
@@ -143,6 +144,7 @@
:boolean/= {:type :boolean, :operator :variadic, :allowed-for #{:boolean :boolean/=}}))
(mr/def ::type
+ "Valid parameter :type"
(into [:enum {:error/message "valid parameter type"
:decode/normalize lib.schema.common/normalize-keyword}]
(keys types)))
@@ -183,12 +185,12 @@
(defn- normalize-legacy-ref [legacy-ref]
((#?(:clj requiring-resolve :cljs resolve) 'metabase.legacy-mbql.normalize/normalize-field-ref) legacy-ref))
-(mr/def ::legacy-field-ref
+(mr/def ::target.legacy-field-ref
[:ref
{:decode/normalize normalize-legacy-ref}
:metabase.legacy-mbql.schema/field])
-(mr/def ::legacy-expression-ref
+(mr/def ::target.legacy-expression-ref
[:ref
{:decode/normalize normalize-legacy-ref}
:metabase.legacy-mbql.schema/expression])
@@ -198,9 +200,10 @@
:error/fn (fn [{:keys [value]} _]
(str "Invalid :dimension target: must be a :field, :template-tag, or :expression, got: "
(pr-str value)))}
- [:field [:ref ::legacy-field-ref]]
- [:expression [:ref ::legacy-expression-ref]]
- [:template-tag [:ref ::template-tag]]])
+ [:expression [:ref ::target.legacy-expression-ref]]
+ [:template-tag [:ref ::template-tag]]
+ ;; other stuff like MBQL 3 `:fk->` and `:field-id` need to get converted to MBQL 4 `:field`
+ [::mc/default [:ref ::target.legacy-field-ref]]])
;;; TODO (Cam 8/8/25) -- is options supposed to be non-empty? It it supposed to be removed from `:dimension` if it's
;;; empty? Unclear. I don't think it matters tho.
@@ -213,6 +216,24 @@
;;; real MBQL clause tho.
(mr/def ::dimension
[:catn
+ ;; this `:decode/normalize` function seems unnecessary but it improves the errors a lot: without it we won't
+ ;; normalize the tag if the target or options are invalid:
+ ;;
+ ;; without:
+ ;;
+ ;; (metabase.lib.core/normalize ::target ["dimension" ["template-tags" "category"]])
+ ;; ;; WARN lib.normalize :: Error normalizing MBQL 5: [["should be :dimension"]]
+ ;; {:value ["dimension" ["template-tags" "category"]], :schema :metabase.lib.schema.parameter/target}
+ ;;
+ ;; with:
+ ;;
+ ;; ;; WARN lib.normalize :: Error normalizing MBQL 5: [nil ["Invalid :dimension target: must be a :field, :template-tag, or :expression, got: [\"template-tags\" \"category\"]"]]
+ ;; {:value [:dimension ["template-tags" "category"]], :schema :metabase.lib.schema.parameter/target}
+ {:decode/normalize (fn [dimension]
+ (if (and (sequential? dimension)
+ (string? (first dimension)))
+ (update (vec dimension) 0 keyword)
+ dimension))}
[:tag [:= {:decode/normalize lib.schema.common/normalize-keyword} :dimension]]
[:target ::dimension.target]
[:options [:? [:maybe ::dimension.options]]]])
@@ -232,7 +253,7 @@
:error/message "A :variable target must be a (legacy) :field or :template-tag"
:error/fn (fn [{:keys [value]} _]
(str "Invalid :variable target: must be a :field or :template-tag, got: " (pr-str value)))}
- [:field [:ref ::legacy-field-ref]]
+ [:field [:ref ::target.legacy-field-ref]]
[:template-tag [:ref ::template-tag]]])
(mr/def ::variable
@@ -247,9 +268,10 @@
(pr-str value)))}
;; TODO (Cam 9/12/25) -- the old legacy MBQL schema also said `:expression` refs where allowed here, but I don't
;; know if we actually did allow that in practice.
- [:field [:ref ::legacy-field-ref]]
- [:dimension [:ref ::dimension]]
- [:variable [:ref ::variable]]])
+ [:dimension [:ref ::dimension]]
+ [:variable [:ref ::variable]]
+ ;; MBQL 3 refs like `:field-id` should get normalized to `:field`
+ [::mc/default [:ref ::target.legacy-field-ref]]])
(defn- normalize-parameter
[param]
@@ -263,6 +285,9 @@
(nil? l) (assoc :type :number/<=, :value [u])))
param))))
+(mr/def ::id
+ [:ref ::lib.schema.common/non-blank-string])
+
(mr/def ::parameter
"Schema for the *value* of a parameter (e.g. a Dashboard parameter or a native query template tag) as passed in as
part of the `:parameters` list in a query."
@@ -272,7 +297,7 @@
[:type [:ref ::type]]
;; TODO -- these definitely SHOULD NOT be optional but a ton of tests aren't passing them in like they should be.
;; At some point we need to go fix those tests and then make these keys required
- [:id {:optional true} ::lib.schema.common/non-blank-string]
+ [:id {:optional true} [:ref ::id]]
[:target {:optional true} [:ref ::target]]
;; not specified if the param has no value. TODO - make this stricter; type of `:value` should be validated based
;; on the [[ParameterType]]
diff --git a/src/metabase/lib/schema/template_tag.cljc b/src/metabase/lib/schema/template_tag.cljc
index 6b73eb15fe1..aeb0f60d10a 100644
--- a/src/metabase/lib/schema/template_tag.cljc
+++ b/src/metabase/lib/schema/template_tag.cljc
@@ -17,8 +17,8 @@
;; TODO -- move this stuff into `metabase.lib`
(keys lib.schema.parameter/types)))
-;; Schema for valid values of template tag `:type`.
(mr/def ::type
+ "Schema for valid values of template tag `:type`."
[:enum
{:decode/normalize common/normalize-keyword}
:snippet :card :dimension :number :text :date :boolean :temporal-unit])
@@ -28,6 +28,11 @@
{:decode/normalize common/normalize-string-key}
::common/non-blank-string])
+(mr/def ::id
+ [:multi {:dispatch uuid?}
+ [true :uuid]
+ [false ::common/non-blank-string]])
+
;;; Things required by all template tag types.
(mr/def ::common
[:map
@@ -35,9 +40,7 @@
[:display-name ::common/non-blank-string]
;; TODO -- `:id` is actually 100% required but we have a lot of tests that don't specify it because this constraint
;; wasn't previously enforced; we need to go in and fix those tests and make this non-optional
- [:id {:optional true} [:multi {:dispatch uuid?}
- [true :uuid]
- [false ::common/non-blank-string]]]])
+ [:id {:optional true} [:ref ::id]]])
;;; Stuff shared between the Field filter and raw value template tag schemas.
(mr/def ::value.common
diff --git a/src/metabase/lib/stage/util.cljc b/src/metabase/lib/stage/util.cljc
index 8f526962868..31e856f6787 100644
--- a/src/metabase/lib/stage/util.cljc
+++ b/src/metabase/lib/stage/util.cljc
@@ -3,6 +3,7 @@
high-level ones like `lib.schema` and `lib.util`. This namespace was spun out from [[metabase.lib.stage]] to avoid
circular dependencies between it and other Lib namespaces."
(:require
+ [medley.core :as m]
[metabase.lib.schema :as lib.schema]
[metabase.lib.util :as lib.util]
[metabase.util.malli :as mu]))
@@ -11,7 +12,11 @@
"Does given query stage have any clauses?"
[query :- ::lib.schema/query
stage-number :- :int]
- (boolean (seq (dissoc (lib.util/query-stage query stage-number) :lib/type :source-table :source-card))))
+ (-> (lib.util/query-stage query stage-number)
+ (dissoc :source-table :source-card)
+ (->> (m/filter-keys simple-keyword?))
+ not-empty
+ boolean))
(mu/defn append-stage :- ::lib.schema/query
"Adds a new blank stage to the end of the pipeline."
diff --git a/src/metabase/lib/util.cljc b/src/metabase/lib/util.cljc
index 9e8402b9e8f..1d65764943b 100644
--- a/src/metabase/lib/util.cljc
+++ b/src/metabase/lib/util.cljc
@@ -2,8 +2,8 @@
(:refer-clojure :exclude [format])
(:require
#?@(:clj
- ([potemkin :as p]))
- #?@(:cljs
+ ([potemkin :as p])
+ :cljs
(["crc-32" :as CRC32]
[goog.string :as gstring]
[goog.string.format :as gstring.format]))
@@ -54,7 +54,8 @@
;;; TODO (Cam 9/8/25) -- some overlap with [[metabase.lib.dispatch/mbql-clause-type]]
(defn clause-of-type?
- "Returns true if this is a clause."
+ "Returns truthy if is a clause of `clause-type`, which can be either a keyword (like `:field`) or a set (like
+ `#{:field :expression}`)."
[clause clause-type]
(and (clause? clause)
(if (set? clause-type)
@@ -395,6 +396,13 @@
([query stage-number]
(native-stage? (query-stage query stage-number))))
+(defn mbql-stage?
+ "Is this query stage an MBQL stage?"
+ ([stage]
+ (= (:lib/type stage) :mbql.stage/mbql))
+ ([query stage-number]
+ (mbql-stage? (query-stage query stage-number))))
+
(mu/defn ensure-mbql-final-stage :- ::lib.schema/query
"Convert query to a MBQL 5 (pipeline) query, and make sure the final stage is an `:mbql` one."
[query]
@@ -715,19 +723,25 @@
(when (#{:mbql/query :query :native :internal} query-type)
query-type))))
-(mu/defn referenced-field-ids :- [:maybe [:set ::lib.schema.id/field]]
- "Find all the integer field IDs in `coll`, Which can arbitrarily be anything that is part of MLv2 query schema."
- [coll]
- (not-empty
- (into #{}
- (comp cat (filter some?))
- (lib.util.match/match coll [:field opts (id :guard int?)] [id (:source-field opts)]))))
+(mu/defn normalized-mbql-version :- [:maybe [:enum :mbql-version/mbql5 :mbql-version/legacy]]
+ "Version of MBQL a `query` map is using, either `:mbql-version/mbql-5` or `:mbql-version/legacy`."
+ [query :- [:maybe :map]]
+ (case (normalized-query-type query)
+ :mbql/query :mbql-version/mbql5
+ (:query :native) :mbql-version/legacy
+ ;; otherwise, this is not a valid MBQL query.
+ nil))
(defn collect-source-tables
- "Return sequence of source tables from `query`."
- [query]
- (let [from-joins (mapcat collect-source-tables (:joins query))]
- (if-let [source-query (:source-query query)]
+ "Return sequence of source tables from `query`.
+
+ DEPRECATED: This operates on legacy MBQL, so it's really out of place here in Lib.
+ Use [[metabase.lib.walk.util/all-source-table-ids]] going forward."
+ {:deprecated "0.57.0"}
+ [legacy-query]
+ #_{:clj-kondo/ignore [:deprecated-var]}
+ (let [from-joins (mapcat collect-source-tables (:joins legacy-query))]
+ (if-let [source-query (:source-query legacy-query)]
(concat (collect-source-tables source-query) from-joins)
(cond->> from-joins
- (:source-table query) (cons (:source-table query))))))
+ (:source-table legacy-query) (cons (:source-table legacy-query))))))
diff --git a/src/metabase/lib/walk.cljc b/src/metabase/lib/walk.cljc
index e39867b8eec..98b89f4fb4c 100644
--- a/src/metabase/lib/walk.cljc
+++ b/src/metabase/lib/walk.cljc
@@ -394,7 +394,7 @@
(walk-clauses* f)))))
:lib.walk/stage
- (when (= (:lib/type stage-or-join) :mbql.stage/mbql)
+ (when (lib.util/mbql-stage? stage-or-join)
(reduce
(fn [stage k]
(m/update-existing stage k walk-clauses* f))
diff --git a/src/metabase/lib/walk/util.cljc b/src/metabase/lib/walk/util.cljc
index 6fbe3cd229a..cfe8452e452 100644
--- a/src/metabase/lib/walk/util.cljc
+++ b/src/metabase/lib/walk/util.cljc
@@ -9,45 +9,65 @@
[metabase.util.malli :as mu]))
(defn- transduce-stages
- ([query rf]
- (transduce-stages query (rf) rf))
+ ([rf query]
+ (transduce-stages rf (rf) query))
- ([query init rf]
+ ([rf init query]
(let [acc (volatile! init)]
(lib.walk/walk-stages
query
(fn [_query _path stage]
(vswap! acc rf stage)
nil))
- (rf @acc))))
+ (rf @acc)))
+
+ ([xform rf init query]
+ (transduce-stages (xform rf) init query)))
(defn- stage-values-set [query xform]
(not-empty
- (transduce-stages query (transient #{}) (-> conj!
- xform
- (completing persistent!)))))
+ (transduce-stages
+ xform
+ (completing conj! persistent!)
+ (transient #{})
+ query)))
(mu/defn all-source-table-ids :- [:maybe [:set {:min 1} ::lib.schema.id/table]]
"Return a set of all `:source-table` Table IDs referenced anywhere in the query."
[query :- ::lib.schema/query]
(stage-values-set query (keep :source-table)))
-(mu/defn all-source-card-ids :- [:maybe [:set {:min 1} ::lib.schema.id/card]]
- "Return a set of all `:source-card` Card IDs referenced anywhere in the query."
+(mu/defn all-template-tags-map :- [:maybe ::lib.schema.template-tag/template-tag-map]
+ "Return a combined template tags map for all native stages of a `query`."
[query :- ::lib.schema/query]
- (stage-values-set query (keep :source-card)))
+ (transduce-stages
+ (comp (filter (fn [stage]
+ (= (:lib/type stage) :mbql.stage/native)))
+ (mapcat :template-tags))
+ (fn rf
+ ([m]
+ (not-empty (persistent! m)))
+ ([m [template-tag-name template-tag]]
+ (assoc! m template-tag-name template-tag)))
+ (transient {})
+ query))
(mu/defn all-template-tags :- [:maybe [:set {:min 1} ::lib.schema.template-tag/template-tag]]
- "Return a map of Param IDs to sets of Field IDs referenced by each template tag parameter in this `card`.
+ "Return a set of all template tags in native stages of a `query`."
+ [query :- ::lib.schema/query]
+ (not-empty
+ (into #{}
+ (map (fn [[template-tag-name template-tag]]
+ (assoc template-tag :lib.walk/template-tag-name template-tag-name)))
+ (all-template-tags-map query))))
- Mostly used for determining Fields referenced by Cards for purposes other than processing queries. Filters out
- `:field` clauses which use names."
+(mu/defn all-source-card-ids :- [:maybe [:set {:min 1} ::lib.schema.id/card]]
+ "Return a set of all `:source-card` Card IDs anywhere in the query, as well as all `:card-id`s in template tags."
[query :- ::lib.schema/query]
- (stage-values-set query (comp (filter (fn [stage]
- (= (:lib/type stage) :mbql.stage/native)))
- (mapcat :template-tags)
- (map (fn [[template-tag-name template-tag]]
- (assoc template-tag :lib.walk/template-tag-name template-tag-name))))))
+ (not-empty
+ (into (set (stage-values-set query (keep :source-card)))
+ (keep :card-id)
+ (all-template-tags query))))
(mu/defn any-native-stage?
"Returns true if any stage of this query is native."
@@ -63,14 +83,54 @@
@has-native-stage?))
(mu/defn all-field-ids :- [:set ::lib.schema.id/field]
- "Set of all Field IDs referenced in `:field` refs in `query`."
- [query :- ::lib.schema/query]
- (let [field-ids (volatile! (transient #{}))]
- (lib.walk/walk-clauses
- query
- (fn [_query _path-type _stage-or-join-path clause]
- (lib.util.match/match-lite clause
- [:field _opts (id :guard pos-int?)]
- (vswap! field-ids conj! id))
- nil))
+ "Set of all Field IDs referenced in `:field` refs in a query or MBQL clause."
+ [query-or-clause :- [:or
+ ::lib.schema/query
+ vector?]]
+ (let [field-ids (volatile! (transient #{}))
+ walk-clause (fn [clause]
+ (lib.util.match/match-lite clause
+ [:field _opts (id :guard pos-int?)]
+ (vswap! field-ids conj! id))
+ nil)]
+ (if (map? query-or-clause)
+ (lib.walk/walk-clauses query-or-clause (fn [_query _path-type _stage-or-join-path clause]
+ (walk-clause clause)))
+ (lib.walk/walk-clause query-or-clause walk-clause))
(persistent! @field-ids)))
+
+(mu/defn all-template-tags-id->field-ids :- [:maybe
+ [:map-of
+ ::lib.schema.template-tag/id
+ [:set ::lib.schema.id/field]]]
+ "Return a map of
+
+ template-tag-id -> template-tag-field-ids-set
+
+ For all template tags in `query`."
+ [query :- ::lib.schema/query]
+ (not-empty
+ (into {}
+ (comp (filter :dimension)
+ (filter :id)
+ (map (fn [template-tag]
+ [(:id template-tag) (all-field-ids (:dimension template-tag))])))
+ (all-template-tags query))))
+
+(mu/defn all-template-tag-field-ids :- [:maybe [:set {:min 1} ::lib.schema.id/field]]
+ "Set of all `:field` IDs used in template tags."
+ [query :- ::lib.schema/query]
+ (not-empty
+ (into #{}
+ (comp (keep :dimension)
+ (mapcat all-field-ids))
+ (all-template-tags query))))
+
+(mu/defn all-template-tag-snippet-ids :- [:maybe [:set {:min 1} ::lib.schema.id/snippet]]
+ "Set of all Native Query Snippet IDs used in template tags."
+ [query :- ::lib.schema/query]
+ (not-empty
+ (into #{}
+ (comp (filter #(= (:type %) :snippet))
+ (keep :snippet-id))
+ (all-template-tags query))))
diff --git a/src/metabase/lib_be/core.clj b/src/metabase/lib_be/core.clj
index 70b39aabce2..f832ad06c77 100644
--- a/src/metabase/lib_be/core.clj
+++ b/src/metabase/lib_be/core.clj
@@ -1,11 +1,27 @@
(ns metabase.lib-be.core
(:require
+ [metabase.lib-be.metadata.bootstrap]
+ [metabase.lib-be.metadata.jvm]
+ [metabase.lib-be.models.transforms]
[metabase.lib-be.settings]
[potemkin :as p]))
-(comment metabase.lib-be.settings/keep-me)
+(comment
+ metabase.lib-be.metadata.bootstrap/keep-me
+ metabase.lib-be.metadata.jvm/keep-me
+ metabase.lib-be.models.transforms/keep-me
+ metabase.lib-be.settings/keep-me)
(p/import-vars
+ [metabase.lib-be.metadata.bootstrap
+ resolve-database]
+ [metabase.lib-be.metadata.jvm
+ application-database-metadata-provider
+ metadata-provider-cache
+ with-metadata-provider-cache]
+ [metabase.lib-be.models.transforms
+ normalize-query
+ transform-query]
[metabase.lib-be.settings
breakout-bin-width
breakout-bins-num
diff --git a/src/metabase/lib_be/metadata/bootstrap.clj b/src/metabase/lib_be/metadata/bootstrap.clj
new file mode 100644
index 00000000000..75007d46d54
--- /dev/null
+++ b/src/metabase/lib_be/metadata/bootstrap.clj
@@ -0,0 +1,114 @@
+(ns metabase.lib-be.metadata.bootstrap
+ (:require
+ [clojure.set :as set]
+ [metabase.lib.metadata.protocols :as lib.metadata.protocols]
+ [metabase.lib.schema.id :as lib.schema.id]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
+ [metabase.lib.util :as lib.util]
+ [metabase.util.i18n :as i18n]
+ [metabase.util.malli :as mu]
+ [toucan2.core :as t2]))
+
+(mu/defn- query-type :- [:enum :query :native :internal :mbql/query]
+ [query :- :map]
+ (or (some-> ((some-fn :lib/type :type) query) keyword)
+ (throw (ex-info (i18n/tru "Invalid query: missing or invalid query type (:lib/type or :type)")
+ {:query query, :type :invalid-query}))))
+
+(mu/defn- source-card-id-for-mbql5-query :- [:maybe ::lib.schema.id/card]
+ [query :- :map]
+ (-> query :stages first :source-card))
+
+(mu/defn- source-card-id-for-legacy-query :- [:maybe ::lib.schema.id/card]
+ [query :- :map]
+ (let [inner-query (:query query)
+ deepest-inner-query (loop [inner-query inner-query]
+ (let [source-query (:source-query inner-query)]
+ (if source-query
+ (recur source-query)
+ inner-query)))
+ source-table (:source-table deepest-inner-query)]
+ (lib.util/legacy-string-table-id->card-id source-table)))
+
+(defn- bootstrap-metadatas [{metadata-type :lib/type, id-set :id, :as _metadata-spec}]
+ (when (and (seq id-set)
+ (= metadata-type :metadata/card))
+ (t2/select-fn-vec
+ (fn [card]
+ {:lib/type :metadata/card
+ :id (:id card)
+ :name (format "Card #%d" (:id card))
+ :database-id (:database_id card)})
+ [:model/Card :id :database_id :card_schema]
+ :id [:in (set id-set)])))
+
+(deftype ^:private BootstrapMetadataProvider []
+ lib.metadata.protocols/MetadataProvider
+ (database [_this]
+ nil)
+ (metadatas [_this metadata-spec]
+ (bootstrap-metadatas metadata-spec))
+ (setting [_this _setting-key]
+ nil))
+
+(mu/defn- bootstrap-metadata-provider :- ::lib.schema.metadata/metadata-provider
+ "A super-basic metadata provider used only for resolving the database ID associated with a source Card, only for
+ queries that use the [[lib.schema.id/saved-questions-virtual-database-id]] e.g.
+
+ {:database -1337, :type :query, :query {:source-table \"card__1\"}}
+
+ Once the *actual* Database ID is resolved, we will create a
+ real [[metabase.lib-be.metadata.jvm/application-database-metadata-provider]]. (The App DB provider needs to be
+ initialized with an actual Database ID)."
+ []
+ (->BootstrapMetadataProvider))
+
+(mu/defn- resolve-database-id-for-source-card :- ::lib.schema.id/database
+ [source-card-id :- ::lib.schema.id/card]
+ (let [card (or (lib.metadata.protocols/card (bootstrap-metadata-provider) source-card-id)
+ (throw (ex-info (i18n/tru "Card {0} does not exist." source-card-id)
+ {:card-id source-card-id, :type :invalid-query, :status-code 404})))]
+ (:database-id card)))
+
+(mu/defn- source-card-id :- ::lib.schema.id/card
+ [query :- :map]
+ (case (query-type query)
+ :mbql/query
+ (source-card-id-for-mbql5-query query)
+
+ (:query :native)
+ (source-card-id-for-legacy-query query)
+
+ #_else
+ (throw (ex-info (i18n/tru "Invalid query: cannot use the Saved Questions Virtual Database ID unless query has a source Card")
+ {:query query, :type :invalid-query}))))
+
+(mu/defn- resolved-database-id :- [:maybe ::lib.schema.id/database]
+ [query :- :map]
+ (when-not (= (query-type query) :internal)
+ (let [database-id (:database query)]
+ (cond
+ (pos-int? database-id)
+ database-id
+
+ (= database-id lib.schema.id/saved-questions-virtual-database-id)
+ (resolve-database-id-for-source-card (source-card-id query))
+
+ :else
+ (throw (ex-info (i18n/tru "Invalid query: missing or invalid Database ID (:database)")
+ {:query query, :type :invalid-query}))))))
+
+(mu/defn resolve-database :- [:maybe
+ [:or
+ [:map
+ [:database ::lib.schema.id/database]]
+ [:map
+ [:type [:= :internal]]]]]
+ "If query has `:database` `-1337` (the legacy database ID for queries using a source Card that had an unknown
+ database), resolve the correct database ID and assoc it into the query."
+ [query :- [:maybe :map]]
+ (when (seq query)
+ (let [query (set/rename-keys query {"database" :database})
+ database-id (resolved-database-id query)]
+ (cond-> query
+ database-id (assoc :database database-id)))))
diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.clj
index a090a2f06fd..ade2af7fce6 100644
--- a/src/metabase/lib_be/metadata/jvm.clj
+++ b/src/metabase/lib_be/metadata/jvm.clj
@@ -14,6 +14,7 @@
[metabase.models.interface :as mi]
[metabase.settings.core :as setting]
[metabase.util :as u]
+ [metabase.util.json :as json]
[metabase.util.malli :as mu]
[metabase.util.memoize :as u.memo]
[metabase.util.performance :as perf]
@@ -497,6 +498,11 @@
This is useful for an API request, or group fo API requests like a dashboard load, to reduce appdb traffic."
nil)
+(defn metadata-provider-cache
+ "The currently bound [[*metadata-provider-cache*]], for Potemkin-export friendliness."
+ []
+ *metadata-provider-cache*)
+
(defmacro with-metadata-provider-cache
"Wrapper to create a [[*metadata-provider-cache*]] for the duration of the `body`.
@@ -518,3 +524,9 @@
(if-let [cache-atom *metadata-provider-cache*]
(cache.wrapped/lookup-or-miss cache-atom database-id application-database-metadata-provider-factory)
(application-database-metadata-provider-factory database-id)))
+
+;;; do not encode MetadataProviders to JSON, just generate `nil` instead.
+(json/add-encoder
+ UncachedApplicationDatabaseMetadataProvider
+ (fn [_mp json-generator]
+ (json/generate-nil nil json-generator)))
diff --git a/src/metabase/lib_be/models/transforms.clj b/src/metabase/lib_be/models/transforms.clj
new file mode 100644
index 00000000000..0b29813d0ef
--- /dev/null
+++ b/src/metabase/lib_be/models/transforms.clj
@@ -0,0 +1,27 @@
+(ns metabase.lib-be.models.transforms
+ (:require
+ [metabase.lib-be.metadata.bootstrap :as lib-be.bootstrap]
+ [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]
+ [metabase.lib.core :as lib]
+ [metabase.models.interface :as mi]
+ [metabase.util.malli :as mu]))
+
+(mu/defn normalize-query
+ "Normalize an MBQL `query` to MBQL 5 and attach a metadata provider."
+ [query :- [:maybe :map]]
+ (cond
+ (empty? query)
+ {}
+
+ (lib/cached-metadata-provider-with-cache? (:lib/metadata query))
+ query
+
+ :else
+ (->> query
+ lib-be.bootstrap/resolve-database
+ (lib/query lib.metadata.jvm/application-database-metadata-provider))))
+
+(def transform-query
+ "Toucan 2 transform spec for Card `dataset_query` and other columns that store MBQL."
+ {:in (comp mi/json-in lib/prepare-for-serialization normalize-query)
+ :out (comp (mi/catch-normalization-exceptions normalize-query) mi/json-out-without-keywordization)})
diff --git a/src/metabase/queries/metadata.clj b/src/metabase/queries/metadata.clj
index 5b018422ece..1c4f7379ff5 100644
--- a/src/metabase/queries/metadata.clj
+++ b/src/metabase/queries/metadata.clj
@@ -98,7 +98,10 @@
(defn- batch-fetch-query-metadata*
"Fetch dependent metadata for ad-hoc queries."
[queries]
- (let [source-ids (into #{} (mapcat #(lib.util/collect-source-tables (:query %)))
+ (let [source-ids (into #{}
+ ;; existing usage -- do not use in new code
+ #_{:clj-kondo/ignore [:deprecated-var]}
+ (mapcat #(lib.util/collect-source-tables (:query %)))
queries)
{source-table-ids :tables
source-card-ids :cards} (split-tables-and-legacy-card-refs source-ids)
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 0741c550aef..affcd0acf1e 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -224,6 +224,8 @@
"Return integer source table ids for card's :dataset_query."
[card]
(when-some [query (-> card :dataset_query :query)]
+ ;; existing usage -- do not use in new code
+ #_{:clj-kondo/ignore [:deprecated-var]}
(not-empty (filter pos-int? (lib.util/collect-source-tables query)))))
(defn- prefetch-tables-for-cards!
diff --git a/test/metabase/lib/schema/parameter_test.cljc b/test/metabase/lib/schema/parameter_test.cljc
index 368786583dc..d47746994f0 100644
--- a/test/metabase/lib/schema/parameter_test.cljc
+++ b/test/metabase/lib/schema/parameter_test.cljc
@@ -1,7 +1,7 @@
(ns metabase.lib.schema.parameter-test
(:require
#?@(:cljs ([metabase.test-runner.assert-exprs.approximately-equal]))
- [clojure.test :refer [are deftest is]]
+ [clojure.test :refer [are deftest is testing]]
[malli.error :as me]
[metabase.lib.core :as lib]
[metabase.lib.normalize :as lib.normalize]
@@ -46,3 +46,18 @@
(is (=? {:stages [{:parameters [{:target [:dimension [:expression "my_stringExpr" {:base-type :type/Text}] {:stage-number 0}]}]}]}
query))
(is (not (me/humanize (mr/explain ::lib.schema/query query))))))
+
+(deftest ^:parallel dimension-target-test
+ (testing "Failures"
+ (are [v] (not (mr/validate ::lib.schema.parameter/dimension.target v))
+ [:aggregation 0]
+ [:field "name" {}]))
+ (testing "Successes"
+ (are [v] (mr/validate ::lib.schema.parameter/dimension.target v)
+ [:field 3 nil])))
+
+(deftest ^:parallel normalize-mbql-3-refs-test
+ (are [clause expected] (= expected
+ (lib/normalize ::lib.schema.parameter/dimension clause))
+ [:dimension ["field-id" 76331]] [:dimension [:field 76331 nil]]
+ ["dimension" ["fk->" 23 30]] [:dimension [:field 30 {:source-field 23}]]))
diff --git a/test/metabase/lib/walk/util_test.cljc b/test/metabase/lib/walk/util_test.cljc
index decfdcd40d6..287cf69ead8 100644
--- a/test/metabase/lib/walk/util_test.cljc
+++ b/test/metabase/lib/walk/util_test.cljc
@@ -2,7 +2,9 @@
(:require
[clojure.test :refer [deftest is testing]]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-metadata :as meta]
+ [metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.macros :as lib.tu.macros]
[metabase.lib.walk.util :as lib.walk.util]
[metabase.util.malli :as mu]))
@@ -44,3 +46,44 @@
[:field {} 2]
[:field {} "wow"]]
:source-table 1}]})))))
+
+(deftest ^:parallel all-source-card-ids-legacy-native-query-template-tags-test
+ (let [query {:database 1
+ :type :native
+ :native {:query "SELECT *;"
+ :template-tags {"tag_1" {:type :card
+ :card-id 100}
+ "tag_2" {:type :card
+ :card-id 200}}}}]
+ (mu/disable-enforcement
+ (is (= #{100 200}
+ (lib/all-source-card-ids (lib/query meta/metadata-provider query)))))))
+
+(deftest ^:parallel all-source-card-ids-legacy-query-source-card-test
+ (let [query {:database 1
+ :type :query
+ :query {:source-query {:source-table "card__1000"}}}]
+ (is (= #{1000}
+ (lib/all-source-card-ids (lib/query meta/metadata-provider query))))))
+
+(deftest ^:parallel all-source-card-ids-pmbql-native-query-template-tags-test
+ (let [query (lib/query meta/metadata-provider {:lib/type :mbql.stage/native
+ :native "SELECT *;"
+ :template-tags {"tag_1" {:name "tag_1"
+ :display-name "Tag 1"
+ :type :card
+ :card-id 100}
+ "tag_2" {:name "tag_2"
+ :display-name "Tag 2"
+ :type :card
+ :card-id 200}}})]
+ (is (= #{100 200}
+ (lib/all-source-card-ids (lib/query meta/metadata-provider query))))))
+
+(deftest ^:parallel all-source-card-ids-source-card-test
+ (let [mp (lib.tu/metadata-provider-with-cards-for-queries
+ meta/metadata-provider
+ [(lib/query meta/metadata-provider (lib.metadata/table meta/metadata-provider (meta/id :venues)))])
+ query-with-source-card (lib/query mp (lib.metadata/card mp 1))]
+ (is (= #{1}
+ (lib/all-source-card-ids query-with-source-card)))))
diff --git a/test/metabase/lib_be/models/transforms_test.clj b/test/metabase/lib_be/models/transforms_test.clj
new file mode 100644
index 00000000000..3dccad305cf
--- /dev/null
+++ b/test/metabase/lib_be/models/transforms_test.clj
@@ -0,0 +1,29 @@
+(ns metabase.lib-be.models.transforms-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.util.json :as json]))
+
+(deftest ^:parallel handle-bad-template-tags-test
+ (testing (str "an malformed template tags map like the one below is invalid. Rather than potentially destroy an entire API "
+ "response because of one malformed Card, dump the error to the logs and return nil.")
+ (is (= nil
+ ((:out lib-be/transform-query)
+ (json/encode
+ {:database 1
+ :type :native
+ :native {:template-tags 1000}}))))))
+
+(deftest ^:parallel template-tag-validate-saves-test
+ (testing "on the other hand we should be a little more strict on the way and disallow you from saving the invalid stuff"
+ ;; TODO -- we should make sure this returns a good error message so we don't have to dig thru the exception chain.
+ (is (thrown?
+ Exception
+ ((:in lib-be/transform-query)
+ {:database 1
+ :type :native
+ :native {:template-tags {100 [:field-id "WOW"]}}})))))
+
+(deftest ^:parallel normalize-empty-query-test
+ (is (= {}
+ ((:out lib-be/transform-query) "{}"))))