Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

resolve-joins middleware => Lib, remove add-source-metadata (#63608)

Cam Sauldec95aabe733
diff --git a/enterprise/backend/src/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions.clj b/enterprise/backend/src/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions.clj
index 978a29c20e1..caddd68d99f 100644
--- a/enterprise/backend/src/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions.clj
+++ b/enterprise/backend/src/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions.clj
@@ -4,7 +4,6 @@
[metabase.api.common :as api]
[metabase.lib.core :as lib]
[metabase.lib.schema :as lib.schema]
- [metabase.lib.walk :as lib.walk]
[metabase.permissions.core :as perms]
[metabase.premium-features.core :refer [defenterprise-schema]]
[metabase.query-permissions.core :as query-perms]
@@ -29,23 +28,10 @@
[{database-id :database, :as _query}]
(perms/native-download-permission-for-user api/*current-user-id* database-id))
-;;; TODO (Cam 9/11/25) -- move this to `lib.walk.util` once my PR that adds it finally gets reviewed
-(mu/defn- any-native-stage?
- "Returns true if any stage of this query is native."
- [query :- ::lib.schema/query]
- (let [has-native-stage? (atom false)]
- (lib.walk/walk-stages
- query
- (fn [_query _path stage]
- (when (= (:lib/type stage) :mbql.stage/native)
- (reset! has-native-stage? true))
- nil))
- @has-native-stage?))
-
(mu/defmethod current-user-download-perms-level :mbql.stage/mbql
[{db-id :database, :as query} :- ::lib.schema/query]
(let [{:keys [table-ids native?]} (query-perms/query->source-ids query)
- perms (if (or native? (any-native-stage? query))
+ perms (if (or native? (lib/any-native-stage? query))
;; If we detect any native subqueries/joins, even with source-card IDs, require full native
;; download perms
#{(perms/native-download-permission-for-user api/*current-user-id* db-id)}
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index a2c6a63420c..4ee5fdb65ac 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -154,6 +154,7 @@
[lib.convert
->legacy-MBQL
->pMBQL
+ legacy-default-join-alias
without-cleaning]
[metabase.lib.convert.metadata-to-legacy
lib-metadata-column->legacy-metadata-column
@@ -444,6 +445,8 @@
source-table-id
update-query-stage]
[metabase.lib.walk.util
+ all-field-ids
all-source-card-ids
all-source-table-ids
- all-template-tags])
+ all-template-tags
+ any-native-stage?])
diff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljc
index d456892007e..69dc15f28bd 100644
--- a/src/metabase/lib/field/resolution.cljc
+++ b/src/metabase/lib/field/resolution.cljc
@@ -84,30 +84,30 @@
:lib/original-display-name (:display-name col))
(->> (add-parent-column-metadata metadata-providerable)))))
-(mu/defn- column-with-name :- [:maybe ::lib.schema.metadata/column]
- [columns :- [:sequential ::lib.schema.metadata/column]
- column-name :- :string]
- ;; look for a match with the same `desired-column-alias`; if that fails, look for a match with using
+(mu/defn- column-with-name-in-previous-stage-returned-columns :- [:maybe ::lib.schema.metadata/column]
+ [previous-stage-cols :- ::lib.metadata.calculation/returned-columns
+ column-name :- :string]
+ ;; look for a match with the same `desired-colum-alias`; if that fails, look for a match with using
;; legacy `deduplicated-name`
(some (fn [k]
(m/find-first #(= (k %) column-name)
- columns))
+ previous-stage-cols))
[:lib/desired-column-alias
:lib/deduplicated-name]))
-(mu/defn- resolve-in-previous-stage-metadata-without-updating-keys :- [:maybe ::lib.schema.metadata/column]
+(mu/defn- resolve-in-previous-stage-returned-columns-without-updating-keys :- [:maybe ::lib.metadata.calculation/returned-column]
"Find the matching column metadata in `cols` for `id-or-name`.
This metadata should be relative to the previous stage, or join, or whatever!!!! It should not be relative to the
current stage."
[metadata-providerable :- ::lib.schema.metadata/metadata-providerable
- cols :- [:sequential ::lib.schema.metadata/column]
+ previous-stage-cols :- [:maybe ::lib.metadata.calculation/returned-columns]
id-or-name :- ::id-or-name]
(letfn [(resolve* [id-or-name]
(if (string? id-or-name)
- (column-with-name cols id-or-name)
+ (column-with-name-in-previous-stage-returned-columns previous-stage-cols id-or-name)
;; `id-or-name` is an ID
- (or (m/find-first #(= (:id %) id-or-name) cols)
+ (or (m/find-first #(= (:id %) id-or-name) previous-stage-cols)
(do
(log/debugf "Failed to find column in metadata with ID %s" (pr-str id-or-name))
(when-some [field (lib.metadata/field metadata-providerable id-or-name)]
@@ -125,7 +125,7 @@
(log/debugf "Failed to find match for %s. Found:\n%s"
(pr-str id-or-name)
(u/pprint-to-str (map #(select-keys % [:id :lib/desired-column-alias :lib/deduplicated-name])
- cols)))))))
+ previous-stage-cols)))))))
(def ^:private opts-propagated-keys
"Keys to copy non-nil values directly from `:field` opts into column metadata."
@@ -252,6 +252,8 @@
:fingerprint
:visibility-type})
+(declare resolve-in-previous-stage-returned-columns-and-update-keys)
+
(mu/defn- additional-metadata-from-source-card :- [:maybe :map]
"Calculate additional metadata to include from a source model or Card for an already-resolved column."
[query :- ::lib.schema/query
@@ -266,29 +268,28 @@
;; if we have `id` then filter out anything that is definitely not a
;; match
(:id col) (filter #(= (:id %) (:id col)))))]
- (let [card-cols (for [col card-cols]
- (-> col
- lib.field.util/update-keys-for-col-from-previous-stage
- (assoc :lib/source :source/card, :lib/card-id card-id)))]
- ;; prefer resolution with `:lib/source-column-alias` over `:id` if we have it because it will be
- ;; unique/unambiguous if multiple versions of the column (e.g. with different bucketing units) are
- ;; returned
- (when-some [col (resolve-in-previous-stage-metadata-without-updating-keys query card-cols (:lib/source-column-alias col))]
- (let [propagated-keys (if (= (:type card) :model)
- model-propagated-keys
- regular-card-propagated-keys)]
- (select-keys col propagated-keys)))))))
+ ;; prefer resolution with `:lib/source-column-alias` over `:id` if we have it because it will be
+ ;; unique/unambiguous if multiple versions of the column (e.g. with different bucketing units) are
+ ;; returned
+ (when-some [col (resolve-in-previous-stage-returned-columns-and-update-keys query card-cols (:lib/source-column-alias col))]
+ (let [col (assoc col :lib/source :source/card, :lib/card-id card-id)
+ propagated-keys (if (= (:type card) :model)
+ model-propagated-keys
+ regular-card-propagated-keys)]
+ (select-keys col propagated-keys))))))
(:qp/stage-is-from-source-card stage)
(let [card-id (:qp/stage-is-from-source-card stage)]
{:lib/card-id card-id}))))
-(mu/defn- resolve-in-previous-stage-metadata-and-update-keys :- [:maybe ::lib.metadata.calculation/visible-column]
+(mu/defn- resolve-in-previous-stage-returned-columns-and-update-keys :- [:maybe ::lib.metadata.calculation/visible-column]
+ "Resolves by `:id` = `id-or-name`, or `:lib/desired-column-alias` in the previous stage returned columns (=
+ `:lib/source-column-alias` in the current stage) = `id-or-name`."
[query :- ::lib.schema/query
- previous-stage-columns :- [:sequential ::lib.schema.metadata/column]
+ previous-stage-columns :- [:maybe ::lib.metadata.calculation/returned-columns]
id-or-name :- [:or ::lib.schema.id/field :string]]
(log/tracef "Previous stage columns: %s" (pr-str (map (juxt :id :lib/desired-column-alias) previous-stage-columns)))
- (when-some [col (resolve-in-previous-stage-metadata-without-updating-keys query previous-stage-columns id-or-name)]
+ (when-some [col (resolve-in-previous-stage-returned-columns-without-updating-keys query previous-stage-columns id-or-name)]
(lib.field.util/update-keys-for-col-from-previous-stage col)))
(mu/defn- resolve-in-join :- [:maybe ::lib.metadata.calculation/visible-column]
@@ -308,14 +309,18 @@
(pr-str id-or-name)
(pr-str join-alias)
(pr-str stage-number))
- (let [join-cols (cond->> (lib.join/join-returned-columns-relative-to-parent-stage query stage-number join nil)
+ (let [join-cols (cond->> (lib.metadata.calculation/returned-columns query stage-number join)
source-field (remove (fn [col]
(when-some [col-source-field ((some-fn :fk-field-id :lib/original-fk-field-id) col)]
(not= col-source-field source-field)))))]
- (when-some [col (resolve-in-previous-stage-metadata-and-update-keys query join-cols id-or-name)]
- (-> col
- (as-> $col (lib.join/column-from-join query stage-number $col join-alias))
- (merge (select-keys join [:fk-field-id]))))))
+ (or (when-some [col (resolve-in-previous-stage-returned-columns-and-update-keys query join-cols id-or-name)]
+ (log/debugf "Got: %s" (pr-str col))
+ (-> col
+ (as-> $col (lib.join/column-from-join query stage-number $col join-alias))
+ (merge (select-keys join [:fk-field-id]))))
+ (do
+ (log/debugf "Failed to resolve %s in join" (pr-str id-or-name))
+ nil))))
;; a join with this alias does not exist at this stage of the query... try looking recursively in previous
;; stage(s)
(do
@@ -332,13 +337,13 @@
;; `Join__COLUMN`... if this fails then fall back to looking for matches that ignore join alias entirely
;; e.g. just `COLUMN`
(or (when (string? id-or-name)
- (resolve-in-previous-stage-metadata-and-update-keys query previous-stage-cols (str join-alias "__" id-or-name)))
- (resolve-in-previous-stage-metadata-and-update-keys query previous-stage-cols id-or-name)))
+ (resolve-in-previous-stage-returned-columns-and-update-keys query previous-stage-cols (str join-alias "__" id-or-name)))
+ (resolve-in-previous-stage-returned-columns-and-update-keys query previous-stage-cols id-or-name)))
(do
(log/debug "Unable to resolve in previous stage =(")
nil))))))
-(mu/defn- resolve-in-implicit-join-previous-stage
+(mu/defn- resolve-in-implicit-join-previous-stage :- [:maybe ::lib.metadata.calculation/visible-column]
"First, try to resolve the implicit join from the previous stage columns -- the join might have already been
performed there and `:source-field` was specified incorrectly. (You're only supposed to specify this in the stage
the implicit join happens; after that you should drop it and use field name refs instead e.g.
@@ -356,7 +361,7 @@
(let [previous-stage-cols (filter #(= ((some-fn :fk-field-id :lib/original-fk-field-id) %)
source-field-id)
(lib.metadata.calculation/returned-columns query previous-stage-number))]
- (resolve-in-previous-stage-metadata-and-update-keys query previous-stage-cols id-or-name))))
+ (resolve-in-previous-stage-returned-columns-and-update-keys query previous-stage-cols id-or-name))))
(mu/defn- find-reified-implicit-join-with-fk-field-id :- [:maybe [:tuple ::lib.schema.join/join :int]]
"Find the reified implicit join (i.e., a join added by
@@ -394,7 +399,7 @@
;; of think of a table as being the Ur-source of the entire query... either way this
;; actually still works since `returned-columns` for a table includes desired column
;; aliases.
- (resolve-in-previous-stage-metadata-without-updating-keys
+ (resolve-in-previous-stage-returned-columns-without-updating-keys
query
(lib.metadata.calculation/returned-columns query target-table)
field-name))))))
@@ -459,7 +464,7 @@
id-or-name :- ::id-or-name]
(log/debugf "Resolving %s in previous stage returned columns" (pr-str id-or-name))
(when-some [previous-stage-columns (lib.metadata.calculation/returned-columns query previous-stage-number)]
- (resolve-in-previous-stage-metadata-and-update-keys query previous-stage-columns id-or-name)))
+ (resolve-in-previous-stage-returned-columns-and-update-keys query previous-stage-columns id-or-name)))
(mu/defn- resolve-in-card-returned-columns :- [:maybe ::lib.metadata.calculation/visible-column]
[query :- ::lib.schema/query
@@ -468,7 +473,7 @@
(log/debugf "Resolving %s in source Card %s metadata" (pr-str id-or-name) (pr-str source-card-id))
(when-some [card (lib.metadata/card query source-card-id)]
(let [card-metadata-columns (lib.metadata.calculation/returned-columns query card)]
- (when-some [col (resolve-in-previous-stage-metadata-and-update-keys query card-metadata-columns id-or-name)]
+ (when-some [col (resolve-in-previous-stage-returned-columns-and-update-keys query card-metadata-columns id-or-name)]
(-> col
(assoc :lib/source :source/card, :lib/card-id source-card-id))))))
@@ -478,15 +483,24 @@
id-or-name :- ::id-or-name]
(log/debugf "Resolving %s in current stage metadata" (pr-str id-or-name))
(let [stage (lib.util/query-stage query stage-number)]
- (if-some [current-stage-metadata-columns (not-empty (get-in stage [:lib/stage-metadata :columns]))]
- ;; TODO (Cam 8/7/25) -- sort of a hacc to add deduplicated names here just so we can properly
- ;; use [[resolve-in-previous-stage-metadata-without-updating-keys]], but we don't have any sort of
- ;; `resolve-in-current-stage-metadata` function now. I guess it works tho
+ (when-some [current-stage-metadata-columns (or (not-empty (get-in stage [:lib/stage-metadata :columns]))
+ (do
+ (log/debug "stage has no attached metadata")
+ nil))]
(let [current-stage-metadata-columns (lib.field.util/add-deduplicated-names current-stage-metadata-columns)]
- (resolve-in-previous-stage-metadata-without-updating-keys query current-stage-metadata-columns id-or-name))
- (do
- (log/debug "stage has no attached metadata")
- nil))))
+ (u/prog1 (if (string? id-or-name)
+ (some (fn [k]
+ (m/find-first #(= (k %) id-or-name)
+ current-stage-metadata-columns))
+ [:lib/source-column-alias
+ :lib/deduplicated-name])
+ (m/find-first #(= (:id %) id-or-name) current-stage-metadata-columns))
+ (if <>
+ (log/debugf "Found match: %s" (pr-str (select-keys <> [:id :lib/source-column-alias :lib/deduplicated-name])))
+ (log/debugf "Failed to find match for %s. Found:\n%s"
+ (pr-str id-or-name)
+ (u/pprint-to-str (map #(select-keys % [:id :lib/source-column-alias :lib/deduplicated-name])
+ current-stage-metadata-columns)))))))))
(mu/defn- resolve-in-source-card-metadata :- [:maybe ::lib.metadata.calculation/visible-column]
[query :- ::lib.schema/query
diff --git a/src/metabase/lib/schema/join.cljc b/src/metabase/lib/schema/join.cljc
index c3b57b34e24..aff32f30fa4 100644
--- a/src/metabase/lib/schema/join.cljc
+++ b/src/metabase/lib/schema/join.cljc
@@ -113,9 +113,6 @@
[:map
{:default {}, :decode/normalize normalize-join}
[:lib/type [:= {:default :mbql/join, :decode/normalize common/normalize-keyword} :mbql/join]]
- ;; TODO (Cam 7/23/25) -- why would a join need an options map? If we need to add extra keys we can just add them
- ;; to the join itself.
- [:lib/options ::common/options]
[:stages [:ref :metabase.lib.schema/stages]]
[:conditions ::conditions]
[:alias ::alias]
diff --git a/src/metabase/lib/schema/util.cljc b/src/metabase/lib/schema/util.cljc
index 4c8ae6dbf87..e26170c806d 100644
--- a/src/metabase/lib/schema/util.cljc
+++ b/src/metabase/lib/schema/util.cljc
@@ -73,9 +73,11 @@
(defn- opts-distinct-key [opts]
;; Using reduce-kv to remove namespaced keys and some other keys to perform the comparison. This is allegedly faster.
- (reduce-kv (fn [acc k _v]
+ (reduce-kv (fn [acc k v]
(if (or (qualified-keyword? k)
- (#{:base-type :effective-type} k))
+ (#{:base-type :effective-type} k)
+ (and (#{:temporal-unit :inherited-temporal-unit} k)
+ (= v :default)))
(dissoc acc k)
acc))
opts
diff --git a/src/metabase/lib/util/match.clj b/src/metabase/lib/util/match.clj
index 74e2e7e6080..9883f5c542d 100644
--- a/src/metabase/lib/util/match.clj
+++ b/src/metabase/lib/util/match.clj
@@ -383,6 +383,9 @@
(defmacro match-lite
"Pattern matching macro, simplified version of [[clojure.core.match]].
+ TODO (Cam 9/16/25) -- what exactly is the difference between this and [[match]]? It doesn't recurse? Someone please
+ write an explanation here.
+
Usage:
(match-lite value
pattern1 result1
diff --git a/src/metabase/lib/walk/util.cljc b/src/metabase/lib/walk/util.cljc
index 2617662b0df..6fbe3cd229a 100644
--- a/src/metabase/lib/walk/util.cljc
+++ b/src/metabase/lib/walk/util.cljc
@@ -4,6 +4,7 @@
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.template-tag :as lib.schema.template-tag]
+ [metabase.lib.util.match :as lib.util.match]
[metabase.lib.walk :as lib.walk]
[metabase.util.malli :as mu]))
@@ -47,3 +48,29 @@
(mapcat :template-tags)
(map (fn [[template-tag-name template-tag]]
(assoc template-tag :lib.walk/template-tag-name template-tag-name))))))
+
+(mu/defn any-native-stage?
+ "Returns true if any stage of this query is native."
+ [query :- ::lib.schema/query]
+ (let [has-native-stage? (volatile! false)]
+ (lib.walk/walk-stages
+ query
+ (fn [_query _path stage]
+ (when (and (not @has-native-stage?)
+ (= (:lib/type stage) :mbql.stage/native))
+ (vreset! has-native-stage? true))
+ nil))
+ @has-native-stage?))
+
+(mu/defn all-field-ids :- [:set ::lib.schema.id/field]
+ "Set of all Field IDs referenced in `:field` refs in `query`."
+ [query :- ::lib.schema/query]
+ (let [field-ids (volatile! (transient #{}))]
+ (lib.walk/walk-clauses
+ query
+ (fn [_query _path-type _stage-or-join-path clause]
+ (lib.util.match/match-lite clause
+ [:field _opts (id :guard pos-int?)]
+ (vswap! field-ids conj! id))
+ nil))
+ (persistent! @field-ids)))
diff --git a/src/metabase/query_processor/compile.clj b/src/metabase/query_processor/compile.clj
index 42bb8dd0b78..7c69aec1c1b 100644
--- a/src/metabase/query_processor/compile.clj
+++ b/src/metabase/query_processor/compile.clj
@@ -5,7 +5,6 @@
[metabase.driver :as driver]
[metabase.lib.core :as lib]
[metabase.lib.schema :as lib.schema]
- [metabase.lib.walk :as lib.walk]
[metabase.query-processor.debug :as qp.debug]
[metabase.query-processor.error-type :as qp.error-type]
[metabase.query-processor.preprocess :as qp.preprocess]
@@ -46,19 +45,6 @@
[query]
(= (:lib/type (lib/query-stage query -1)) :mbql.stage/native))
-;;; TODO (Cam 9/11/25) -- move this to `lib.walk.util` once my PR that adds it finally gets reviewed
-(mu/defn- any-native-stage?
- "Returns true if any stage of this query is native."
- [query :- ::lib.schema/query]
- (let [has-native-stage? (atom false)]
- (lib.walk/walk-stages
- query
- (fn [_query _path stage]
- (when (= (:lib/type stage) :mbql.stage/native)
- (reset! has-native-stage? true))
- nil))
- @has-native-stage?))
-
(mu/defn- compile* :- ::compiled
[query :- ::lib.schema/query]
(assert (not (:qp/compiled query)) "This query has already been compiled!")
@@ -95,7 +81,7 @@
(assoc :qp/compiled (compile-preprocessed preprocessed))
;; if this query is pure-MBQL then we can reliably (re)compile it with inline parameters. If it has any native
;; stage then it might already have parameters, and we're not about to try to splice them back in.
- (cond-> (not (any-native-stage? preprocessed))
+ (cond-> (not (lib/any-native-stage? preprocessed))
(assoc :qp/compiled-inline (binding [driver/*compile-with-inline-parameters* true]
(compile-preprocessed preprocessed)))))))
diff --git a/src/metabase/query_processor/middleware/add_remaps.clj b/src/metabase/query_processor/middleware/add_remaps.clj
index 65fe9bd068a..5dc44874e1b 100644
--- a/src/metabase/query_processor/middleware/add_remaps.clj
+++ b/src/metabase/query_processor/middleware/add_remaps.clj
@@ -60,6 +60,7 @@
::lib.schema.id/field
:string]])
+;;; TODO (Cam 9/16/25) -- use [[metabase.lib.schema.util/mbql-clause-distinct-key]] for this
(mu/defn- simplify-ref-options :- ::simplified-ref
[a-ref :- ::lib.schema.ref/ref]
(lib/update-options a-ref (fn [opts]
@@ -309,16 +310,7 @@
(update :new-field-clause lib/with-join-alias (:alias join))))
new-fields (into
[]
- ;; TODO (Cam 7/25/25) the join fields may already include a remap, but `:source-field` or
- ;; other distinguishing information doesn't get propagated in refs beyond the stage where the
- ;; implicit join happens; thus we should ignore any duplicates with the same `:source-field`.
- ;; Joins with multiple remaps to the same table still work because we switch to using name
- ;; refs e.g. `NAME` and `NAME_2` instead of duplicate ID refs in this situation --
- ;; see [[multiple-fk-remaps-test-in-joins-e2e-test]].
- (m/distinct-by (fn [field-ref]
- (-> field-ref
- simplify-ref-options
- (lib/update-options dissoc :source-field))))
+ (m/distinct-by simplify-ref-options)
(add-fk-remaps-to-fields infos fields))]
(assoc join :fields new-fields))
;; there are no remaps to add, discard any changes that happen inside of the join (such as adding additional
diff --git a/src/metabase/query_processor/middleware/add_source_metadata.clj b/src/metabase/query_processor/middleware/add_source_metadata.clj
deleted file mode 100644
index 7741ab5bcfc..00000000000
--- a/src/metabase/query_processor/middleware/add_source_metadata.clj
+++ /dev/null
@@ -1,136 +0,0 @@
-(ns metabase.query-processor.middleware.add-source-metadata
- (:require
- [clojure.walk :as walk]
- ;; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use
- ;; Lib soon
- ^{:clj-kondo/ignore [:discouraged-namespace]}
- [metabase.legacy-mbql.schema :as mbql.s]
- ^{:clj-kondo/ignore [:discouraged-namespace]}
- [metabase.legacy-mbql.util :as mbql.u]
- [metabase.lib.metadata :as lib.metadata]
- [metabase.lib.util.match :as lib.util.match]
- [metabase.query-processor.interface :as qp.i]
- [metabase.query-processor.store :as qp.store]
- [metabase.request.core :as request]
- [metabase.util.log :as log]
- [metabase.util.malli :as mu]))
-
-(defn- has-same-fields-as-nested-source?
- "Whether this source query itself has a nested source query, and will have the exact same fields in the results as its
- nested source. If this is the case, we can return the `source-metadata` for the nested source as-is, if it is
- present."
- [{nested-source-query :source-query
- nested-source-metadata :source-metadata
- breakouts :breakout
- aggregations :aggregation
- fields :fields}]
- (when nested-source-query
- (and (every? empty? [breakouts aggregations])
- (or (empty? fields)
- (and (= (count fields) (count nested-source-metadata))
- (every? #(lib.util.match/match-one % [:field (_ :guard string?) _])
- fields))))))
-
-(mu/defn- native-source-query->metadata :- [:maybe [:sequential ::mbql.s/legacy-column-metadata]]
- "Given a `source-query`, return the source metadata that should be added at the parent level (i.e., at the same
- level where this `source-query` was present.) This metadata is used by other middleware to determine what Fields to
- expect from the source query."
- [{nested-source-metadata :source-metadata, :as source-query} :- mbql.s/SourceQuery]
- ;; If the source query has a nested source with metadata and does not change the fields that come back, return
- ;; metadata as-is
- (if (has-same-fields-as-nested-source? source-query)
- nested-source-metadata
- ;; Otherwise we cannot determine the metadata automatically; usually, this is because the source query itself has
- ;; a native source query
- (do
- (when-not qp.i/*disable-qp-logging*
- (log/warn "Cannot infer `:source-metadata` for source query with native source query without source metadata."
- {:source-query source-query}))
- nil)))
-
-(mu/defn mbql-source-query->metadata :- [:maybe [:sequential ::mbql.s/legacy-column-metadata]]
- "Preprocess a `source-query` so we can determine the result columns."
- [source-query :- mbql.s/MBQLQuery]
- (try
- (let [cols (request/as-admin
- ((requiring-resolve 'metabase.query-processor.preprocess/query->expected-cols)
- {:database (:id (lib.metadata/database (qp.store/metadata-provider)))
- :type :query
- ;; don't add remapped columns to the source metadata for the source query, otherwise we're going
- ;; to end up adding it again when the middleware runs at the top level
- :query (assoc-in source-query [:middleware :disable-remaps?] true)}))]
- (remove :remapped_from cols))
- (catch Throwable e
- (log/errorf e "Error determining expected columns for query: %s" (ex-message e))
- nil)))
-
-(mu/defn- add-source-metadata :- [:map
- [:source-metadata
- {:optional true}
- [:maybe [:sequential ::mbql.s/legacy-column-metadata]]]]
- [{{native-source-query? :native, :as source-query} :source-query, :as inner-query} :- :map]
- (let [metadata ((if native-source-query?
- native-source-query->metadata
- mbql-source-query->metadata) source-query)
- metadata (when metadata
- (let [unique-name-fn (mbql.u/unique-name-generator)]
- (for [col metadata
- :let [original-name ((some-fn :lib/original-name :name) col)]]
- (assoc col
- :lib/original-name original-name
- :lib/deduplicated-name (unique-name-fn original-name)))))]
- (cond-> inner-query
- (seq metadata) (assoc :source-metadata metadata))))
-
-(defn- legacy-source-metadata-without-field-ref?
- "Whether this source metadata is *legacy* source metadata from < 0.38.0. Legacy source metadata did not include
- `:field_ref` or `:id`, which made it hard to correctly construct queries with. For MBQL queries, we're better off
- ignoring legacy source metadata and using [[metabase.query-processor.preprocess/query->expected-cols]] to infer the
- source metadata rather than relying on old stuff that can produce incorrect queries. See #14788 for more
- information."
- [source-metadata]
- (and (seq source-metadata)
- (every? nil? (map :field_ref source-metadata))))
-
-(defn- should-add-source-metadata?
- "Should we add `:source-metadata` about the `:source-query` in this map? True if all of the following are true:
-
- * The map (e.g. an 'inner' MBQL query or a Join) has a `:source-query`
-
- * The map does not *already* have `:source-metadata`, or the `:source-metadata` is 'legacy' source metadata from
- versions < 0.38.0
-
- * The `:source-query` is an MBQL query, or a native source query with `:source-metadata`"
- [{{native-source-query? :native
- source-query-has-source-metadata? :source-metadata
- :as source-query} :source-query
- :keys [source-metadata]}]
- (let [source-metadata source-metadata]
- (and source-query
- (let [valid-source-metadata? (and source-metadata
- (not (legacy-source-metadata-without-field-ref? source-metadata)))]
- (not valid-source-metadata?))
- (or (not native-source-query?)
- source-query-has-source-metadata?
- (:qp/stage-is-from-source-card source-query)))))
-
-(defn- maybe-add-source-metadata [x]
- (if (and (map? x) (should-add-source-metadata? x))
- (add-source-metadata x)
- x))
-
-(defn- add-source-metadata-at-all-levels [inner-query]
- (walk/postwalk maybe-add-source-metadata inner-query))
-
-(mu/defn add-source-metadata-for-source-queries :- ::mbql.s/Query
- "Middleware that attempts to recursively add `:source-metadata`, if not already present, to any maps with a
- `:source-query`.
-
- `:source-metadata` is information about the columns we can expect to come back from the source
- query; this is added automatically for source queries added via the `card__id` source table form, but for *explicit*
- source queries that do not specify this information, we can often infer it by looking at the shape of the source
- query."
- [{query-type :type, :as query} :- ::mbql.s/Query]
- (if-not (= query-type :query)
- query
- (update query :query add-source-metadata-at-all-levels)))
diff --git a/src/metabase/query_processor/middleware/ensure_joins_use_source_query.clj b/src/metabase/query_processor/middleware/ensure_joins_use_source_query.clj
deleted file mode 100644
index 1dc9b9b62e8..00000000000
--- a/src/metabase/query_processor/middleware/ensure_joins_use_source_query.clj
+++ /dev/null
@@ -1,30 +0,0 @@
-(ns metabase.query-processor.middleware.ensure-joins-use-source-query
- (:require
- [metabase.lib.schema :as lib.schema]
- [metabase.lib.walk :as lib.walk]
- [metabase.util.malli :as mu]))
-
-(mu/defn ensure-joins-use-source-query :- ::lib.schema/query
- "Super simple middleware to ensure that joins keep
-
- {:source-query {:source-table ...}}
-
- instead of getting automatically collapsed back to
-
- {:source-table ...}
-
- when the query is converted to legacy. The main reason we want to do this is
- so [[metabase.query-processor.middleware.add-implicit-clauses/add-implicit-clauses]] will add `:fields` to the
- join's `:source-query` so that [[metabase.query-processor.middleware.add-remaps/add-remapped-columns]] can add
- remaps as appropriate.
-
- This is accomplished by adding a dummy key to the last stage of the join; [[metabase.lib.convert/->legacy-MBQL]] will
- not collapse the last stage into the top level of the join if extra keys are present.
-
- Once we convert the `add-implicit-clauses` middleware to use Lib we can remove this middleware entirely."
- [query :- ::lib.schema/query]
- (lib.walk/walk
- query
- (fn [_query path-type _path join]
- (when (= path-type :lib.walk/join)
- (assoc-in join [:stages (dec (count (:stages join))) ::dummy-key] true)))))
diff --git a/src/metabase/query_processor/middleware/resolve_fields.clj b/src/metabase/query_processor/middleware/resolve_fields.clj
index 25470be9c30..8629f5762f3 100644
--- a/src/metabase/query_processor/middleware/resolve_fields.clj
+++ b/src/metabase/query_processor/middleware/resolve_fields.clj
@@ -1,14 +1,11 @@
(ns metabase.query-processor.middleware.resolve-fields
"Middleware that resolves the Fields referenced by a query."
(:require
+ [metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
- [metabase.lib.schema :as lib.schema]
- [metabase.lib.schema.id :as lib.schema.id]
- [metabase.lib.util.match :as lib.util.match]
[metabase.query-processor.error-type :as qp.error-type]
[metabase.util :as u]
- [metabase.util.i18n :refer [tru]]
- [metabase.util.malli :as mu]))
+ [metabase.util.i18n :refer [tru]]))
(defn- resolve-fields-with-ids!
[metadata-providerable field-ids]
@@ -21,23 +18,12 @@
field-ids))]
(recur metadata-providerable parent-ids)))
-(mu/defn- field-ids :- [:set ::lib.schema.id/field]
- [query :- ::lib.schema/query]
- (into #{}
- (lib.util.match/match (:stages query)
- [:field _opts (id :guard pos-int?)]
- id
-
- ;; stage metadata
- {:lib/type :metadata/column, :id (id :guard pos-int?)}
- id)))
-
;;; TODO (Cam 9/10/25) -- give this a more accurate name like `prefetch-all-fields` or
;;; `warm-metadata-provider-cache-with-all-fields` or something.
(defn resolve-fields
"Resolve all field referenced in the `query`, and store them in the Metadata Provider."
[query]
- (let [ids (field-ids query)]
+ (let [ids (lib/all-field-ids query)]
(try
(u/prog1 query
(resolve-fields-with-ids! query ids))
diff --git a/src/metabase/query_processor/middleware/resolve_joins.clj b/src/metabase/query_processor/middleware/resolve_joins.clj
index 38e9d53a3e5..651c60849aa 100644
--- a/src/metabase/query_processor/middleware/resolve_joins.clj
+++ b/src/metabase/query_processor/middleware/resolve_joins.clj
@@ -3,148 +3,81 @@
options, and adds information to the query about what joins should be done and how they should be performed."
(:refer-clojure :exclude [alias])
(:require
+ [clojure.string :as str]
[medley.core :as m]
- ;; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use
- ;; Lib soon
- ^{:clj-kondo/ignore [:discouraged-namespace]}
- [metabase.legacy-mbql.schema :as mbql.s]
- [metabase.lib.metadata :as lib.metadata]
- [metabase.lib.schema.id :as lib.schema.id]
- [metabase.lib.util :as lib.util]
- [metabase.lib.util.match :as lib.util.match]
- [metabase.query-processor.error-type :as qp.error-type]
- [metabase.query-processor.store :as qp.store]
- [metabase.util :as u]
- [metabase.util.i18n :refer [tru]]
+ [metabase.lib.core :as lib]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.lib.schema.join :as lib.schema.join]
+ [metabase.lib.schema.util :as lib.schema.util]
+ [metabase.lib.walk :as lib.walk]
[metabase.util.malli :as mu]))
-(def ^:private Joins
- "Schema for a non-empty sequence of Joins. Unlike [[mbql.s/Joins]], this does not enforce the constraint that all join
- aliases be unique."
- [:sequential {:min 1} mbql.s/Join])
-
-(def ^:private UnresolvedMBQLQuery
- "Schema for the parts of the query we're modifying. For use in the various intermediate transformations in the
- middleware."
- [:map
- [:joins [:sequential mbql.s/Join]]
- [:fields {:optional true} mbql.s/Fields]])
-
-(def ^:private ResolvedMBQLQuery
- "Schema for the final results of this middleware."
- [:and
- UnresolvedMBQLQuery
- [:fn
- {:error/message "Valid MBQL query where `:joins` `:fields` is sequence of Fields or removed"}
- (fn [{:keys [joins]}]
- (every?
- (fn [{:keys [fields]}]
- (or
- (empty? fields)
- (sequential? fields)))
- joins))]])
-
-;;; +----------------------------------------------------------------------------------------------------------------+
-;;; | Resolving Tables & Fields / Saving in QP Store |
-;;; +----------------------------------------------------------------------------------------------------------------+
-
-(mu/defn- resolve-fields! :- :nil
- [joins :- Joins]
- (lib.metadata/bulk-metadata-or-throw (qp.store/metadata-provider)
- :metadata/column
- (lib.util.match/match joins [:field (id :guard integer?) _] id))
- nil)
-
-(mu/defn- resolve-tables! :- :nil
- "Add Tables referenced by `:joins` to the Query Processor Store. This is only really needed for implicit joins,
- because their Table references are added after `resolve-source-tables` runs."
- [joins :- Joins]
- (lib.metadata/bulk-metadata-or-throw (qp.store/metadata-provider)
- :metadata/table
- (remove nil? (map :source-table joins)))
- nil)
-
-;;; +----------------------------------------------------------------------------------------------------------------+
-;;; | :Joins Transformations |
-;;; +----------------------------------------------------------------------------------------------------------------+
-
-(def ^:private default-join-alias "__join")
-
-(mu/defn- merge-defaults :- mbql.s/Join
+(mu/defn- merge-defaults :- ::lib.schema.join/join
[join]
- (merge {:alias default-join-alias, :strategy :left-join} join))
-
-(defn- source-metadata->fields [{:keys [alias], :as join} source-metadata]
- (when-not (seq source-metadata)
- (throw (ex-info (tru "Cannot use :fields :all in join against source query unless it has :source-metadata.")
- {:join join})))
+ ;;; TODO (Cam 9/16/25) -- if we made `:strategy` a required key with a `:default` value in
+ ;;; `:metabase.lib.schema.join/join` then Lib normalization could handle this for us and we wouldn't need QP
+ ;;; middleware to do it. Seems like the better way to deal with defaults. (QUE-2469)
+ (merge {:strategy :left-join}
+ (when (str/starts-with? (:alias join) lib/legacy-default-join-alias)
+ {:qp/keep-default-join-alias true})
+ join))
+
+(defn- join-field-refs [cols]
(let [duplicate-ids (into #{}
(keep (fn [[item freq]]
(when (> freq 1)
item)))
- (frequencies (map :id source-metadata)))]
- (for [{field-name :name
- base-type :base_type
- field-id :id
- field-ref :field_ref} source-metadata]
- ;; If `field-ref` is an id-based reference, only use it if the source query uses it.
- (or (when (and field-id (not (contains? duplicate-ids field-id)))
- (lib.util.match/match-one field-ref
- [:field (id :guard pos-int?) _opts]
- [:field field-id {:join-alias alias}]))
- [:field field-name {:base-type base-type, :join-alias alias}]))))
-
-(mu/defn- sorted-implicit-fields-for-table :- mbql.s/Fields
- "For use when adding implicit Field IDs to a query. Return a sequence of field clauses, sorted by the rules listed
- in [[metabase.query-processor.sort]], for all the Fields in a given Table."
- [table-id :- ::lib.schema.id/table]
- (let [fields (lib.metadata/active-fields (qp.store/metadata-provider) table-id)]
- (when (empty? fields)
- (throw (ex-info (tru "No fields found for table {0}." (pr-str (:name (lib.metadata/table (qp.store/metadata-provider) table-id))))
- {:table-id table-id
- :type qp.error-type/invalid-query})))
- (mapv
- (fn [field]
- [:field (u/the-id field) nil])
- fields)))
-
-(mu/defn- handle-all-fields :- mbql.s/Join
+ (frequencies (map :id cols)))]
+ ;; TODO (Cam 9/16/25) -- forcing refs of certain types like this is wonky, but when I try to change this tons of
+ ;; stuff breaks. Forcing ID refs doesn't work because a join can return multiple versions of the same column
+ ;; bucketed in different ways in previous stages; joins thus ought to be using field name refs; but this ends up
+ ;; breaking a ton of stuff, especially `lib.equality`... #63109 was my attempt to make this stuff work when using
+ ;; field name refs for joins but it's a long way off from landing.
+ ;;
+ ;; NOCOMMIT
+ (for [{field-id :id, :as col} cols
+ :let [[_tag opts id-or-name, :as field-ref] (lib/ref col)
+ force-id-ref? (and (string? id-or-name)
+ field-id
+ (not (contains? duplicate-ids field-id)))
+ force-field-name-ref? (and (pos-int? id-or-name)
+ (contains? duplicate-ids field-id))]]
+
+ (cond
+ force-id-ref?
+ [:field opts field-id]
+
+ force-field-name-ref?
+ [:field opts (:lib/source-column-alias col)]
+
+ :else
+ field-ref))))
+
+(mu/defn- handle-all-fields :- ::lib.schema.join/join
"Replace `:fields :all` in a join with an appropriate list of Fields."
- [{:keys [source-table source-query alias fields source-metadata], :as join} :- mbql.s/Join]
+ [query :- ::lib.schema/query
+ path :- ::lib.walk/path
+ {:keys [fields], :as join} :- ::lib.schema.join/join]
(merge
join
(when (= fields :all)
- {:fields (if source-query
- (source-metadata->fields join source-metadata)
- (for [[_ id-or-name opts] (sorted-implicit-fields-for-table source-table)]
- [:field id-or-name (assoc opts :join-alias alias)]))})))
-
-(defn- deduplicate-aliases []
- (let [unique-name-generator (lib.util/non-truncating-unique-name-generator)]
- (map (fn [join]
- (update join :alias unique-name-generator)))))
-
-(mu/defn- resolve-references :- Joins
- [joins :- Joins]
- (resolve-tables! joins)
- (u/prog1 (into []
- (comp (map merge-defaults)
- (map handle-all-fields)
- (deduplicate-aliases))
- joins)
- (resolve-fields! <>)))
-
-(declare resolve-joins-in-mbql-query-all-levels)
-
-(mu/defn- resolve-join-source-queries :- Joins
- [joins :- Joins]
- (for [{:keys [source-query], :as join} joins]
- (cond-> join
- source-query resolve-joins-in-mbql-query-all-levels)))
-
-;;; +----------------------------------------------------------------------------------------------------------------+
-;;; | MBQL-Query Transformations |
-;;; +----------------------------------------------------------------------------------------------------------------+
+ ;; do not `:include-remaps?` here, they will get added by the [[metabase.query-processor.middleware.add-remaps]]
+ ;; middleware.
+ {:fields (join-field-refs
+ (lib.walk/apply-f-for-stage-at-path
+ lib/join-fields-to-add-to-parent-stage
+ query
+ path
+ join
+ {:include-remaps? false}))})))
+
+(mu/defn- resolve-join :- ::lib.schema.join/join
+ [query :- ::lib.schema/query
+ path :- ::lib.walk/path
+ join :- ::lib.schema.join/join]
+ (->> join
+ merge-defaults
+ (handle-all-fields query path)))
(defn- joins->fields
"Return a flattened list of all `:fields` referenced in `joins`."
@@ -155,89 +88,64 @@
;; make sure the field ref has `:join-alias`... it already SHOULD but if the query is NAUGHTY then
;; we better just add it in to be safe. In #61398 which is pending I actually make this happen
;; automatically in MBQL 5 normalization, so we can take this out eventually.
- (for [[tag id-or-name opts] fields]
- [tag id-or-name (assoc opts
- :join-alias (:alias join)
- ;; Any coercion or temporal bucketing will already have been done in the
- ;; subquery for the join itself. Mark the parent ref to make sure it is
- ;; not double-coerced, which leads to SQL errors.
- :qp/ignore-coercion true)]))))
+ (for [field-ref fields]
+ (-> field-ref
+ (lib/with-join-alias (:alias join))
+ ;; Any coercion or temporal bucketing will already have been done in the
+ ;; subquery for the join itself. Mark the parent ref to make sure it is
+ ;; not double-coerced, which leads to SQL errors.
+ (lib/update-options assoc :qp/ignore-coercion true))))))
joins))
(defn- should-add-join-fields?
"Should we append the `:fields` from `:joins` to the parent-level query's `:fields`? True unless the parent-level
query has breakouts or aggregations."
- [{breakouts :breakout, aggregations :aggregation}]
+ [{breakouts :breakout, aggregations :aggregation, :as _stage}]
(every? empty? [aggregations breakouts]))
+;;; TODO (Cam 9/16/25) -- update this to use [[metabase.lib.stage/add-cols-from-join]] or share more logic with it
(defn- append-join-fields
- "This (supposedly) matches the behavior of [[metabase.lib.stage/add-cols-from-join]]. When we migrate this namespace
- to Lib we can maybe use that."
+ "This (supposedly) matches the behavior of [[metabase.lib.stage/add-cols-from-join]]."
[fields join-fields]
;; we shouldn't consider different type info to mean two Fields are different even if everything else is the same. So
;; give everything `:base-type` of `:type/*` (it will complain if we remove `:base-type` entirely from fields with a
;; string name)
- (letfn [(opts-signature [opts]
- (not-empty
- (merge
- (u/select-non-nil-keys opts [:join-alias :binning])
- ;; for purposes of deduplicating stuff, temporal unit = default is the same as not specifying temporal
- ;; unit at all. Should that be part of normalization? Maybe, but there is some logic around adding default
- ;; temporal bucketing that we don't do if `:default` is explicitly specified.
- (when-let [temporal-unit (:temporal-unit opts)]
- (when-not (= temporal-unit :default)
- {:temporal-unit temporal-unit})))))
- (ref-signature [[tag id-or-name opts, :as _ref]]
- [tag id-or-name (opts-signature opts)])]
- (into []
- (comp cat
- (m/distinct-by ref-signature))
- [fields join-fields])))
+ (into []
+ (comp cat
+ (m/distinct-by lib.schema.util/mbql-clause-distinct-key))
+ [fields (lib/fresh-uuids join-fields)]))
(defn append-join-fields-to-fields
"Add the fields from join `:fields`, if any, to the parent-level `:fields`."
- [inner-query join-fields]
- (cond-> inner-query
+ [stage join-fields]
+ (cond-> stage
(seq join-fields) (update :fields append-join-fields join-fields)))
-(mu/defn- merge-joins-fields :- UnresolvedMBQLQuery
+(mu/defn- merge-joins-fields :- ::lib.schema/stage.mbql
"Append the `:fields` from `:joins` into their parent level as appropriate so joined columns appear in the final
query results, and remove the `:fields` entry for all joins.
If the parent-level query has breakouts and/or aggregations, this function won't append the joins fields to the
parent level, because we should only be returning the ones from the ags and breakouts in the final results."
- [{:keys [joins], :as inner-query} :- UnresolvedMBQLQuery]
- (let [join-fields (when (should-add-join-fields? inner-query)
+ [{:keys [joins], :as stage} :- ::lib.schema/stage.mbql]
+ (let [join-fields (when (should-add-join-fields? stage)
(joins->fields joins))
- ;; remove remaining keyword `:fields` like `:none` from joins
- inner-query (update inner-query :joins (fn [joins]
- (mapv (fn [{:keys [fields], :as join}]
- (cond-> join
- (keyword? fields) (dissoc :fields)))
- joins)))]
- (append-join-fields-to-fields inner-query join-fields)))
-
-(mu/defn- resolve-joins-in-mbql-query :- ResolvedMBQLQuery
- [query :- mbql.s/MBQLQuery]
+ stage (update stage :joins (fn [joins]
+ (mapv (fn [{:keys [fields], :as join}]
+ (cond-> join
+ (keyword? fields) (dissoc :fields)))
+ joins)))]
+ (append-join-fields-to-fields stage join-fields)))
+
+(mu/defn resolve-joins :- ::lib.schema/query
+ "1. Walk joins and merge defaults like `:strategy :left-join`
+ 2. Walk joins and resolve `:fields :all` to a vector of field refs
+ 3. Walk stages and merge in `:fields` from `:joins`"
+ [query :- ::lib.schema/query]
(-> query
- (update :joins (comp resolve-join-source-queries resolve-references))
- merge-joins-fields))
-
-;;; +----------------------------------------------------------------------------------------------------------------+
-;;; | Middleware & Boring Recursive Application Stuff |
-;;; +----------------------------------------------------------------------------------------------------------------+
-
-(defn- resolve-joins-in-mbql-query-all-levels
- [{:keys [joins source-query], :as query}]
- (cond-> query
- (seq joins) resolve-joins-in-mbql-query
- source-query (update :source-query resolve-joins-in-mbql-query-all-levels)))
-
-;; TODO (Cam 9/10/25) -- once we convert this to Lib we can remove
-;; the [[metabase.query-processor.middleware.ensure-joins-use-source-query/ensure-joins-use-source-query]] middleware
-;; entirely
-(defn resolve-joins
- "Add any Tables and Fields referenced by the `:joins` clause to the QP store."
- [{inner-query :query, :as outer-query}]
- (cond-> outer-query
- inner-query (update :query resolve-joins-in-mbql-query-all-levels)))
+ (lib.walk/walk (fn [query path-type path join]
+ (when (= path-type :lib.walk/join)
+ (resolve-join query path join))))
+ (lib.walk/walk-stages (fn [__query _path stage]
+ (when (seq (:joins stage))
+ (merge-joins-fields stage))))))
diff --git a/src/metabase/query_processor/middleware/resolve_source_table.clj b/src/metabase/query_processor/middleware/resolve_source_table.clj
index 906d16c930b..7c3fbdab2ab 100644
--- a/src/metabase/query_processor/middleware/resolve_source_table.clj
+++ b/src/metabase/query_processor/middleware/resolve_source_table.clj
@@ -1,28 +1,13 @@
(ns metabase.query-processor.middleware.resolve-source-table
"Fetches Tables corresponding to any `:source-table` IDs anywhere in the query."
(:require
- [metabase.lib.metadata :as lib.metadata]
- [metabase.lib.schema.id :as lib.schema.id]
- [metabase.lib.walk :as lib.walk]
- [metabase.util.malli :as mu]))
-
-;;; TODO (Cam 9/11/25) -- update this to use the `lib.walk.util` functions once #63321 is merged
-(mu/defn ^:private query->source-table-ids :- [:maybe [:set {:min 1} ::lib.schema.id/table]]
- "Fetch a set of all `:source-table` IDs anywhere in `query`."
- [query]
- (let [source-table-ids (volatile! #{})]
- (lib.walk/walk-stages
- query
- (fn [_query _path {:keys [source-table], :as _stage}]
- (when source-table
- (vswap! source-table-ids conj source-table))
- nil))
- (not-empty @source-table-ids)))
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]))
(defn resolve-source-tables
"Middleware that will take any `:source-table`s (integer IDs) anywhere in the query and fetch and save the
corresponding Table in the Query Processor Store."
[query]
;; this is done for side effects
- (lib.metadata/bulk-metadata-or-throw query :metadata/table (query->source-table-ids query))
+ (lib.metadata/bulk-metadata-or-throw query :metadata/table (lib/all-source-table-ids query))
query)
diff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.clj
index 90ea02d8d9f..307123a106f 100644
--- a/src/metabase/query_processor/preprocess.clj
+++ b/src/metabase/query_processor/preprocess.clj
@@ -15,7 +15,6 @@
[metabase.query-processor.middleware.add-implicit-clauses :as qp.add-implicit-clauses]
[metabase.query-processor.middleware.add-implicit-joins :as qp.add-implicit-joins]
[metabase.query-processor.middleware.add-remaps :as qp.add-remaps]
- [metabase.query-processor.middleware.add-source-metadata :as qp.add-source-metadata]
[metabase.query-processor.middleware.annotate :as annotate]
[metabase.query-processor.middleware.auto-bucket-datetimes :as qp.auto-bucket-datetimes]
[metabase.query-processor.middleware.auto-parse-filter-values :as auto-parse-filter-values]
@@ -25,7 +24,6 @@
[metabase.query-processor.middleware.cumulative-aggregations :as qp.cumulative-aggregations]
[metabase.query-processor.middleware.desugar :as desugar]
[metabase.query-processor.middleware.drop-fields-in-summaries :as drop-fields-in-summaries]
- [metabase.query-processor.middleware.ensure-joins-use-source-query :as ensure-joins-use-source-query]
[metabase.query-processor.middleware.enterprise :as qp.middleware.enterprise]
[metabase.query-processor.middleware.expand-aggregations :as expand-aggregations]
[metabase.query-processor.middleware.expand-macros :as expand-macros]
@@ -110,11 +108,7 @@
(ensure-mbql5 #'parameters/substitute-parameters)
(ensure-mbql5 #'qp.resolve-source-table/resolve-source-tables)
(ensure-mbql5 #'qp.auto-bucket-datetimes/auto-bucket-datetimes)
- ;; TODO (Cam 9/10/25) -- once we convert the [[metabase.query-processor.middleware.resolve-joins/resolve-joins]]
- ;; middleware to Lib we can remove this middleware entirely
- (ensure-mbql5 #'ensure-joins-use-source-query/ensure-joins-use-source-query)
(ensure-mbql5 #'reconcile-bucketing/reconcile-breakout-and-order-by-bucketing)
- (ensure-legacy #'qp.add-source-metadata/add-source-metadata-for-source-queries)
(ensure-mbql5 #'qp.middleware.enterprise/apply-impersonation)
(ensure-mbql5 #'qp.middleware.enterprise/attach-destination-db-middleware)
(ensure-mbql5 #'qp.middleware.enterprise/apply-sandboxing)
@@ -123,14 +117,14 @@
;; this needs to be done twice, once before adding remaps (since we want to add remaps inside joins) and then again
;; after adding any implicit joins. Implicit joins do not need to get remaps since we only use them for fetching
;; specific columns.
- (ensure-legacy #'resolve-joins/resolve-joins)
+ (ensure-mbql5 #'resolve-joins/resolve-joins)
(ensure-mbql5 #'qp.add-remaps/add-remapped-columns)
#'qp.resolve-fields/resolve-fields ; this middleware actually works with either MBQL 5 or legacy
(ensure-mbql5 #'binning/update-binning-strategy)
(ensure-mbql5 #'desugar/desugar)
(ensure-mbql5 #'qp.add-default-temporal-unit/add-default-temporal-unit)
(ensure-mbql5 #'qp.add-implicit-joins/add-implicit-joins)
- (ensure-legacy #'resolve-joins/resolve-joins) ; #61398
+ (ensure-mbql5 #'resolve-joins/resolve-joins)
(ensure-mbql5 #'fix-bad-field-id-refs/fix-bad-field-id-refs)
(ensure-mbql5 #'qp.remove-inactive-field-refs/remove-inactive-field-refs)
;; yes, this is called a second time, because we need to handle any joins that got added
diff --git a/test/metabase/lib/field/resolution_test.cljc b/test/metabase/lib/field/resolution_test.cljc
index 337f01e33ef..7a097313216 100644
--- a/test/metabase/lib/field/resolution_test.cljc
+++ b/test/metabase/lib/field/resolution_test.cljc
@@ -34,11 +34,12 @@
(doseq [f [#'lib/metadata
#'lib.field.resolution/resolve-field-ref]]
(testing f
- (is (=? {:name "sum"
- :display-name "sum of User ID"
- :base-type :type/Integer
- :semantic-type :type/FK
- :lib/source :source/native}
+ (is (=? {:name "sum"
+ :display-name "sum of User ID"
+ :base-type :type/Integer
+ :semantic-type :type/FK
+ :lib/source :source/native
+ ::lib.field.resolution/fallback-metadata? (symbol "nil #_\"key is not present.\"")}
(lib/metadata
(lib.tu/native-query)
-1
@@ -1447,14 +1448,14 @@
{:stages [{:joins [{:alias "J"
:stages [{:source-table (meta/id :venues)
:joins [{:alias "CATEGORIES__via__ID"
- :fk-field-id (meta/id :venues :category-id)
+ :fk-field-id (meta/id :venues :id)
:stages [{:source-table (meta/id :categories)
:fields [[:field {} (meta/id :categories :id)]
[:field {} (meta/id :categories :name)]]}]
:conditions [[:= {} 1 1]]
:fields :none}
{:alias "CATEGORIES__via__CATEGORY_ID"
- :fk-field-id (meta/id :venues :id)
+ :fk-field-id (meta/id :venues :category-id)
:stages [{:source-table (meta/id :categories)
:fields [[:field {} (meta/id :categories :id)]
[:field {} (meta/id :categories :name)]]}]
@@ -1508,3 +1509,66 @@
:lib/source-column-alias "CATEGORIES__via__CATEGORY_ID__NAME"
:metabase.lib.join/join-alias "J"}
(lib.field.resolution/resolve-field-ref query 0 field-ref)))))))))
+
+(deftest ^:parallel resolve-implicit-column-test
+ (testing "fields implicitly joined in a previous stage"
+ (let [query (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query venues
+ {:fields [$category-id->categories.name]
+ :order-by [[:asc $id]]}))
+ lib/append-stage)]
+ (is (=? {:id (meta/id :categories :name)
+ :table-id (meta/id :categories)
+ :lib/original-fk-field-id (meta/id :venues :category-id)
+ ::lib.field.resolution/fallback-metadata? (symbol "nil #_\"key is not present.\"")}
+ (lib.field.resolution/resolve-field-ref query -1 [:field
+ {:base-type :type/Text, :effective-type :type/Text, :lib/uuid "ee91656b-08c7-4ca2-9f9e-5845e2edd80a"}
+ "CATEGORIES__via__CATEGORY_ID__NAME"]))
+ ":fk-field-id needs to get propagated as :lib/previous-stage-fk-field-id this to work correctly"))))
+
+(deftest ^:parallel resolve-by-name-from-join-test
+ (let [query (lib/query
+ meta/metadata-provider
+ {:lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table (meta/id :people)
+ :joins [{:lib/type :mbql/join
+ :strategy :left-join
+ :alias "Q1"
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table (meta/id :orders)
+ :fields [[:field {} (meta/id :orders :id)]
+ [:field {:base-type :type/Integer, :join-alias "O"} "ID"]]
+ :joins [{:lib/type :mbql/join
+ :strategy :left-join
+ :alias "O"
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table (meta/id :orders)
+ :fields [[:field {} (meta/id :orders :id)]]}]
+ :conditions [[:=
+ {}
+ [:field {} (meta/id :orders :id)]
+ [:field {:join-alias "O"} (meta/id :orders :id)]]]
+ :fields [[:field {:base-type :type/Integer, :join-alias "O"} "ID"]]}]}
+ {:lib/type :mbql.stage/mbql
+ :fields [[:field {:base-type :type/BigInteger} "ID"]
+ [:field {:base-type :type/Integer} "O__ID"]]}]
+ :conditions [[:= {}
+ [:field {} (meta/id :people :id)]
+ [:field {:base-type :type/BigInteger, :join-alias "Q1"} (meta/id :orders :user-id)]]]
+ :fields [[:field {:base-type :type/Integer, :join-alias "Q1"} "ID"]
+ [:field {:base-type :type/Integer, :join-alias "Q1"} "o__ID"]]}]
+ :fields [[:field {} (meta/id :orders :id)]
+ [:field {:base-type :type/Integer, :join-alias "Q1"} "ID"]
+ [:field {:base-type :type/Integer, :join-alias "Q1"} "O__ID"]]}]
+ :database (meta/id)})]
+ (is (=? {:id (meta/id :orders :id)
+ :table-id (meta/id :orders)
+ :metabase.lib.join/join-alias "Q1"
+ :lib/source-column-alias "O__ID"
+ ::lib.field.resolution/fallback-metadata? (symbol "nil #_\"key is not present.\"")}
+ (lib.field.resolution/resolve-field-ref query -1 [:field {:base-type :type/Integer
+ :lib/uuid "00000000-0000-0000-0000-000000000000"
+ :join-alias "Q1"}
+ "O__ID"])))))
diff --git a/test/metabase/lib/schema/join_test.cljc b/test/metabase/lib/schema/join_test.cljc
index b191cabef6d..3bfd512d6b9 100644
--- a/test/metabase/lib/schema/join_test.cljc
+++ b/test/metabase/lib/schema/join_test.cljc
@@ -20,7 +20,6 @@
(is (=? {:lib/type "missing required key"
:stages [{:joins [{:stages [[{:lib/type "missing required key"}
"Invalid stage :lib/type: expected :mbql.stage/native or :mbql.stage/mbql"]]
- :lib/options "missing required key"
:conditions "should have at least 1 elements"}]}]}
(mu.humanize/humanize (mr/explain ::lib.schema/query {:stages [{:lib/type :mbql.stage/mbql
:joins [{:lib/type :mbql/join
diff --git a/test/metabase/lib/schema/util_test.cljc b/test/metabase/lib/schema/util_test.cljc
index e091ace6028..3f961ba2985 100644
--- a/test/metabase/lib/schema/util_test.cljc
+++ b/test/metabase/lib/schema/util_test.cljc
@@ -77,6 +77,13 @@
[:field {:lib/uuid "00000000-0000-0000-0000-000000000001", :effective-type :type/Number} 1]]
[[:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :effective-type :type/Integer} 1]
+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 1]]
+
+ ;; do not consider (inherited-)temporal-unit = default and no temporal-unit to be distinct
+ [[:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :temporal-unit :default} 1]
+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 1]]
+
+ [[:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :inherited-temporal-unit :default} 1]
[:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 1]]))
(deftest ^:parallel distinct-mbql-clauses-schema-test
diff --git a/test/metabase/lib/walk/util_test.cljc b/test/metabase/lib/walk/util_test.cljc
index af02823e1ad..decfdcd40d6 100644
--- a/test/metabase/lib/walk/util_test.cljc
+++ b/test/metabase/lib/walk/util_test.cljc
@@ -4,7 +4,8 @@
[metabase.lib.core :as lib]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util.macros :as lib.tu.macros]
- [metabase.lib.walk.util :as lib.walk.util]))
+ [metabase.lib.walk.util :as lib.walk.util]
+ [metabase.util.malli :as mu]))
(deftest ^:parallel all-source-table-ids-test
(testing (str "make sure that `all-table-ids` can properly find all Tables in the query, even in cases where a map "
@@ -31,3 +32,15 @@
:condition [:=
$checkins.user-id
&U.users.id]}]}))))))
+
+(deftest ^:parallel all-field-ids-test
+ (mu/disable-enforcement
+ (is (= #{1 2}
+ (lib/all-field-ids
+ {:lib/type :mbql/query
+ :database 1
+ :stages [{:lib/type :mbql.stage/mbql
+ :fields [[:field {} 1]
+ [:field {} 2]
+ [:field {} "wow"]]
+ :source-table 1}]})))))
diff --git a/test/metabase/lib/walk_test.cljc b/test/metabase/lib/walk_test.cljc
index ea6732f5577..8f18451b425 100644
--- a/test/metabase/lib/walk_test.cljc
+++ b/test/metabase/lib/walk_test.cljc
@@ -4,7 +4,9 @@
:cljs ([metabase.test-runner.assert-exprs.approximately-equal]))
[clojure.test :refer [are deftest is testing use-fixtures]]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-metadata :as meta]
+ [metabase.lib.util.match :as lib.util.match]
[metabase.lib.walk :as lib.walk]
[metabase.util :as u]
[metabase.util.malli.registry :as mr]))
@@ -223,6 +225,28 @@
60]]]
@calls))))
+(deftest ^:parallel walk-clauses-join-conditions-test
+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/join (meta/table-metadata :categories)))]
+ (is (=? {:stages [{:joins [{:alias "Categories"
+ :conditions [[:=
+ {}
+ [:field {} (meta/id :venues :category-id)]
+ [:field {:join-alias "Categories"} (meta/id :categories :id)]]]}]}]}
+ query))
+ (is (=? {:stages [{:joins [{:alias "Categories"
+ :conditions [[:=
+ {}
+ [:field {} "CATEGORY_ID"]
+ [:field {:join-alias "Categories"} "ID"]]]}]}]}
+ (lib.walk/walk-clauses
+ query
+ (fn [query _path-type _stage-or-join-path clause]
+ (lib.util.match/match-lite clause
+ [:field opts id]
+ (let [col (lib.metadata/field query id)]
+ [:field (merge (select-keys col [:base-type]) opts) (:name col)]))))))))
+
(deftest ^:parallel walk-clauses-identity-test
(testing "If we don't update any clauses then we should return the original query"
(let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
diff --git a/test/metabase/query_processor/middleware/add_implicit_joins_test.clj b/test/metabase/query_processor/middleware/add_implicit_joins_test.clj
index 9281ec464be..ed5126fd449 100644
--- a/test/metabase/query_processor/middleware/add_implicit_joins_test.clj
+++ b/test/metabase/query_processor/middleware/add_implicit_joins_test.clj
@@ -10,7 +10,6 @@
[metabase.query-processor :as qp]
[metabase.query-processor.middleware.add-implicit-clauses :as qp.add-implicit-clauses]
[metabase.query-processor.middleware.add-implicit-joins :as qp.add-implicit-joins]
- [metabase.query-processor.middleware.add-source-metadata :as qp.add-source-metadata]
[metabase.query-processor.middleware.fetch-source-query]
[metabase.query-processor.schema :as qp.schema]
[metabase.test :as mt]
@@ -352,7 +351,6 @@
[:field %products.category {:source-field %product-id
:source-field-name "Orders__PRODUCT_ID"
:source-field-join-alias "Card"}]]})
- qp.add-source-metadata/add-source-metadata-for-source-queries
add-implicit-clauses
add-implicit-joins)))))
diff --git a/test/metabase/query_processor/middleware/add_remaps_test.clj b/test/metabase/query_processor/middleware/add_remaps_test.clj
index a3173bb00eb..59aaf37d99f 100644
--- a/test/metabase/query_processor/middleware/add_remaps_test.clj
+++ b/test/metabase/query_processor/middleware/add_remaps_test.clj
@@ -693,10 +693,10 @@
;; -- Cam
;;
;;
- ;; 1 remap for self-joined orders.user-id => people.email
- [:field (meta/id :people :email) {:join-alias "j"}]
;; 1 remap for source table orders.user-id => people.email
- [:field (meta/id :people :email) {:join-alias "PEOPLE__via__USER_ID", ::qp.add-remaps/new-field-dimension-id pos-int?}]]}
+ [:field (meta/id :people :email) {:join-alias "PEOPLE__via__USER_ID", ::qp.add-remaps/new-field-dimension-id pos-int?}]
+ ;; 1 remap for self-joined orders.user-id => people.email
+ [:field (meta/id :people :email) {:join-alias "j"}]]}
preprocessed-query)))))))
(deftest ^:parallel add-remaps-to-joins-e2e-test-2
@@ -760,10 +760,8 @@
[:field (meta/id :venues :longitude) {:join-alias "J"}]
[:field (meta/id :venues :price) {:join-alias "J"}]
;; we shouldn't use IDs here because they would be ambiguous.
- [:field "NAME_2" {:join-alias "J"}]
- [:field "NAME_3" {:join-alias "J"}]
- ;; TODO -- this shouldn't be here! DUPLICATE!
- [:field (meta/id :categories :name) {}]]}
+ [:field (meta/id :categories :name) {:join-alias "J", :source-field (meta/id :venues :id)}]
+ [:field (meta/id :categories :name) {:join-alias "J", :source-field (meta/id :venues :category-id)}]]}
;; these are in the opposite order as the join source query because `CATEGORY_ID`
;; appears before `ID` here but the other way around above.
{:alias "CATEGORIES__via__CATEGORY_ID"}
@@ -777,15 +775,14 @@
[:field (meta/id :venues :latitude) {:join-alias "J"}]
[:field (meta/id :venues :longitude) {:join-alias "J"}]
[:field (meta/id :venues :price) {:join-alias "J"}]
- [:field "NAME_2" {:join-alias "J"}]
- [:field "NAME_3" {:join-alias "J"}]
[:field (meta/id :categories :name) {:join-alias "CATEGORIES__via__CATEGORY_ID"}]
[:field (meta/id :categories :name) {:join-alias "CATEGORIES__via__ID"}]
- ;; TODO DUPLICATE!!!
- [:field (meta/id :categories :name) {:join-alias "J"}]]}}
+ [:field (meta/id :categories :name) {:join-alias "J", :source-field (meta/id :venues :id)}]
+ [:field (meta/id :categories :name) {:join-alias "J", :source-field (meta/id :venues :category-id)}]]}}
(-> query
qp.preprocess/preprocess
- lib/->legacy-MBQL))))))
+ lib/->legacy-MBQL
+ (select-keys [:query])))))))
(deftest ^:parallel do-not-include-remaps-in-joins-for-columns-that-are-not-in-fields-e2e-test
(testing "Do not include remaps in joins for columns that are not in :fields (#63165)"
diff --git a/test/metabase/query_processor/middleware/add_source_metadata_test.clj b/test/metabase/query_processor/middleware/add_source_metadata_test.clj
deleted file mode 100644
index 2b8d2724bd7..00000000000
--- a/test/metabase/query_processor/middleware/add_source_metadata_test.clj
+++ /dev/null
@@ -1,382 +0,0 @@
-(ns metabase.query-processor.middleware.add-source-metadata-test
- (:require
- [clojure.test :refer :all]
- [medley.core :as m]
- [metabase.driver :as driver]
- [metabase.lib.test-metadata :as meta]
- [metabase.lib.test-util :as lib.tu]
- [metabase.lib.test-util.macros :as lib.tu.macros]
- [metabase.query-processor.middleware.add-source-metadata :as add-source-metadata]
- [metabase.query-processor.preprocess :as qp.preprocess]
- [metabase.query-processor.store :as qp.store]
- [metabase.test :as mt]
- [metabase.util :as u]))
-
-(defn- add-source-metadata [query]
- (driver/with-driver :h2
- (qp.store/with-metadata-provider (if (qp.store/initialized?)
- (qp.store/metadata-provider)
- meta/metadata-provider)
- (add-source-metadata/add-source-metadata-for-source-queries query))))
-
-(defn- results-col [col]
- (select-keys
- col
- [:id :table_id :name :display_name :base_type :effective_type :coercion_strategy
- :semantic_type :unit :fingerprint :settings :field_ref :nfc_path :parent_id]))
-
-(defn- results-metadata [cols]
- (map results-col cols))
-
-(defn- venues-source-metadata
- ([]
- (venues-source-metadata :id :name :category-id :latitude :longitude :price))
-
- ([& field-names]
- (let [field-ids (map #(meta/id :venues (keyword (u/lower-case-en (name %))))
- field-names)]
- (qp.store/with-metadata-provider (if (qp.store/initialized?)
- (qp.store/metadata-provider)
- meta/metadata-provider)
- (results-metadata
- (qp.preprocess/query->expected-cols
- (lib.tu.macros/mbql-query venues
- {:fields (for [id field-ids] [:field id nil])
- :limit 1})))))))
-
-(deftest ^:parallel basic-test
- (testing "Can we automatically add source metadata to the parent level of a query? If the source query has `:fields`"
- (is (=? (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues
- :fields [$id $name]}
- :source-metadata (venues-source-metadata :id :name)})
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues
- :fields [$id $name]}}))))))
-
-(deftest ^:parallel basic-parent-level-test
- (testing (str "Can we automatically add source metadata to the parent level of a query? If the source query does not "
- "have `:fields`")
- (is (=? (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues}
- :source-metadata (venues-source-metadata)})
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues}}))))))
-
-(deftest ^:parallel basic-summary-columns-test
- (testing "Can we add source metadata for a source query that has breakouts/aggregations?"
- (is (=? (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues
- :aggregation [[:count]]
- :breakout [$price]}
- :source-metadata [(-> (venues-source-metadata :price)
- first)
- {:name "count"
- :display_name "Count"
- :base_type :type/Integer
- :semantic_type :type/Quantity
- :field_ref [:aggregation 0]}]})
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues
- :aggregation [[:count]]
- :breakout [$price]}}))))))
-
-(deftest ^:parallel basic-aggregation-with-field-test
- (testing "Can we add source metadata for a source query that has an aggregation for a specific Field?"
- (is (=? (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues
- :aggregation [[:avg $id]]
- :breakout [$price]}
- :source-metadata [(-> (venues-source-metadata :price)
- first)
- {:name "avg"
- :display_name "Average of ID"
- :base_type :type/Float
- :semantic_type :type/PK
- :field_ref [:aggregation 0]}]})
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues
- :aggregation [[:avg $id]]
- :breakout [$price]}}))))))
-
-(defn- source-metadata [query]
- (get-in query [:query :source-metadata] query))
-
-(deftest ^:parallel named-aggregations-test
- (testing "adding source metadata for source queries with named aggregations"
- (testing "w/ `:name` and `:display-name`"
- (is (=? (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues
- :aggregation [[:aggregation-options
- [:avg $id]
- {:name "some_generated_name", :display-name "My Cool Ag"}]]
- :breakout [$price]}
- :source-metadata [(-> (venues-source-metadata :price)
- first)
- {:name "some_generated_name"
- :display_name "My Cool Ag"
- :base_type :type/Float
- :semantic_type :type/PK
- :field_ref [:aggregation 0]}]})
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues
- :aggregation [[:aggregation-options
- [:avg $id]
- {:name "some_generated_name", :display-name "My Cool Ag"}]]
- :breakout [$price]}})))))))
-
-(deftest ^:parallel named-aggregations-name-only-test
- (testing "w/ `:name` only"
- (is (=? [{:name "some_generated_name"
- :display_name "Average of ID"
- :base_type :type/Float
- :semantic_type :type/PK
- :field_ref [:aggregation 0]}]
- (source-metadata
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues
- :aggregation [[:aggregation-options [:avg $id] {:name "some_generated_name"}]]}})))))))
-
-(deftest ^:parallel named-aggregations-display-name-only-test
- (testing "w/ `:display-name` only"
- (is (=? [{:name "avg"
- :display_name "My Cool Ag"
- :base_type :type/Float
- :semantic_type :type/PK
- :field_ref [:aggregation 0]}]
- (source-metadata
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues
- :aggregation [[:aggregation-options [:avg $id] {:display-name "My Cool Ag"}]]}})))))))
-
-(deftest ^:parallel nested-sources-test
- (testing (str "Can we automatically add source metadata to the parent level of a query? If the source query has a "
- "source query with source metadata")
- (is (=? (lib.tu.macros/mbql-query venues
- {:source-query {:source-query {:source-table $$venues
- :fields [$id $name]}
- :source-metadata (venues-source-metadata :id :name)}
- :source-metadata (venues-source-metadata :id :name)})
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query {:source-query {:source-table $$venues
- :fields [$id $name]}
- :source-metadata (venues-source-metadata :id :name)}}))))))
-
-(deftest ^:parallel nested-sources-3-levels-with-source-metadata-test
- (testing "Can we automatically add source metadata if a source-query nested 3 levels has `:source-metadata`?"
- (is (=? (lib.tu.macros/mbql-query venues
- {:source-query {:source-query {:source-query {:source-table $$venues
- :fields [$id $name]}
- :source-metadata (venues-source-metadata :id :name)}
- :source-metadata (venues-source-metadata :id :name)}
- :source-metadata (venues-source-metadata :id :name)})
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query
- {:source-query
- {:source-query {:source-table $$venues
- :fields [$id $name]}
- :source-metadata (venues-source-metadata :id :name)}}}))))))
-
-(deftest ^:parallel nested-sources-3-levels-with-no-source-metadata-test
- (testing "Ok, how about a source query nested 3 levels with no `source-metadata`?"
- (is (=? (lib.tu.macros/mbql-query venues
- {:source-query {:source-query {:source-query {:source-table $$venues}
- :source-metadata (venues-source-metadata)}
- :source-metadata (venues-source-metadata)}
- :source-metadata (venues-source-metadata)})
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query {:source-query {:source-query {:source-table $$venues}}}}))))))
-
-(deftest ^:parallel nested-sources-3-levels-with-fields-test
- (testing "nested 3 levels with `fields`"
- (is (=? (lib.tu.macros/mbql-query venues
- {:source-query {:source-query {:source-query {:source-table $$venues
- :fields [$id $name]}
- :source-metadata (venues-source-metadata :id :name)}
- :source-metadata (venues-source-metadata :id :name)}
- :source-metadata (venues-source-metadata :id :name)})
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query {:source-query {:source-query {:source-table $$venues
- :fields [$id $name]}}}}))))))
-
-(deftest ^:parallel nested-sources-3-levels-with-summary-columns-test
- (testing "nested 3 levels with breakouts/aggregations"
- (qp.store/with-metadata-provider meta/metadata-provider
- ;; field ref for the count aggregation differs slightly depending on what level of the query we're at; at the
- ;; most-deeply-nested level we can use the `[:aggregation 0]` ref to refer to it; at higher levels we have to
- ;; refer to it with a field literal
- (is (=? (letfn [(metadata-with-count-field-ref [field-ref]
- [(-> (venues-source-metadata :price) first)
- (let [[count-col] (->> (lib.tu.macros/mbql-query venues
- {:aggregation [[:count]]})
- qp.preprocess/query->expected-cols
- results-metadata)]
- (-> count-col
- (dissoc :effective_type)
- (assoc :field_ref field-ref
- :base_type :type/Integer)))])]
- (lib.tu.macros/mbql-query venues
- {:source-query {:source-query {:source-query {:source-table $$venues
- :aggregation [[:count]]
- :breakout [$price]}
- :source-metadata (metadata-with-count-field-ref [:aggregation 0])}
- :source-metadata (metadata-with-count-field-ref *count/Integer)}
- :source-metadata (metadata-with-count-field-ref *count/Integer)}))
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query {:source-query {:source-query {:source-table $$venues
- :aggregation [[:count]]
- :breakout [$price]}}}})))))))
-
-(deftest ^:parallel nested-sources-with-source-native-query-test
- (testing "can we add `source-metadata` to the parent level if the source query has a native source query, but itself has `source-metadata`?"
- (is (=? (lib.tu.macros/mbql-query venues
- {:source-query {:source-query {:native "SELECT \"ID\", \"NAME\" FROM \"VENUES\";"}
- :source-metadata (venues-source-metadata :id :name)}
- :source-metadata (venues-source-metadata :id :name)})
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query {:source-query {:native "SELECT \"ID\", \"NAME\" FROM \"VENUES\";"}
- :source-metadata (venues-source-metadata :id :name)}}))))))
-
-(deftest ^:parallel joins-test
- (testing "should work inside JOINS as well"
- (is (=? (lib.tu.macros/mbql-query venues
- {:source-table $$venues
- :joins [{:source-query {:source-table $$venues
- :fields [$id $name]}
- :source-metadata (venues-source-metadata :id :name)}]})
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-table $$venues
- :joins [{:source-query {:source-table $$venues
- :fields [$id $name]}
- :condition [:= "A" "B"]}]}))))))
-
-(deftest ^:parallel binned-fields-test
- (testing "source metadata should handle source queries that have binned fields"
- (qp.store/with-metadata-provider (lib.tu/mock-metadata-provider
- meta/metadata-provider
- {:settings {:breakout-bin-width 5.0}})
- (is (=? (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues
- :aggregation [[:count]]
- :breakout [[:field %latitude {:binning {:strategy :default}}]]}
- :source-metadata (concat
- (let [[lat-col] (venues-source-metadata :latitude)
- [count-col] (results-metadata (qp.preprocess/query->expected-cols
- (lib.tu.macros/mbql-query venues
- {:aggregation [[:count]]})))]
- [(assoc lat-col
- :field_ref [:field
- (meta/id :venues :latitude)
- {:binning {:strategy :bin-width
- :min-value 10.0
- :max-value 45.0
- :num-bins 7
- :bin-width 5.0}}]
- :display_name "Latitude: 5°")
- ;; computed column doesn't have an effective type in middleware before query
- (-> count-col
- (dissoc :effective_type)
- ;; the type that comes back from H2 is :type/BigInteger but the type that comes
- ;; back from calculating it with MLv2 is just plain :type/Integer
- (assoc :base_type :type/Integer))]))})
- (add-source-metadata
- (lib.tu.macros/mbql-query venues
- {:source-query
- {:source-table $$venues
- :aggregation [[:count]]
- :breakout [[:field %latitude {:binning {:strategy :default}}]]}})))))))
-
-(deftest ^:parallel deduplicate-column-names-test
- (testing "Metadata that gets added to source queries should have deduplicated column names"
- (let [query (add-source-metadata
- (lib.tu.macros/mbql-query checkins
- {:source-query {:source-table $$checkins
- :joins [{:fields :all
- :alias "u"
- :source-table $$users
- :condition [:= $user-id &u.users.id]}]}
- :joins [{:fields :all
- :alias "v"
- :source-table $$venues
- :condition [:= *user-id &v.venues.id]}]
- :order-by [[:asc $id]]
- :limit 2}))]
- (is (= ["ID" "DATE" "USER_ID" "VENUE_ID" "ID_2" "NAME" "LAST_LOGIN"]
- (map :name (get-in query [:query :source-metadata])))))))
-
-(deftest ^:parallel inception-test
- (testing "Should be able to do an 'inception-style' nesting of source > source > source with a join (#14724)"
- ;; these tests look at the metadata for just one column so it's easier to spot the differences.
- (letfn [(ean-metadata [query]
- (as-> query query
- (get-in query [:query :source-metadata])
- (m/index-by :name query)
- (get query "EAN")
- (select-keys query [:name :display_name :base_type :semantic_type :id :field_ref])))]
- (let [base-query (lib.tu.macros/mbql-query orders
- {:source-table $$orders
- :joins [{:fields :all
- :source-table $$products
- :condition [:= $product-id &Products.products.id]
- :alias "Products"}]
- :limit 10})]
- (testing "Make sure metadata is correct for the 'EAN' column with"
- (doseq [level (range 1 4)
- ;; existing usage, do not use this going forward
- :let [query #_{:clj-kondo/ignore [:deprecated-var]} (mt/nest-query base-query level)]]
- (testing (format "%d level(s) of nesting" level)
- (is (= (lib.tu.macros/$ids products
- {:name "EAN"
- :display_name "Products → Ean"
- :base_type :type/Text
- :id %ean
- :field_ref &Products.ean})
- (ean-metadata (add-source-metadata query)))))))))))
-
-(deftest ^:parallel ignore-legacy-source-metadata-test
- (testing "Should ignore 'legacy' < 0.38.0 source metadata and recalculate it for MBQL queries (#14788)"
- ;; normally this middleware will use existing source metadata rather than recalculating it, but if we encounter <
- ;; 0.38.0 source metadata that is missing `:field_ref` and `:id` information we should ignore it.
- (let [query (lib.tu.macros/mbql-query orders
- {:source-query {:source-table $$orders
- :joins [{:source-table $$products
- :alias "ℙ"
- :fields :all
- :condition [:= $product-id &ℙ.products.id]}]
- :order-by [[:asc $id]]
- :limit 2}})
- metadata (qp.store/with-metadata-provider meta/metadata-provider
- (qp.preprocess/query->expected-cols query))
- ;; the actual metadata this middleware should return. Doesn't have all the columns that come back from
- ;; `qp.preprocess/query->expected-cols`
- expected-metadata (map results-col metadata)]
- (letfn [(added-metadata [query]
- (get-in (add-source-metadata query) [:query :source-metadata]))]
- (testing "\nShould add source metadata if there's none already"
- (is (=? expected-metadata
- (added-metadata query))))
- (testing "\nShould use existing metadata if it's already there"
- ;; since it's using the existing metadata, it should have all the extra keys instead of the subset in
- ;; `expected-metadata`
- (is (= metadata
- (added-metadata (assoc-in query [:query :source-metadata] metadata)))))
- (testing "\nShould ignore legacy metadata"
- ;; pre-0.38.0 metadata didn't have `field_ref` or `id.`
- (let [legacy-metadata (for [col metadata]
- (dissoc col :field_ref :id))]
- (is (=? expected-metadata
- (added-metadata (assoc-in query [:query :source-metadata] legacy-metadata))))))))))
diff --git a/test/metabase/query_processor/middleware/resolve_fields_test.clj b/test/metabase/query_processor/middleware/resolve_fields_test.clj
deleted file mode 100644
index eac6dffc742..00000000000
--- a/test/metabase/query_processor/middleware/resolve_fields_test.clj
+++ /dev/null
@@ -1,18 +0,0 @@
-(ns metabase.query-processor.middleware.resolve-fields-test
- (:require
- [clojure.test :refer :all]
- [metabase.query-processor.middleware.resolve-fields :as qp.resolve-fields]
- [metabase.util.malli.fn :as mu.fn]))
-
-(deftest ^:parallel field-ids-test
- (testing "Field IDs in metadata should get picked up correctly"
- (binding [mu.fn/*enforce* false]
- (is (= #{1}
- (#'qp.resolve-fields/field-ids
- {:lib/type :mbql/query
- :database 1
- :stages [{:lib/type :mbql.stage/mbql
- :lib/stage-metadata {:lib/type :metadata/results
- :columns [{:lib/type :metadata/column, :id 1}
- {:lib/type :metadata/column, :id 0}]}
- :source-table 1}]}))))))
diff --git a/test/metabase/query_processor/middleware/resolve_joins_test.clj b/test/metabase/query_processor/middleware/resolve_joins_test.clj
index 5ed2ad16aa1..4b939e49ba6 100644
--- a/test/metabase/query_processor/middleware/resolve_joins_test.clj
+++ b/test/metabase/query_processor/middleware/resolve_joins_test.clj
@@ -6,144 +6,132 @@
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.macros :as lib.tu.macros]
- [metabase.query-processor :as qp]
[metabase.query-processor.middleware.resolve-joins :as resolve-joins]
- [metabase.query-processor.preprocess :as qp.preprocess]
- [metabase.query-processor.store :as qp.store]
- [metabase.test :as mt]))
+ [metabase.query-processor.preprocess :as qp.preprocess]))
-(defn- resolve-joins [query]
- (if (qp.store/initialized?)
- (resolve-joins/resolve-joins query)
- (qp.store/with-metadata-provider (mt/id)
- (resolve-joins/resolve-joins query))))
+(defn- resolve-joins
+ ([query]
+ (resolve-joins meta/metadata-provider query))
+
+ ([mp query]
+ (if (:lib/type query)
+ (resolve-joins/resolve-joins query)
+ (-> (lib/query mp query)
+ resolve-joins/resolve-joins
+ lib/->legacy-MBQL))))
(deftest ^:parallel joins->fields-test
- (is (= [[:field 1 {:join-alias "B", :qp/ignore-coercion true}]
- [:field 2 {:join-alias "B", :qp/ignore-coercion true}]
- [:field 3 {:join-alias "C", :qp/ignore-coercion true}]
- [:field 4 {:join-alias "C", :qp/ignore-coercion true}]]
- (#'resolve-joins/joins->fields [{:alias "A"
- :fields :all}
- {:alias "B"
- :fields [[:field 1 {:join-alias "B"}]
- ;; missing join alias
- [:field 2 nil]]}
- {:alias "C"
- :fields [[:field 3 {:join-alias "C"}]
- ;; wrong join alias
- [:field 4 {:join-alias "X"}]]}]))))
+ (is (=? [[:field {:join-alias "B", :qp/ignore-coercion true} 1]
+ [:field {:join-alias "B", :qp/ignore-coercion true} 2]
+ [:field {:join-alias "C", :qp/ignore-coercion true} 3]
+ [:field {:join-alias "C", :qp/ignore-coercion true} 4]]
+ (#'resolve-joins/joins->fields [{:alias "A"
+ :fields :all}
+ {:alias "B"
+ :fields [[:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :join-alias "B"} 1]
+ ;; missing join alias
+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 2]]}
+ {:alias "C"
+ :fields [[:field {:lib/uuid "00000000-0000-0000-0000-000000000002", :join-alias "C"} 3]
+ ;; wrong join alias
+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000003", :join-alias "X"} 4]]}]))))
(deftest ^:parallel no-op-test
(testing "Does the middleware function if the query has no joins?"
- (is (= (mt/mbql-query venues)
+ (is (= (lib.tu.macros/mbql-query venues)
(resolve-joins
- (mt/mbql-query venues))))))
+ (lib.tu.macros/mbql-query venues))))))
(deftest ^:parallel fields-none-test
(testing "Can we resolve some joins w/ fields = none?"
- (is (=? (mt/mbql-query venues
+ (is (=? (lib.tu.macros/mbql-query venues
{:joins
[{:source-table $$categories
:alias "c"
:strategy :left-join
- :condition [:= $category_id &c.categories.id]}]})
+ :condition [:= $category-id &c.categories.id]}]})
(resolve-joins
- (mt/mbql-query venues
+ (lib.tu.macros/mbql-query venues
{:joins [{:source-table $$categories
:alias "c"
- :condition [:= $category_id &c.categories.id]
+ :condition [:= $category-id &c.categories.id]
:fields :none}]}))))))
(deftest ^:parallel fields-all-test
(testing "Can we resolve some joins w/ fields = all ???"
- (is (= (mt/mbql-query venues
- {:joins
- [{:source-table $$categories
- :alias "c"
- :strategy :left-join
- :condition [:= $category_id &c.categories.id]
- :fields [&c.categories.id
- &c.categories.name]}]
- :fields [$venues.id
- $venues.name
- [:field %categories.id {:join-alias "c"
- :qp/ignore-coercion true}]
- [:field %categories.name {:join-alias "c"
- :qp/ignore-coercion true}]]})
- (resolve-joins
- (mt/mbql-query venues
- {:fields [$venues.id $venues.name]
- :joins [{:source-table $$categories
- :alias "c"
- :condition [:= $category_id &c.categories.id]
- :fields :all}]}))))))
+ (is (=? (lib.tu.macros/mbql-query venues
+ {:joins
+ [{:source-table $$categories
+ :alias "c"
+ :strategy :left-join
+ :condition [:= $category-id &c.categories.id]
+ :fields [&c.categories.id
+ &c.categories.name]}]
+ :fields [$venues.id
+ $venues.name
+ [:field %categories.id {:join-alias "c"
+ :qp/ignore-coercion true}]
+ [:field %categories.name {:join-alias "c"
+ :qp/ignore-coercion true}]]})
+ (resolve-joins
+ (lib.tu.macros/mbql-query venues
+ {:fields [$venues.id $venues.name]
+ :joins [{:source-table $$categories
+ :alias "c"
+ :condition [:= $category-id &c.categories.id]
+ :fields :all}]}))))))
(deftest ^:parallel fields-sequence-test
(testing "can we resolve joins w/ fields = <sequence>"
- (is (= (mt/mbql-query venues
+ (is (= (lib.tu.macros/mbql-query venues
{:joins
[{:source-table $$categories
:alias "c"
:strategy :left-join
- :condition [:= $category_id &c.categories.id]
+ :condition [:= $category-id &c.categories.id]
:fields [&c.categories.name]}]
:fields [$venues.id
$venues.name
[:field %categories.name {:join-alias "c"
:qp/ignore-coercion true}]]})
(resolve-joins
- (mt/mbql-query venues
+ (lib.tu.macros/mbql-query venues
{:fields [$venues.id $venues.name]
:joins [{:source-table $$categories
:alias "c"
- :condition [:= $category_id &c.categories.id]
+ :condition [:= $category-id &c.categories.id]
:fields [&c.categories.name]}]}))))))
(deftest ^:parallel join-table-without-alias-test
(testing "Does joining a table an explicit alias add a default alias?"
- (is (=? (mt/mbql-query venues
+ (is (=? (lib.tu.macros/mbql-query venues
{:joins [{:source-table $$categories
:alias "__join"
:strategy :left-join
- :condition [:= $category_id 1]}
+ :condition [:= $category-id 1]}
{:source-table $$categories
:alias "__join_2"
:strategy :left-join
- :condition [:= $category_id 2]}]
- :source-table (mt/id :venues)})
+ :condition [:= $category-id 2]}]
+ :source-table (meta/id :venues)})
(resolve-joins
- (mt/mbql-query venues
+ (lib.tu.macros/mbql-query venues
{:joins [{:source-table $$categories
- :condition [:= $category_id 1]}
+ :condition [:= $category-id 1]}
{:source-table $$categories
- :condition [:= $category_id 2]}]}))))))
-
-(deftest ^:parallel disallow-joins-against-table-on-different-db-test
- (testing "Test that joining against a table in a different DB throws an Exception"
- (qp.store/with-metadata-provider (lib.tu/mock-metadata-provider
- {:database meta/database
- :tables [(meta/table-metadata :venues)]})
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"\QFailed to fetch :metadata/table\E"
- (resolve-joins
- (mt/mbql-query venues
- {:joins [{:source-table (meta/id :categories)
- :alias "t"
- :condition [:= $category_id 1]}]})))))))
+ :condition [:= $category-id 2]}]}))))))
(deftest ^:parallel resolve-explicit-joins-when-implicit-joins-are-present-test
(testing "test that resolving explicit joins still works if implict joins are present"
- (is (=? (mt/mbql-query checkins
+ (is (=? (lib.tu.macros/mbql-query checkins
{:source-table $$checkins
:aggregation [[:sum &USERS__via__USER_ID.users.id]]
:breakout [$id]
:joins [{:source-table $$users
:alias "USERS__via__USER_ID"
:strategy :left-join
- :condition [:= $user_id &USERS__via__USER_ID.users.id]
- :fk-field-id (mt/id :checkins :user_id)}
+ :condition [:= $user-id &USERS__via__USER_ID.users.id]
+ :fk-field-id (meta/id :checkins :user-id)}
{:alias "u"
:source-table $$users
:strategy :left-join
@@ -152,15 +140,15 @@
&u.users.id]}]
:limit 10})
(resolve-joins
- (mt/mbql-query checkins
+ (lib.tu.macros/mbql-query checkins
{:source-table $$checkins
:aggregation [[:sum &USERS__via__USER_ID.users.id]]
:breakout [$id]
:joins [{:source-table $$users
:alias "USERS__via__USER_ID"
:strategy :left-join
- :condition [:= $user_id &USERS__via__USER_ID.users.id]
- :fk-field-id (mt/id :checkins :user_id)
+ :condition [:= $user-id &USERS__via__USER_ID.users.id]
+ :fk-field-id (meta/id :checkins :user-id)
:fields :none}
{:alias "u"
:source-table $$users
@@ -171,98 +159,96 @@
(deftest ^:parallel join-with-source-query-test
(testing "Does a join using a source query get its Tables resolved?"
- (is (=? (mt/mbql-query venues
+ (is (=? (lib.tu.macros/mbql-query venues
{:joins [{:alias "cat"
- :source-query {:source-table $$categories}
+ :source-table $$categories
:strategy :left-join
:condition [:=
- $category_id
+ $category-id
[:field "ID" {:base-type :type/BigInteger, :join-alias "cat"}]]}]
:order-by [[:asc $name]]
:limit 3})
(resolve-joins
- (mt/mbql-query venues
+ (lib.tu.macros/mbql-query venues
{:joins [{:alias "cat"
:source-query {:source-table $$categories}
:condition [:=
- $category_id
+ $category-id
[:field "ID" {:base-type :type/BigInteger, :join-alias "cat"}]]}]
:order-by [[:asc $name]]
:limit 3}))))))
(deftest ^:parallel resolve-source-query-with-fields-all-test
(testing "Can we resolve joins using a `:source-query` and `:fields` `:all`?"
- (let [source-metadata (get-in (qp/process-query
- (qp/userland-query
- (mt/mbql-query categories {:limit 1})))
- [:data :results_metadata :columns])
+ (let [source-metadata (qp.preprocess/query->expected-cols
+ (lib/query meta/metadata-provider (lib.tu.macros/mbql-query categories {:limit 1})))
resolved (resolve-joins
- (mt/mbql-query venues
+ (lib.tu.macros/mbql-query venues
{:joins [{:alias "cat"
:source-query {:source-table $$categories}
:source-metadata source-metadata
:fields :all
:condition [:=
- $category_id
+ $category-id
[:field "ID" {:base-type :type/BigInteger, :join-alias "cat"}]]}]
:order-by [[:asc $name]]
:limit 3}))]
- (is (= (mt/mbql-query venues
- {:fields [[:field (mt/id :categories :id) {:join-alias "cat", :qp/ignore-coercion true}]
- [:field (mt/id :categories :name) {:join-alias "cat", :qp/ignore-coercion true}]]
- :joins [{:alias "cat"
- :source-query {:source-table $$categories}
- :source-metadata source-metadata
- :strategy :left-join
- :condition [:= $category_id [:field "ID" {:base-type :type/BigInteger, :join-alias "cat"}]]
- :fields [&cat.categories.id
- &cat.categories.name]}]
- :order-by [[:asc $name]]
- :limit 3})
- resolved)))))
+ (is (=? (lib.tu.macros/mbql-query venues
+ {:fields [[:field (meta/id :categories :id) {:join-alias "cat", :qp/ignore-coercion true}]
+ [:field (meta/id :categories :name) {:join-alias "cat", :qp/ignore-coercion true}]]
+ :joins [{:alias "cat"
+ :source-table $$categories
+ :source-metadata source-metadata
+ :strategy :left-join
+ :condition [:= $category-id [:field "ID" {:base-type :type/BigInteger, :join-alias "cat"}]]
+ :fields [&cat.categories.id
+ &cat.categories.name]}]
+ :order-by [[:asc $name]]
+ :limit 3})
+ resolved)))))
(deftest ^:parallel dont-append-fields-if-parent-has-breakout-or-aggregation-test
(testing "if the parent level has a breakout or aggregation, we shouldn't append Join fields to the parent level"
- (is (= (mt/mbql-query users
- {:joins [{:source-table $$checkins
- :alias "c"
- :strategy :left-join
- :condition [:= $id [:field "USER_ID" {:base-type :type/Integer, :join-alias "c"}]]
- :fields [&c.checkins.id
- &c.checkins.date
- &c.checkins.user_id
- &c.checkins.venue_id]}]
- :aggregation [[:sum [:field "id" {:base-type :type/Float, :join-alias "c"}]]]
- :breakout [[:field %last_login {:temporal-unit :month}]]})
- (resolve-joins
- (mt/mbql-query users
- {:joins [{:fields :all
+ (is (=? (lib.tu.macros/mbql-query users
+ {:joins [{:source-table $$checkins
:alias "c"
- :source-table $$checkins
- :condition [:= $id [:field "USER_ID" {:base-type :type/Integer, :join-alias "c"}]]}]
+ :strategy :left-join
+ :condition [:= $id [:field "USER_ID" {:base-type :type/Integer, :join-alias "c"}]]
+ :fields [&c.checkins.id
+ &c.checkins.date
+ &c.checkins.user-id
+ &c.checkins.venue-id]}]
:aggregation [[:sum [:field "id" {:base-type :type/Float, :join-alias "c"}]]]
- :breakout [[:field %last_login {:temporal-unit :month}]]}))))))
+ :breakout [[:field %last-login {:temporal-unit :month}]]})
+ (resolve-joins
+ (lib.tu.macros/mbql-query users
+ {:joins [{:fields :all
+ :alias "c"
+ :source-table $$checkins
+ :condition [:= $id [:field "USER_ID" {:base-type :type/Integer, :join-alias "c"}]]}]
+ :aggregation [[:sum [:field "id" {:base-type :type/Float, :join-alias "c"}]]]
+ :breakout [[:field %last-login {:temporal-unit :month}]]}))))))
(deftest ^:parallel aggregation-field-ref-test
(testing "Should correctly handle [:aggregation n] field refs"
(is (some? (resolve-joins
- (mt/mbql-query users
+ (lib.tu.macros/mbql-query users
{:fields [$id
$name
- [:field %last_login {:temporal-unit :default}]]
+ [:field %last-login {:temporal-unit :default}]]
:joins [{:fields :all
:alias "__alias__"
- :condition [:= $id [:field %checkins.user_id {:join-alias "__alias__"}]]
+ :condition [:= $id [:field %checkins.user-id {:join-alias "__alias__"}]]
:source-query {:source-table $$checkins
:aggregation [[:sum $checkins.id]]
- :breakout [$checkins.user_id]}
+ :breakout [$checkins.user-id]}
:source-metadata
[{:name "USER_ID"
:display_name "User ID"
:base_type :type/Integer
:semantic_type :type/FK
- :id %checkins.user_id
- :field_ref $checkins.user_id
+ :id %checkins.user-id
+ :field_ref $checkins.user-id
:fingerprint {:global {:distinct-count 15, :nil% 0.0}}}
{:name "sum"
:display_name "Sum of ID"
@@ -274,14 +260,13 @@
(deftest ^:parallel native-model-field-ref-test
(testing "should use name-based field refs for joined native models with mapped database fields (metabase#58829)"
- (let [source-metadata [{:id (mt/id :checkins :id)
- :name "_USER_ID"
+ (let [source-metadata [{:name "_USER_ID"
:display_name "User ID"
:base_type :type/Integer
:semantic_type :type/FK
:field_ref [:field "_USER_ID" {:base-type :type/Integer :join-alias "alias"}]
:fingerprint {:global {:distinct-count 15, :nil% 0.0}}}]]
- (is (= (mt/mbql-query users
+ (is (= (lib.tu.macros/mbql-query users
{:fields [$id
[:field "_USER_ID" {:base-type :type/Integer
:join-alias "alias"
@@ -294,7 +279,7 @@
:source-metadata source-metadata}]
:limit 10})
(resolve-joins
- (mt/mbql-query users
+ (lib.tu.macros/mbql-query users
{:fields [$id]
:joins [{:fields :all
:alias "alias"
@@ -304,31 +289,31 @@
:limit 10})))))))
(deftest ^:parallel join-against-source-query-test
- (is (= (mt/mbql-query venues
+ (is (= (lib.tu.macros/mbql-query venues
{:joins [{:source-query {:source-table $$categories
:fields [$categories.id
$categories.name]}
:alias "cat"
- :condition [:= $venues.category_id &cat.*ID/BigInteger]
+ :condition [:= $venues.category-id &cat.*ID/BigInteger]
:strategy :left-join}]
:fields [$venues.id
$venues.name
- $venues.category_id
+ $venues.category-id
$venues.latitude
$venues.longitude
$venues.price]
:order-by [[:asc $venues.name]]
:limit 3})
(resolve-joins
- (mt/mbql-query venues
+ (lib.tu.macros/mbql-query venues
{:joins [{:source-query {:source-table $$categories
:fields [$categories.id
$categories.name]}
:alias "cat"
- :condition [:= $venues.category_id &cat.*ID/BigInteger]}]
+ :condition [:= $venues.category-id &cat.*ID/BigInteger]}]
:fields [$venues.id
$venues.name
- $venues.category_id
+ $venues.category-id
$venues.latitude
$venues.longitude
$venues.price]
@@ -337,27 +322,28 @@
(deftest ^:parallel do-not-duplicate-columns-with-default-temporal-bucketing-test
(testing "Do not add a duplicate column from a join if it uses :default temporal bucketing"
- (let [original-query {:fields [[:field (meta/id :orders :id) nil]
- [:field (meta/id :people :birth-date) {:join-alias "P", :temporal-unit :default}]]}]
+ (let [original-stage {:fields [[:field {:lib/uuid "00000000-0000-0000-0000-000000000000"}
+ (meta/id :orders :id)]
+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000001", :join-alias "P", :temporal-unit :default}
+ (meta/id :people :birth-date)]]}]
(doseq [temporal-unit [nil :default]
base-type [nil :type/Date]
effective-type [nil :type/Date]
inherited-temporal-unit [nil :default]
;; make sure random keys don't affect this either
- nonsense-key [nil 1337]
lib-key [nil "PRODUCTS"]
- :let [opts (cond-> {:join-alias "P"}
+ :let [opts (cond-> {:join-alias "P"
+ :lib/uuid "00000000-0000-0000-0000-000000000002"}
temporal-unit (assoc :temporal-unit temporal-unit)
base-type (assoc :base-type base-type)
effective-type (assoc :effective-type effective-type)
inherited-temporal-unit (assoc :inherited-temporal-unit inherited-temporal-unit)
- nonsense-key (assoc :nonsense-key nonsense-key)
lib-key (assoc :lib/nonsense-key lib-key))
- clause [:field (meta/id :people :birth-date) opts]]]
+ clause [:field opts (meta/id :people :birth-date)]]]
(testing (pr-str clause)
- (is (= original-query
+ (is (= original-stage
(resolve-joins/append-join-fields-to-fields
- original-query
+ original-stage
[clause]))))))))
(deftest ^:parallel do-not-duplicate-columns-with-default-temporal-bucketing-e2e-test
diff --git a/test/metabase/query_processor/middleware/resolve_source_table_test.clj b/test/metabase/query_processor/middleware/resolve_source_table_test.clj
index 3d1d1cbbd4c..c3d716c37db 100644
--- a/test/metabase/query_processor/middleware/resolve_source_table_test.clj
+++ b/test/metabase/query_processor/middleware/resolve_source_table_test.clj
@@ -6,8 +6,8 @@
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
- [metabase.query-processor.middleware.resolve-source-table :as qp.resolve-source-table]
- [metabase.test :as mt]))
+ [metabase.lib.test-util.macros :as lib.tu.macros]
+ [metabase.query-processor.middleware.resolve-source-table :as qp.resolve-source-table]))
(defn- cached-metadata
"Fetch the names of all the objects currently in the QP Store."
@@ -15,12 +15,12 @@
(let [tables [:venues :categories :users :checkins]]
{:tables (into #{}
(keep (fn [table]
- (:name (lib.metadata.protocols/cached-metadata metadata-provider :metadata/table (mt/id table)))))
+ (:name (lib.metadata.protocols/cached-metadata metadata-provider :metadata/table (meta/id table)))))
tables)}))
(defn- resolve-and-return-cached-metadata
([query]
- (resolve-and-return-cached-metadata (mt/metadata-provider) query))
+ (resolve-and-return-cached-metadata (lib.metadata.cached-provider/cached-metadata-provider meta/metadata-provider) query))
([metadata-provider query]
(let [query (lib/query metadata-provider query)]
@@ -30,7 +30,7 @@
(deftest ^:parallel basic-test
(testing "does `resolve-source-tables` resolve source tables?"
(is (= {:tables #{"VENUES"}}
- (resolve-and-return-cached-metadata (mt/mbql-query venues))))))
+ (resolve-and-return-cached-metadata (lib.tu.macros/mbql-query venues))))))
(deftest ^:parallel validate-database-test
(testing "If the Table does not belong to the current Database, does it throw an Exception?"
@@ -56,7 +56,7 @@
clojure.lang.ExceptionInfo
#"Invalid output:.*should be a positive int, got: 0"
(resolve-and-return-cached-metadata
- {:database (mt/id)
+ {:database (meta/id)
:type :query
:query {:source-table 0}})))))
@@ -64,49 +64,65 @@
(testing "Does `resolve-source-tables` resolve source tables in nested source queries?"
(is (= {:tables #{"VENUES"}}
(resolve-and-return-cached-metadata
- (mt/mbql-query nil
+ (lib.tu.macros/mbql-query nil
{:source-query {:source-table $$venues}}))))
(is (= {:tables #{"VENUES"}}
(resolve-and-return-cached-metadata
- (mt/mbql-query nil
+ (lib.tu.macros/mbql-query nil
{:source-query {:source-query {:source-table $$venues}}}))))))
(deftest ^:parallel joins-test
(testing "Does `resolve-source-tables` resolve source tables in joins?"
(is (= {:tables #{"CATEGORIES" "VENUES"}}
(resolve-and-return-cached-metadata
- (mt/mbql-query venues
+ (lib.tu.macros/mbql-query venues
{:joins [{:source-table $$categories
:alias "c"
- :condition [:= $category_id &c.categories.id]}]}))))))
+ :condition [:= $category-id &c.categories.id]}]}))))))
(deftest ^:parallel joins-in-nested-queries-test
(testing "Does `resolve-source-tables` resolve source tables in joins inside nested source queries?"
(is (= {:tables #{"CATEGORIES" "VENUES"}}
(resolve-and-return-cached-metadata
- (mt/mbql-query venues
+ (lib.tu.macros/mbql-query venues
{:source-query {:source-table $$venues
:joins [{:source-table $$categories
:alias "c"
- :condition [:= $category_id &c.categories.id]}]}}))))))
+ :condition [:= $category-id &c.categories.id]}]}}))))))
(deftest ^:parallel nested-queries-in-joins-test
(testing "Does `resolve-source-tables` resolve source tables inside nested source queries inside joins?"
(is (= {:tables #{"CATEGORIES" "VENUES"}}
(resolve-and-return-cached-metadata
- (mt/mbql-query venues
+ (lib.tu.macros/mbql-query venues
{:joins [{:source-query {:source-table $$categories}
:alias "c"
- :condition [:= $category_id &c.categories.id]}]}))))))
+ :condition [:= $category-id &c.categories.id]}]}))))))
(deftest ^:parallel nested-queries-in-joins-in-nested-queries-test
(testing (str "Does `resolve-source-tables` resolve source tables inside nested source queries inside joins inside "
"nested source queries?")
(is (= {:tables #{"CATEGORIES" "VENUES"}}
(resolve-and-return-cached-metadata
- (mt/mbql-query venues
+ (lib.tu.macros/mbql-query venues
{:source-query {:source-table $$venues
:joins [{:source-query {:source-table $$categories}
:alias "c"
- :condition [:= $category_id &c.categories.id]}]}}))))))
+ :condition [:= $category-id &c.categories.id]}]}}))))))
+
+(deftest ^:parallel disallow-joins-against-table-on-different-db-test
+ (testing "Test that joining against a table in a different DB throws an Exception"
+ (let [mp (lib.tu/mock-metadata-provider
+ {:database meta/database
+ :tables [(meta/table-metadata :venues)]})]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"\QFailed to fetch :metadata/table\E"
+ (qp.resolve-source-table/resolve-source-tables
+ (lib/query
+ mp
+ (lib.tu.macros/mbql-query venues
+ {:joins [{:source-table (meta/id :categories)
+ :alias "t"
+ :condition [:= $category-id 1]}]}))))))))
diff --git a/test/metabase/query_processor/preprocess_test.clj b/test/metabase/query_processor/preprocess_test.clj
index b3ea18e1ca8..797cc05e96d 100644
--- a/test/metabase/query_processor/preprocess_test.clj
+++ b/test/metabase/query_processor/preprocess_test.clj
@@ -2,7 +2,6 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
- [medley.core :as m]
[metabase.driver :as driver]
[metabase.lib.card :as lib.card]
[metabase.lib.card-test]
@@ -233,21 +232,6 @@
{:source-table "card__2"
:aggregation [[:sum [:field "TOTAL" {:base-type :type/Float}]]]
:breakout [[:field "RATING" {:base-type :type/Float}]]}))]
- (let [preprocessed (qp.preprocess/preprocess query)
- stages (:stages preprocessed)]
- (testing "added metadata"
- (testing "first stage (from Card 1)"
- (is (=? {:name "RATING"
- :display-name "Product → Rating"
- :metabase.lib.join/join-alias "Product"}
- (m/find-first #(= (:name %) "RATING")
- (get-in (nth stages 0) [:lib/stage-metadata :columns])))))
- (testing "second stage (from Card 2)"
- (is (=? {:name "RATING"
- :display-name "Product → Rating"
- :lib/original-join-alias "Product"}
- (m/find-first #(= (:name %) "RATING")
- (get-in (nth stages 1) [:lib/stage-metadata :columns])))))))
(is (=? [{:display_name "Product → Rating"}
{:display_name "Sum of Total"}]
(qp.preprocess/query->expected-cols query))))))
diff --git a/test/metabase/query_processor/util/add_alias_info_test.clj b/test/metabase/query_processor/util/add_alias_info_test.clj
index 12991c88fed..deba39f61fc 100644
--- a/test/metabase/query_processor/util/add_alias_info_test.clj
+++ b/test/metabase/query_processor/util/add_alias_info_test.clj
@@ -1116,8 +1116,12 @@
:aggregation [[:sum $orders.quantity]]}
:alias "Orders"
:condition [:= $id &Orders.orders.product-id]
- ;; we can get title since product_id is remapped to title
- :fields [&Orders.title
+ ;; we can get title since product_id is remapped to title.
+ ;;
+ ;; TODO (Cam 9/16/25) -- says who? This is not something we support in
+ ;; the FE.
+ :fields [[:field %products.title {:join-alias "Orders"
+ :source-field (meta/id :orders :product-id)}]
&Orders.*sum/Integer]}]
:fields [$title $category]
:order-by [[:asc $id]]
diff --git a/test/metabase/query_processor_test/field_ref_repro_test.clj b/test/metabase/query_processor_test/field_ref_repro_test.clj
index 4fdaae48e22..5f075d46056 100644
--- a/test/metabase/query_processor_test/field_ref_repro_test.clj
+++ b/test/metabase/query_processor_test/field_ref_repro_test.clj
@@ -200,8 +200,8 @@
;; The order of these columns seems to be 'flexible' (I would consider either to be correct), and
;; I've seen both in two different branches of mine attempting to fix this bug. The order doesn't
;; matter at all to the FE, so if this changes in the future it's ok. -- Cam
- "j__PEOPLE__via__USER_ID__EMAIL" #_"j__EMAIL"
- "PEOPLE__via__USER_ID__EMAIL"]
+ "PEOPLE__via__USER_ID__EMAIL"
+ "j__PEOPLE__via__USER_ID__EMAIL"]
(map :lib/desired-column-alias (mt/cols results))))
(is (= [[1 ; <= orders.id
1 ; <= orders.user-id
@@ -221,8 +221,8 @@
nil
"2016-12-25T22:19:38.656Z"
2
- "[email protected]" ; (joined) orders.user-id --[remap]--> people.email (email of People row with ID = 1902)
- "[email protected]"] ; orders.user-id --[remap]--> people.email (email of People row with ID = 1)
+ "[email protected]" ; orders.user-id --[remap]--> people.email (email of People row with ID = 1)
+ "[email protected]"] ; (joined) orders.user-id --[remap]--> people.email (email of People row with ID = 1902)
[1
1
14
@@ -241,8 +241,8 @@
nil
"2017-02-04T10:16:00.936Z"
1
(mt/rows results))))))))))
(deftest ^:parallel multi-stage-with-external-remapping-test
diff --git a/test/metabase/query_processor_test/remapping_test.clj b/test/metabase/query_processor_test/remapping_test.clj
index 518624d19de..9d41b9bd24d 100644
--- a/test/metabase/query_processor_test/remapping_test.clj
+++ b/test/metabase/query_processor_test/remapping_test.clj
@@ -335,7 +335,7 @@
:alias "Orders"
:condition [:= $id &Orders.orders.product_id]
;; we can get products.title since orders.product_id is remapped to title
- :fields [&Orders.title
+ :fields [[:field %products.title {:source-field (mt/id :orders :product_id), :join-alias "Orders"}]
&Orders.*sum/Integer]}]
:fields [$title $category]
:order-by [[:asc $id]]
@@ -466,25 +466,21 @@
;;
;; The order of remaps is not important to the FE. If it changes in the future that is ok.
;;
- ;; 2 remaps from the join against `VENUES`
- "J__CATEGORIES__via__ID__NAME"
- "J__CATEGORIES__via__CATEGORY_ID__NAME"
;; 2 remaps for the top-level query
"CATEGORIES__via__CATEGORY_ID__NAME"
"CATEGORIES__via__ID__NAME"
- ;; BROKEN! This is a duplicate and should not be returned. Interestingly enough, both Lib and QP
- ;; incorrectly calculate the set of returned columns and both include this. (Probably because Lib and QP
- ;; use mostly the same code these days.)
- "J__CATEGORIES__via__ID__NAME_2"]
+ ;; 2 remaps from the join against `VENUES`
+ "J__CATEGORIES__via__ID__NAME"
+ "J__CATEGORIES__via__CATEGORY_ID__NAME"]
(map :lib/desired-column-alias (mt/cols results))))
;; The extra incorrect duplicate column seems to be sorta indetermiate? I've seen it match the value of
;; `J__CATEGORIES__via__ID__NAME` and `J__CATEGORIES__via__CATEGORY_ID__NAME` in different test runs and I'm not
;; sure why. Not bothering to debug since it's not even supposed to be returned anyway.
;;
- ;; <top-level :fields> <join> <join remaps> <fields remaps> <incorrect duplicate>
- (is (=? [[11 2 "Stout Burgers & Beers" 1 "Red Medicine" 4 10.0646 -165.374 3 "African" "Asian" "Burger" "American" string?]
- [11 3 "The Apple Pan" 2 "Stout Burgers & Beers" 11 34.0996 -118.329 2 "American" "Burger" "Burger" "Artisan" string?]
- [29 4 "Wurstküche" 3 "The Apple Pan" 11 34.0406 -118.428 2 "Artisan" "Burger" "German" "Asian" string?]]
+ ;; <top-level :fields> <join> <fields remaps> <join remaps>
+ (is (=? [[11 2 "Stout Burgers & Beers" 1 "Red Medicine" 4 10.0646 -165.374 3 "Burger" "American" "African" "Asian"]
+ [11 3 "The Apple Pan" 2 "Stout Burgers & Beers" 11 34.0996 -118.329 2 "Burger" "Artisan" "American" "Burger"]
+ [29 4 "Wurstküche" 3 "The Apple Pan" 11 34.0406 -118.428 2 "German" "Asian" "Artisan" "Burger"]]
(mt/rows results))))))
(deftest ^:parallel explicit-join-with-fields-and-implicitly-joined-remaps-test