Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[QP] Push required implicitly joined fields down through stages (#64305)

Braden Shepherdson33db17c7edc2
diff --git a/src/metabase/lib/walk.cljc b/src/metabase/lib/walk.cljc
index de5e239c2ed..795906def8e 100644
--- a/src/metabase/lib/walk.cljc
+++ b/src/metabase/lib/walk.cljc
@@ -18,7 +18,16 @@
(declare walk-stages*)
-(defn- walk-items* [query path-to-items walk-item-fn f]
+(defn- walk-items-backward* [query path-to-items walk-item-fn f]
+ (u/reduce-preserving-reduced
+ (fn [query absolute-item-number]
+ (let [path-to-item (conj (vec path-to-items) absolute-item-number)]
+ (walk-item-fn query path-to-item f)))
+ query
+ ;; With 3 items, we want (2 1 0), so (range 2 -1 -1).
+ (range (dec (count (get-in query path-to-items []))) -1 -1)))
+
+(defn- walk-items-forward* [query path-to-items walk-item-fn f]
;; negative-item-offset below is a negative index e.g. [-3 -2 -1] rather than normal positive index e.g. [0 1 2] to
;; handle splicing in additional stages/joins if the walk function `f` returns more than one stage/join. The total
;; number of stages/joins might change, but for walking purposes the negative indexes will refer to the same things
@@ -34,34 +43,39 @@
query
(range (- (count (get-in query path-to-items []))) 0 1)))
-(defn- walk-join* [query path-to-join f]
+(defn- walk-items* [query path-to-items walk-item-fn f {:keys [reversed?] :as _opts}]
+ (if reversed?
+ (walk-items-backward* query path-to-items walk-item-fn f)
+ (walk-items-forward* query path-to-items walk-item-fn f)))
+
+(defn- walk-join* [query path-to-join f opts]
(let [path-to-join-stages (conj (vec path-to-join) :stages)
- query' (walk-stages* query path-to-join-stages f)]
+ query' (walk-stages* query path-to-join-stages f opts)]
(if (reduced? query')
query'
(f query' :lib.walk/join path-to-join))))
-(defn- walk-joins* [query path-to-joins f]
- (walk-items* query path-to-joins walk-join* f))
+(defn- walk-joins* [query path-to-joins f opts]
+ (walk-items* query path-to-joins #(walk-join* %1 %2 %3 opts) f opts))
-(defn- walk-stage* [query path-to-stage f]
+(defn- walk-stage* [query path-to-stage f opts]
(let [stage (get-in query path-to-stage)
path-to-joins (conj (vec path-to-stage) :joins)
;; only walk joins in MBQL stages, if someone tries to put them in a native stage ignore them since they're
;; not allowed there anyway.
query' (if (and (= (:lib/type stage :mbql.stage/mbql) :mbql.stage/mbql)
(seq (get-in query path-to-joins)))
- (walk-joins* query path-to-joins f)
+ (walk-joins* query path-to-joins f opts)
query)]
(if (reduced? query')
query'
(f query' :lib.walk/stage path-to-stage))))
-(defn- walk-stages* [query path-to-stages f]
- (walk-items* query path-to-stages walk-stage* f))
+(defn- walk-stages* [query path-to-stages f opts]
+ (walk-items* query path-to-stages #(walk-stage* %1 %2 %3 opts) f opts))
-(defn- walk-query* [query f]
- (walk-stages* query [:stages] f))
+(defn- walk-query* [query f opts]
+ (walk-stages* query [:stages] f opts))
(defn- splice-at-point
"Splice multiple `new-items` into `m` at `path`.
@@ -104,6 +118,10 @@
::walk-stages-fn-result
::lib.schema.join/join]])
+(mr/def ::walk-opts
+ "Schema for options to [[walk]], [[walk-stages]], etc."
+ [:map [:reversed? {:optional true} :boolean]])
+
(mu/defn walk :- ::lib.schema/query
"Depth-first recursive walk and replace for a `query`; call
@@ -134,40 +152,46 @@
(fn [_query _path-type _path stage-or-join]
(when (:source-card stage-or-join)
(reduced true))))))"
- [query :- ::lib.schema/query
- f :- [:=>
- [:cat :map ::path-type ::path ::stage-or-join]
- ::walk-fn-result]]
- (unreduced
- (walk-query*
- query
- (mu/fn [query :- :map ; query can be invalid during the walk process, only needs to be valid again at the end.
- path-type :- ::path-type
- path :- ::path]
- (let [stage-or-join (get-in query path)
- stage-or-join' (or (f query path-type path stage-or-join)
- stage-or-join)]
- (cond
- (reduced? stage-or-join') stage-or-join'
- (identical? stage-or-join' stage-or-join) query
- (sequential? stage-or-join') (splice-at-point query path stage-or-join')
- :else (assoc-in query path stage-or-join')))))))
+ ([query f] (walk query f nil))
+ ([query :- ::lib.schema/query
+ f :- [:=>
+ [:cat :map ::path-type ::path ::stage-or-join]
+ ::walk-fn-result]
+ opts :- [:maybe ::walk-opts]]
+ (unreduced
+ (walk-query*
+ query
+ (mu/fn [query :- :map ; query can be invalid during the walk process, only needs to be valid again at the end.
+ path-type :- ::path-type
+ path :- ::path]
+ (let [stage-or-join (get-in query path)
+ stage-or-join' (or (f query path-type path stage-or-join)
+ stage-or-join)]
+ (cond
+ (reduced? stage-or-join') stage-or-join'
+ (identical? stage-or-join' stage-or-join) query
+ (sequential? stage-or-join') (splice-at-point query path stage-or-join')
+ :else (assoc-in query path stage-or-join'))))
+ opts))))
(mu/defn walk-stages :- ::lib.schema/query
"Like [[walk]], but only walks the stages in a query. `f` is invoked like
(f query path stage) => updated-stage-or-nil"
- [query :- ::lib.schema/query
- f :- [:=> [:cat :map ::path ::lib.schema/stage] ::walk-stages-fn-result]]
- (walk
- query
- (mu/fn :- ::walk-stages-fn-result
- [query :- :map ; don't re-validate query at every step in case we make edits that make it temporarily invalid
- path-type :- ::path-type
- path :- ::path
- stage-or-join :- ::stage-or-join]
- (when (= path-type :lib.walk/stage)
- (f query path stage-or-join)))))
+ ([query f] (walk-stages query f nil))
+ ([query :- ::lib.schema/query
+ f :- [:=> [:cat :map ::path ::lib.schema/stage] ::walk-stages-fn-result]
+ opts :- [:maybe ::walk-opts]]
+ (walk
+ query
+ (mu/fn :- ::walk-stages-fn-result
+ [query :- :map ; don't re-validate query at every step in case we make edits that make it temporarily invalid
+ path-type :- ::path-type
+ path :- ::path
+ stage-or-join :- ::stage-or-join]
+ (when (= path-type :lib.walk/stage)
+ (f query path stage-or-join)))
+ opts)))
(mr/def ::path.stages-part
[:cat
diff --git a/src/metabase/query_processor/middleware/add_implicit_joins.clj b/src/metabase/query_processor/middleware/add_implicit_joins.clj
index 017701e7498..4de2fd26759 100644
--- a/src/metabase/query_processor/middleware/add_implicit_joins.clj
+++ b/src/metabase/query_processor/middleware/add_implicit_joins.clj
@@ -425,4 +425,7 @@
[query :- ::lib.schema/query]
(-> query
(lib.walk/walk-stages first-pass)
- (lib.walk/walk-stages second-pass)))
+ ;; The second pass must go backwards, pushing implicitly joined fields downward until they are resolved.
+ ;; See #63245 and
+ ;; [[metabase.query-processor.middleware.add-implicit-joins-test/implicit-join-from-much-earlier-stage-test]].
+ (lib.walk/walk-stages second-pass {:reversed? true})))
diff --git a/test/metabase/lib/walk_test.cljc b/test/metabase/lib/walk_test.cljc
index 8f18451b425..14f32b9f5d7 100644
--- a/test/metabase/lib/walk_test.cljc
+++ b/test/metabase/lib/walk_test.cljc
@@ -37,6 +37,48 @@
(fn [_query path-type path stage-or-join]
(assoc stage-or-join :path-type path-type, :path path, :order (swap! order inc))))))))
+(deftest ^:parallel walk-stages-reversed-test
+ (let [query {:stages [{:joins [{:stages [{:source-card 1}]}]}
+ {:joins [{:stages [{:source-table 3}
+ {}]}
+ {:stages [{:source-table 4}]}]}]}
+ order (atom -1)]
+ (is (= {:stages [{:joins [{:stages [{:source-card 1
+ :path-type :lib.walk/stage
+ :path [:stages 0 :joins 0 :stages 0]
+ :order 6}]
+ :path-type :lib.walk/join
+ :path [:stages 0 :joins 0]
+ :order 7}]
+ :path-type :lib.walk/stage
+ :path [:stages 0]
+ :order 8}
+ {:joins [{:stages [{:source-table 3
+ :path-type :lib.walk/stage
+ :path [:stages 1 :joins 0 :stages 0]
+ :order 3}
+ {:path-type :lib.walk/stage
+ :path [:stages 1 :joins 0 :stages 1]
+ :order 2}]
+ :path-type :lib.walk/join
+ :path [:stages 1 :joins 0]
+ :order 4}
+ {:stages [{:source-table 4
+ :path-type :lib.walk/stage
+ :path [:stages 1 :joins 1 :stages 0]
+ :order 0}]
+ :path-type :lib.walk/join
+ :path [:stages 1 :joins 1]
+ :order 1}]
+ :path-type :lib.walk/stage
+ :path [:stages 1]
+ :order 5}]}
+ (lib.walk/walk
+ query
+ (fn [_query path-type path stage-or-join]
+ (assoc stage-or-join :path-type path-type, :path path, :order (swap! order inc)))
+ {:reversed? true})))))
+
(deftest ^:parallel reduced-test
(let [query {:stages [{:joins [{:stages [{:source-card 1}]}]}]}
nodes-visited (atom [])]
diff --git a/test/metabase/query_processor/middleware/add_implicit_joins_test.clj b/test/metabase/query_processor/middleware/add_implicit_joins_test.clj
index ed5126fd449..5140868b3b1 100644
--- a/test/metabase/query_processor/middleware/add_implicit_joins_test.clj
+++ b/test/metabase/query_processor/middleware/add_implicit_joins_test.clj
@@ -3,14 +3,18 @@
[clojure.test :refer :all]
[medley.core :as m]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.options :as lib.options]
[metabase.lib.schema.join :as lib.schema.join]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.macros :as lib.tu.macros]
+ [metabase.lib.test-util.metadata-providers.remap :as lib.tu.remap]
[metabase.query-processor :as qp]
[metabase.query-processor.middleware.add-implicit-clauses :as qp.add-implicit-clauses]
[metabase.query-processor.middleware.add-implicit-joins :as qp.add-implicit-joins]
[metabase.query-processor.middleware.fetch-source-query]
+ [metabase.query-processor.preprocess :as qp.preprocess]
[metabase.query-processor.schema :as qp.schema]
[metabase.test :as mt]
[metabase.util :as u]
@@ -1090,3 +1094,34 @@
(is (=? {:joins [{:alias "CATEGORIES__via__CATEGORY_ID"}
{:alias "CATEGORIES__via__ID"}]}
(#'qp.add-implicit-joins/resolve-implicit-joins-this-level query path stage))))))
+
+(deftest ^:parallel implicit-join-from-much-earlier-stage-test
+ (testing "if a join in stage 1 is used in stage 2, the field should propagate through stage 1 (#63245)"
+ (let [;; These remaps are required! Otherwise the implicit join clause is added elsewhere, and #63245 does not
+ ;; come into play. With the remaps, the innermost query on `Orders` includes a join on People so it can remap
+ ;; the `USER_ID` column. Then the implicit join in stage 2 of the test query will reuse that join clause on
+ ;; stage 0, which is the condition which causes #63245.
+ mp (lib.tu.remap/remap-metadata-provider
+ meta/metadata-provider
+ (meta/id :orders :user-id) (meta/id :people :name)
+ (meta/id :orders :product-id) (meta/id :products :title))
+ ordersQ (lib/query mp (meta/table-metadata :orders))
+ mp (lib.tu/metadata-provider-with-card-from-query mp 1 ordersQ)
+ ordersQ+peopleT (-> (lib/query mp (lib.metadata/card mp 1))
+ (lib/join (meta/table-metadata :people)))
+ mp (lib.tu/metadata-provider-with-card-from-query mp 2 ordersQ+peopleT)
+ query (-> (lib/query mp (lib.metadata/card mp 2))
+ (lib/aggregate (lib/count))
+ (lib/breakout (lib.options/ensure-uuid
+ [:field {:source-field (meta/id :orders :user-id)
+ :source-field-name "USER_ID"}
+ (meta/id :people :state)])))
+ preprocessed (qp.preprocess/preprocess query)]
+ (is (=? {:stages [{:fields [any? any? any? any? any? any? any? any? any? any? any?
+ [:field {:join-alias "PEOPLE__via__USER_ID"
+ :source-field (meta/id :orders :user-id)
+ :source-field-name "USER_ID"}
+ (meta/id :people :state)]]}
+ {}
+ {}]}
+ preprocessed)))))
diff --git a/test/metabase/query_processor_test/field_ref_repro_test.clj b/test/metabase/query_processor_test/field_ref_repro_test.clj
index 5f075d46056..7839ef3994d 100644
--- a/test/metabase/query_processor_test/field_ref_repro_test.clj
+++ b/test/metabase/query_processor_test/field_ref_repro_test.clj
@@ -263,15 +263,13 @@
(is (= [[1746]]
(mt/rows results))))))))
-;;; This one is a really tricky one to solve, the problem is that the implicit join happens in the first stage (Card 1)
-;;; because of the remappped column, but the parameter asks to be applied to stage 3... it's too late to add a new
-;;; filter against `PEOPLE__via__USER_ID.STATE` at that point because it doesn't come back from stage 1 or 2 (this is
-;;; why resolution trips up and falls back to `source.STATE`. I think the only way to fix this would be to make the
-;;; parameter logic apply the parameter to the correct stage (ignoring `:stage-number` when it's wrong) or add another
-;;; duplicate implicit join in stage 3 to power the filter
+;;; This is tricky case: the implicit join happens in stage 0 (Card 1) because of the remapped column, but the parameter
+;;; asks to be applied to stage 2. Logic in `add-implicit-joins` now propagates that use of the implicitly joined column
+;;; through stage 1 to stage 0, making the column available for the filter in the last stage.
;;;
-;;; See
-;;; also [[metabase.lib.field.resolution-test/resolve-unreturned-column-from-reified-implicit-join-in-previous-stage-test]]
+;;; See also
+;;; [[metabase.lib.field.resolution-test/resolve-unreturned-column-from-reified-implicit-join-in-previous-stage-test]]
+;;; and [[metabase.query-processor.middleware.add-implicit-joins-test/implicit-join-from-much-earlier-stage-test]].
(deftest ^:parallel model-with-implicit-join-and-external-remapping-test
(testing "Should handle models with implicit join on externally remapped field (#57596)"
(qp.store/with-metadata-provider (lib.tu/remap-metadata-provider
@@ -293,9 +291,6 @@
:source-field (mt/id :orders :user_id)
:source-field-name "USER_ID"}]
{:stage-number -1}]}]))]
- ;; should return 613 rows
(mt/with-native-query-testing-context query
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Column \"source\.PEOPLE__via__USER_ID__STATE\" not found"
- (-> query qp/process-query mt/rows count))))))))
+ (is (= 613
+ (-> query qp/process-query mt/rows count))))))))