The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[QP] Push required implicitly joined fields down through stages (#64305)
Braden Shepherdson33db17c7edc2
diff --git a/src/metabase/lib/walk.cljc b/src/metabase/lib/walk.cljcindex de5e239c2ed..795906def8e 100644--- a/src/metabase/lib/walk.cljc+++ b/src/metabase/lib/walk.cljc@@ -18,7 +18,16 @@(declare walk-stages*)-(defn- walk-items* [query path-to-items walk-item-fn f]+(defn- walk-items-backward* [query path-to-items walk-item-fn f]+ (u/reduce-preserving-reduced+ (fn [query absolute-item-number]+ (let [path-to-item (conj (vec path-to-items) absolute-item-number)]+ (walk-item-fn query path-to-item f)))+ query+ ;; With 3 items, we want (2 1 0), so (range 2 -1 -1).+ (range (dec (count (get-in query path-to-items []))) -1 -1)))++(defn- walk-items-forward* [query path-to-items walk-item-fn f];; negative-item-offset below is a negative index e.g. [-3 -2 -1] rather than normal positive index e.g. [0 1 2] to;; handle splicing in additional stages/joins if the walk function `f` returns more than one stage/join. The total;; number of stages/joins might change, but for walking purposes the negative indexes will refer to the same things@@ -34,34 +43,39 @@query(range (- (count (get-in query path-to-items []))) 0 1)))-(defn- walk-join* [query path-to-join f]+(defn- walk-items* [query path-to-items walk-item-fn f {:keys [reversed?] :as _opts}]+ (if reversed?+ (walk-items-backward* query path-to-items walk-item-fn f)+ (walk-items-forward* query path-to-items walk-item-fn f)))++(defn- walk-join* [query path-to-join f opts](let [path-to-join-stages (conj (vec path-to-join) :stages)- query' (walk-stages* query path-to-join-stages f)]+ query' (walk-stages* query path-to-join-stages f opts)](if (reduced? query')query'(f query' :lib.walk/join path-to-join))))-(defn- walk-joins* [query path-to-joins f]- (walk-items* query path-to-joins walk-join* f))+(defn- walk-joins* [query path-to-joins f opts]+ (walk-items* query path-to-joins #(walk-join* %1 %2 %3 opts) f opts))-(defn- walk-stage* [query path-to-stage f]+(defn- walk-stage* [query path-to-stage f opts](let [stage (get-in query path-to-stage)path-to-joins (conj (vec path-to-stage) :joins);; only walk joins in MBQL stages, if someone tries to put them in a native stage ignore them since they're;; not allowed there anyway.query' (if (and (= (:lib/type stage :mbql.stage/mbql) :mbql.stage/mbql)(seq (get-in query path-to-joins)))- (walk-joins* query path-to-joins f)+ (walk-joins* query path-to-joins f opts)query)](if (reduced? query')query'(f query' :lib.walk/stage path-to-stage))))-(defn- walk-stages* [query path-to-stages f]- (walk-items* query path-to-stages walk-stage* f))+(defn- walk-stages* [query path-to-stages f opts]+ (walk-items* query path-to-stages #(walk-stage* %1 %2 %3 opts) f opts))-(defn- walk-query* [query f]- (walk-stages* query [:stages] f))+(defn- walk-query* [query f opts]+ (walk-stages* query [:stages] f opts))(defn- splice-at-point"Splice multiple `new-items` into `m` at `path`.@@ -104,6 +118,10 @@::walk-stages-fn-result::lib.schema.join/join]])+(mr/def ::walk-opts+ "Schema for options to [[walk]], [[walk-stages]], etc."+ [:map [:reversed? {:optional true} :boolean]])+(mu/defn walk :- ::lib.schema/query"Depth-first recursive walk and replace for a `query`; call@@ -134,40 +152,46 @@(fn [_query _path-type _path stage-or-join](when (:source-card stage-or-join)(reduced true))))))"- [query :- ::lib.schema/query- f :- [:=>- [:cat :map ::path-type ::path ::stage-or-join]- ::walk-fn-result]]- (unreduced- (walk-query*- query- (mu/fn [query :- :map ; query can be invalid during the walk process, only needs to be valid again at the end.- path-type :- ::path-type- path :- ::path]- (let [stage-or-join (get-in query path)- stage-or-join' (or (f query path-type path stage-or-join)- stage-or-join)]- (cond- (reduced? stage-or-join') stage-or-join'- (identical? stage-or-join' stage-or-join) query- (sequential? stage-or-join') (splice-at-point query path stage-or-join')- :else (assoc-in query path stage-or-join')))))))+ ([query f] (walk query f nil))+ ([query :- ::lib.schema/query+ f :- [:=>+ [:cat :map ::path-type ::path ::stage-or-join]+ ::walk-fn-result]+ opts :- [:maybe ::walk-opts]]+ (unreduced+ (walk-query*+ query+ (mu/fn [query :- :map ; query can be invalid during the walk process, only needs to be valid again at the end.+ path-type :- ::path-type+ path :- ::path]+ (let [stage-or-join (get-in query path)+ stage-or-join' (or (f query path-type path stage-or-join)+ stage-or-join)]+ (cond+ (reduced? stage-or-join') stage-or-join'+ (identical? stage-or-join' stage-or-join) query+ (sequential? stage-or-join') (splice-at-point query path stage-or-join')+ :else (assoc-in query path stage-or-join'))))+ opts))))(mu/defn walk-stages :- ::lib.schema/query"Like [[walk]], but only walks the stages in a query. `f` is invoked like(f query path stage) => updated-stage-or-nil"- [query :- ::lib.schema/query- f :- [:=> [:cat :map ::path ::lib.schema/stage] ::walk-stages-fn-result]]- (walk- query- (mu/fn :- ::walk-stages-fn-result- [query :- :map ; don't re-validate query at every step in case we make edits that make it temporarily invalid- path-type :- ::path-type- path :- ::path- stage-or-join :- ::stage-or-join]- (when (= path-type :lib.walk/stage)- (f query path stage-or-join)))))+ ([query f] (walk-stages query f nil))+ ([query :- ::lib.schema/query+ f :- [:=> [:cat :map ::path ::lib.schema/stage] ::walk-stages-fn-result]+ opts :- [:maybe ::walk-opts]]+ (walk+ query+ (mu/fn :- ::walk-stages-fn-result+ [query :- :map ; don't re-validate query at every step in case we make edits that make it temporarily invalid+ path-type :- ::path-type+ path :- ::path+ stage-or-join :- ::stage-or-join]+ (when (= path-type :lib.walk/stage)+ (f query path stage-or-join)))+ opts)))(mr/def ::path.stages-part[:catdiff --git a/src/metabase/query_processor/middleware/add_implicit_joins.clj b/src/metabase/query_processor/middleware/add_implicit_joins.cljindex 017701e7498..4de2fd26759 100644--- a/src/metabase/query_processor/middleware/add_implicit_joins.clj+++ b/src/metabase/query_processor/middleware/add_implicit_joins.clj@@ -425,4 +425,7 @@[query :- ::lib.schema/query](-> query(lib.walk/walk-stages first-pass)- (lib.walk/walk-stages second-pass)))+ ;; The second pass must go backwards, pushing implicitly joined fields downward until they are resolved.+ ;; See #63245 and+ ;; [[metabase.query-processor.middleware.add-implicit-joins-test/implicit-join-from-much-earlier-stage-test]].+ (lib.walk/walk-stages second-pass {:reversed? true})))diff --git a/test/metabase/lib/walk_test.cljc b/test/metabase/lib/walk_test.cljcindex 8f18451b425..14f32b9f5d7 100644--- a/test/metabase/lib/walk_test.cljc+++ b/test/metabase/lib/walk_test.cljc@@ -37,6 +37,48 @@(fn [_query path-type path stage-or-join](assoc stage-or-join :path-type path-type, :path path, :order (swap! order inc))))))))+(deftest ^:parallel walk-stages-reversed-test+ (let [query {:stages [{:joins [{:stages [{:source-card 1}]}]}+ {:joins [{:stages [{:source-table 3}+ {}]}+ {:stages [{:source-table 4}]}]}]}+ order (atom -1)]+ (is (= {:stages [{:joins [{:stages [{:source-card 1+ :path-type :lib.walk/stage+ :path [:stages 0 :joins 0 :stages 0]+ :order 6}]+ :path-type :lib.walk/join+ :path [:stages 0 :joins 0]+ :order 7}]+ :path-type :lib.walk/stage+ :path [:stages 0]+ :order 8}+ {:joins [{:stages [{:source-table 3+ :path-type :lib.walk/stage+ :path [:stages 1 :joins 0 :stages 0]+ :order 3}+ {:path-type :lib.walk/stage+ :path [:stages 1 :joins 0 :stages 1]+ :order 2}]+ :path-type :lib.walk/join+ :path [:stages 1 :joins 0]+ :order 4}+ {:stages [{:source-table 4+ :path-type :lib.walk/stage+ :path [:stages 1 :joins 1 :stages 0]+ :order 0}]+ :path-type :lib.walk/join+ :path [:stages 1 :joins 1]+ :order 1}]+ :path-type :lib.walk/stage+ :path [:stages 1]+ :order 5}]}+ (lib.walk/walk+ query+ (fn [_query path-type path stage-or-join]+ (assoc stage-or-join :path-type path-type, :path path, :order (swap! order inc)))+ {:reversed? true})))))+(deftest ^:parallel reduced-test(let [query {:stages [{:joins [{:stages [{:source-card 1}]}]}]}nodes-visited (atom [])]diff --git a/test/metabase/query_processor/middleware/add_implicit_joins_test.clj b/test/metabase/query_processor/middleware/add_implicit_joins_test.cljindex ed5126fd449..5140868b3b1 100644--- a/test/metabase/query_processor/middleware/add_implicit_joins_test.clj+++ b/test/metabase/query_processor/middleware/add_implicit_joins_test.clj@@ -3,14 +3,18 @@[clojure.test :refer :all][medley.core :as m][metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.options :as lib.options][metabase.lib.schema.join :as lib.schema.join][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros]+ [metabase.lib.test-util.metadata-providers.remap :as lib.tu.remap][metabase.query-processor :as qp][metabase.query-processor.middleware.add-implicit-clauses :as qp.add-implicit-clauses][metabase.query-processor.middleware.add-implicit-joins :as qp.add-implicit-joins][metabase.query-processor.middleware.fetch-source-query]+ [metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.schema :as qp.schema][metabase.test :as mt][metabase.util :as u]@@ -1090,3 +1094,34 @@(is (=? {:joins [{:alias "CATEGORIES__via__CATEGORY_ID"}{:alias "CATEGORIES__via__ID"}]}(#'qp.add-implicit-joins/resolve-implicit-joins-this-level query path stage))))))++(deftest ^:parallel implicit-join-from-much-earlier-stage-test+ (testing "if a join in stage 1 is used in stage 2, the field should propagate through stage 1 (#63245)"+ (let [;; These remaps are required! Otherwise the implicit join clause is added elsewhere, and #63245 does not+ ;; come into play. With the remaps, the innermost query on `Orders` includes a join on People so it can remap+ ;; the `USER_ID` column. Then the implicit join in stage 2 of the test query will reuse that join clause on+ ;; stage 0, which is the condition which causes #63245.+ mp (lib.tu.remap/remap-metadata-provider+ meta/metadata-provider+ (meta/id :orders :user-id) (meta/id :people :name)+ (meta/id :orders :product-id) (meta/id :products :title))+ ordersQ (lib/query mp (meta/table-metadata :orders))+ mp (lib.tu/metadata-provider-with-card-from-query mp 1 ordersQ)+ ordersQ+peopleT (-> (lib/query mp (lib.metadata/card mp 1))+ (lib/join (meta/table-metadata :people)))+ mp (lib.tu/metadata-provider-with-card-from-query mp 2 ordersQ+peopleT)+ query (-> (lib/query mp (lib.metadata/card mp 2))+ (lib/aggregate (lib/count))+ (lib/breakout (lib.options/ensure-uuid+ [:field {:source-field (meta/id :orders :user-id)+ :source-field-name "USER_ID"}+ (meta/id :people :state)])))+ preprocessed (qp.preprocess/preprocess query)]+ (is (=? {:stages [{:fields [any? any? any? any? any? any? any? any? any? any? any?+ [:field {:join-alias "PEOPLE__via__USER_ID"+ :source-field (meta/id :orders :user-id)+ :source-field-name "USER_ID"}+ (meta/id :people :state)]]}+ {}+ {}]}+ preprocessed)))))diff --git a/test/metabase/query_processor_test/field_ref_repro_test.clj b/test/metabase/query_processor_test/field_ref_repro_test.cljindex 5f075d46056..7839ef3994d 100644--- a/test/metabase/query_processor_test/field_ref_repro_test.clj+++ b/test/metabase/query_processor_test/field_ref_repro_test.clj@@ -263,15 +263,13 @@(is (= [[1746]](mt/rows results))))))))-;;; This one is a really tricky one to solve, the problem is that the implicit join happens in the first stage (Card 1)-;;; because of the remappped column, but the parameter asks to be applied to stage 3... it's too late to add a new-;;; filter against `PEOPLE__via__USER_ID.STATE` at that point because it doesn't come back from stage 1 or 2 (this is-;;; why resolution trips up and falls back to `source.STATE`. I think the only way to fix this would be to make the-;;; parameter logic apply the parameter to the correct stage (ignoring `:stage-number` when it's wrong) or add another-;;; duplicate implicit join in stage 3 to power the filter+;;; This is tricky case: the implicit join happens in stage 0 (Card 1) because of the remapped column, but the parameter+;;; asks to be applied to stage 2. Logic in `add-implicit-joins` now propagates that use of the implicitly joined column+;;; through stage 1 to stage 0, making the column available for the filter in the last stage.;;;-;;; See-;;; also [[metabase.lib.field.resolution-test/resolve-unreturned-column-from-reified-implicit-join-in-previous-stage-test]]+;;; See also+;;; [[metabase.lib.field.resolution-test/resolve-unreturned-column-from-reified-implicit-join-in-previous-stage-test]]+;;; and [[metabase.query-processor.middleware.add-implicit-joins-test/implicit-join-from-much-earlier-stage-test]].(deftest ^:parallel model-with-implicit-join-and-external-remapping-test(testing "Should handle models with implicit join on externally remapped field (#57596)"(qp.store/with-metadata-provider (lib.tu/remap-metadata-provider@@ -293,9 +291,6 @@:source-field (mt/id :orders :user_id):source-field-name "USER_ID"}]{:stage-number -1}]}]))]- ;; should return 613 rows(mt/with-native-query-testing-context query- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"Column \"source\.PEOPLE__via__USER_ID__STATE\" not found"- (-> query qp/process-query mt/rows count))))))))+ (is (= 613+ (-> query qp/process-query mt/rows count))))))))