The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Apply chain filter constraints behind one-to-many joins as semi-joins (#83955)
Christoph Deil1f48f93db277
diff --git a/src/metabase/parameters/chain_filter.clj b/src/metabase/parameters/chain_filter.cljindex dedd6bfdf9c..0a03948e87c 100644--- a/src/metabase/parameters/chain_filter.clj+++ b/src/metabase/parameters/chain_filter.clj@@ -74,7 +74,7 @@[metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.types.isa :as lib.types.isa]- [metabase.lib.util :as lib.util]+ [metabase.lib.walk :as lib.walk][metabase.parameters.chain-filter.dedupe-joins :as dedupe][metabase.parameters.db :as parameters.db][metabase.parameters.field-values :as params.field-values]@@ -415,7 +415,9 @@join's source table.Assumes no two joins in stage 0 target the same `:table-id` (true for chain-filter's `joined-table-alias`- convention). If that ever breaks, both joins get the union of fields — still correct, only over-projection."+ convention). If that ever breaks, both joins get the union of fields — still correct, only over-projection.++ Semi-joins (see [[add-semi-joins]]) are left alone: their sub-select returns only its breakout column already."[query :- ::lib.schema/query](if (empty? (lib/joins query))query@@ -424,17 +426,170 @@(let [field-ids (lib/all-field-ids query)cols (lib.metadata/bulk-metadata query :metadata/column field-ids)cols-by-tid (group-by :table-id cols)]- (lib.util/update-query-stage- query 0- update :joins- (fn [the-joins]- (mapv (fn [a-join]- (let [thing (lib/joined-thing query a-join)- tid (when (= :metadata/table (:lib/type thing))+ (lib.walk/walk+ query+ (fn [query path-type path a-join]+ (when (and (= path-type :lib.walk/join)+ ;; only the joins of the first stage: the joins inside a semi-join's sub-select were tightened+ ;; when the sub-select was built+ (= (lib.walk/join-parent-stage-path path) [:stages 0]))+ (let [thing (lib/joined-thing query a-join)+ summarized? (seq (lib.walk/apply-f-for-stage-at-path+ lib/breakouts query (lib.walk/join-last-stage-path path a-join)))+ tid (when (and (= :metadata/table (:lib/type thing))+ (not summarized?))(:id thing))]- (cond-> a-join- tid (lib/with-join-source-fields (get cols-by-tid tid)))))- the-joins))))))+ (when tid+ (lib/with-join-source-fields a-join (get cols-by-tid tid))))))))))++(defn- one-to-many-join?+ "Whether `join-info` follows an FK backwards, from the Field it references to the referencing Field, so that one row+ on the left-hand side can match many rows on the right-hand side."+ [id->field {{lhs-field-id :field} :lhs, {rhs-field-id :field} :rhs}]+ (not= rhs-field-id (:fk-target-field-id (id->field lhs-field-id))))++(defn- join-path+ "The joins that lead from `source-table-id` to `table-id`, in order, or nil if there is no path. Where several FKs+ link two Tables, prefer one of `field-ids`."+ [database-id source-table-id table-id field-ids]+ (when-let [joins (not-empty (find-joins database-id source-table-id table-id))]+ (dedupe/dedupe-joins source-table-id field-ids joins #{table-id})))++(mr/def ::one-to-many-path+ "A join path split after its first one-to-many join."+ [:map {:closed true}+ [:prefix [:sequential ::join-info]]+ [:suffix [:sequential ::join-info]]])++(mu/defn- one-to-many-paths :- [:map-of ::lib.schema.id/table ::one-to-many-path]+ "The join paths from `source-table-id` to those Tables of `field-ids` that can only be reached by following an FK+ backwards somewhere, keyed by Table ID."+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ database-id :- ::lib.schema.id/database+ source-table-id :- ::lib.schema.id/table+ field-ids :- [:set ::lib.schema.id/field]]+ (let [table-ids (when (seq field-ids)+ (disj (into #{}+ (map :table-id)+ (lib.metadata/bulk-metadata metadata-providerable :metadata/column field-ids))+ source-table-id))+ paths (into {}+ (keep (fn [table-id]+ (when-let [path (join-path database-id source-table-id table-id field-ids)]+ [table-id path])))+ table-ids)+ id->field (when (seq paths)+ (u/index-by :id (lib.metadata/bulk-metadata metadata-providerable :metadata/column+ (into #{}+ (comp cat+ (mapcat (juxt #(get-in % [:lhs :field])+ #(get-in % [:rhs :field]))))+ (vals paths)))))]+ (into {}+ (keep (fn [[table-id path]]+ (let [[to-one [one-to-many & suffix]] (split-with #(not (one-to-many-join? id->field %)) path)]+ (when one-to-many+ [table-id {:prefix (conj (vec to-one) one-to-many)+ :suffix (vec suffix)}]))))+ paths)))++(mu/defn- semi-join-clause :- :metabase.lib.schema.join/join+ "A join against a sub-select that applies the `constraints` on the Tables with `table-ids` and returns the distinct+ values of the column the source Table joins against. `paths` lead from the source Table to these Tables and share+ their `:prefix`."+ [query :- ::lib.schema/query+ join-alias :- ::lib.schema.common/non-blank-string+ paths :- [:sequential ::one-to-many-path]+ table-ids :- [:set ::lib.schema.id/table]+ constraints :- [:maybe ::constraints]]+ (let [{{lhs-field-id :field} :lhs+ {rhs-table-id :table, rhs-field-id :field} :rhs} (first (:prefix (first paths)))+ field-ids (into #{} (map :field-id) constraints)+ ;; the sub-select starts at the Table the first join leads to+ joins (dedupe/dedupe-joins rhs-table-id+ field-ids+ (into [] (comp (mapcat #(concat (rest (:prefix %)) (:suffix %)))+ (distinct))+ paths)+ (disj table-ids rhs-table-id))+ sub-select (-> (lib/query query (lib.metadata/table query rhs-table-id))+ (add-joins rhs-table-id joins)+ (lib/breakout (lib.metadata/field query rhs-field-id))+ (add-filters rhs-table-id (set (map #(get-in % [:rhs :table]) joins)) constraints)+ schema.metadata-queries/add-required-filters-if-needed+ tighten-join-projections)+ rhs-field (lib/with-join-alias (lib.metadata/field query rhs-field-id) join-alias)]+ (log/tracef "Adding semi-join %s against %s for %s"+ join-alias+ (name-for-logging :model/Table rhs-table-id)+ (str/join ", " (map (partial name-for-logging :model/Table) (sort table-ids))))+ (-> (lib/join-clause sub-select)+ (lib/with-join-alias join-alias)+ (lib/with-join-conditions [(lib/= (lib.metadata/field query lhs-field-id) rhs-field)])+ (lib/with-join-strategy :inner-join))))++(mu/defn- add-semi-joins :- ::lib.schema/query+ "Constrain `query` by the `constraints` on the Tables in `paths` through semi-joins.++ Joining a Table through a one-to-many join repeats each row of the source Table once per matching row, and joining+ several such Tables multiplies these repetitions. With a locked embedding parameter that is mapped to a column of+ every card's Table, this cross product can grow to billions of rows before `GROUP BY` reduces it to at most+ [[max-results]] values.++ So each of these Tables is joined in a sub-select instead, which applies its constraints and returns the distinct+ values of the column the source Table joins against, e.g.++ INNER JOIN (SELECT venue_id FROM checkins WHERE ... GROUP BY venue_id) semi_join_1+ ON venues.id = semi_join_1.venue_id++ Such a join matches each source row at most once. Tables whose paths share everything up to and including their+ first one-to-many join are constrained in the same sub-select, so they still have to match through the same row, as+ in a single query. Paths that part before that join meet only in rows the source row determines, so constraining+ them separately returns the same values."+ [query :- ::lib.schema/query+ paths :- [:map-of ::lib.schema.id/table ::one-to-many-path]+ constraints :- [:maybe ::constraints]]+ (if (empty? paths)+ query+ (let [field-id->table-id (into {}+ (map (juxt :id :table-id))+ (lib.metadata/bulk-metadata query :metadata/column+ (into #{} (map :field-id) constraints)))+ table-id-groups (->> (group-by (comp :prefix paths) (keys paths))+ vals+ (map set)+ (sort-by (comp vec sort)))]+ (reduce (fn [query [i table-ids]]+ (lib/join query (semi-join-clause query+ (format "semi_join_%d" (inc i))+ (map paths table-ids)+ table-ids+ (filter #(contains? table-ids (field-id->table-id (:field-id %)))+ constraints))))+ query+ (map-indexed vector table-id-groups)))))++(mu/defn- constraint-joins :- [:map {:closed true}+ [:joins [:maybe [:sequential ::join-info]]]+ [:semi-join-paths [:map-of ::lib.schema.id/table ::one-to-many-path]]]+ "Plan how a query on `source-table-id` reaches the Fields of `constraints` and the additional `field-ids`. `:joins`+ are ordinary joins for [[add-joins]]; `:semi-join-paths` lead to the Tables whose constraints [[add-semi-joins]]+ applies, i.e. those reached only through a one-to-many join and not joined anyway."+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ database-id :- ::lib.schema.id/database+ source-table-id :- ::lib.schema.id/table+ constraints :- [:maybe ::constraints]+ field-ids :- [:set ::lib.schema.id/field]]+ (let [constraint-field-ids (into #{} (map :field-id) constraints)+ paths (one-to-many-paths metadata-providerable database-id source-table-id constraint-field-ids)+ direct-field-ids (into field-ids+ (comp (remove #(contains? paths (:table-id %)))+ (map :id))+ (lib.metadata/bulk-metadata metadata-providerable :metadata/column+ constraint-field-ids))+ joins (find-all-joins metadata-providerable database-id source-table-id direct-field-ids)]+ {:joins joins+ :semi-join-paths (apply dissoc paths (map #(get-in % [:rhs :table]) joins))}))(mr/def ::options;; if original-field-id is specified, we'll include this in the results. For Field->Field remapping.@@ -463,10 +618,10 @@;; engines like ClickHouse that materialize the right side in memory.reversed? (and original-table-id (not= original-table-id field-table-id))source-table-id (if reversed? original-table-id field-table-id)- joins (find-all-joins mp database-id source-table-id- (cond-> (set (map :field-id constraints))- reversed? (conj field-id)- (not reversed?) (cond-> original-field-id (conj original-field-id))))+ remap-field-ids (cond-> #{}+ reversed? (conj field-id)+ (not reversed?) (cond-> original-field-id (conj original-field-id)))+ {:keys [joins semi-join-paths]} (constraint-joins mp database-id source-table-id constraints remap-field-ids)joined-table-ids (set (map #(get-in % [:rhs :table]) joins))field (cond-> (lib.metadata/field mp field-id)reversed? (lib/with-join-alias (joined-table-alias field-table-id)))@@ -489,6 +644,7 @@(lib/limit ((fnil min Integer/MAX_VALUE) limit max-results))(assoc-in [:middleware :disable-remaps?] true)(add-joins source-table-id joins)+ (add-semi-joins semi-join-paths constraints)(cond-> original-field (->;; don't return rows that don't have values for the original Field. e.g. if;; venues.category_id is remapped to categories.name and we do a search with query@@ -687,7 +843,7 @@(let [database-id (field/field-id->database-id field-id)mp (lib-be/application-database-metadata-provider database-id)source-table-id (:table-id (lib.metadata/field mp field-id))- joins (find-all-joins mp database-id source-table-id (set (map :field-id constraints)))+ {:keys [joins semi-join-paths]} (constraint-joins mp database-id source-table-id constraints #{})joined-table-ids (set (map #(get-in % [:rhs :table]) joins))field (lib.metadata/field mp field-id)](when (seq joins)@@ -696,6 +852,7 @@(-> (lib/query mp (lib.metadata/table mp source-table-id))(assoc-in [:middleware :disable-remaps?] true)(add-joins source-table-id joins)+ (add-semi-joins semi-join-paths constraints)(lib/aggregate (lib/min field))(lib/aggregate (lib/max field))(lib/aggregate (lib/distinct field))@@ -860,5 +1017,12 @@(let [mbql-query (chain-filter-mbql-query field-id(for [id filter-field-ids]{:field-id id, :op :=, :value nil})- nil)]- (into #{} (mapcat lib/all-field-ids) (lib/filters mbql-query)))))+ nil)+ ;; constraints on Tables behind one-to-many joins are filters of the semi-joins' sub-selects, so collect the+ ;; filters of every stage, including the stages inside joins+ filters (volatile! [])]+ (lib.walk/walk-stages mbql-query+ (fn [query path _stage]+ (vswap! filters into (lib.walk/apply-f-for-stage-at-path lib/filters query path))+ nil))+ (into #{} (mapcat lib/all-field-ids) @filters))))diff --git a/test/metabase/parameters/chain_filter_test.clj b/test/metabase/parameters/chain_filter_test.cljindex 25bd223cebb..fa206366c00 100644--- a/test/metabase/parameters/chain_filter_test.clj+++ b/test/metabase/parameters/chain_filter_test.clj@@ -57,6 +57,20 @@[n result](update result :values #(take n %)))+(defn- join-source-queries+ "The queries joined by the joins of `query`'s first stage, in join order, each as a query of its own."+ [query]+ (for [join-number (range (count (lib/joins query 0)))]+ (:query (lib.walk/query-for-path query [:stages 0 :joins join-number]))))++(defn- join-summaries+ "For each join of `query`'s first stage: its strategy, and the field IDs its source query breaks out and filters by."+ [query]+ (for [[a-join source-query] (map vector (lib/joins query 0) (join-source-queries query))]+ {:strategy (lib/raw-join-strategy a-join)+ :breakout (into #{} (mapcat lib/all-field-ids) (lib/breakouts source-query 0))+ :filters (into #{} (mapcat lib/all-field-ids) (lib/filters source-query 0))}))+(deftest ^:parallel special-form-filter-clause-test(testing "Can handle multi-arg string filters when chaining (#57287)"(let [mp (mt/metadata-provider)]@@ -138,6 +152,57 @@(is (= {:values [["Festa"] ["Fred 62"]], :has_more_values false}(chain-filter venues.name {venues.price [:between 2 3], venues.name [:starts-with "f" {:case-sensitive false}]})))))+(deftest ^:parallel one-to-many-constraints-semi-join-test+ (testing (str "Constraints on Tables reached through one-to-many joins are applied in de-duplicated sub-selects, "+ "so the joins can't multiply the source Table's rows")+ (mt/dataset test-data+ (mt/$ids+ (let [query (#'chain-filter/chain-filter-mbql-query %products.category+ [(shorthand->constraint %orders.user_id 1)+ (shorthand->constraint %reviews.rating 1)]+ nil)]+ (is (= #{{:strategy :inner-join, :breakout #{%orders.product_id}, :filters #{%orders.user_id}}+ {:strategy :inner-join, :breakout #{%reviews.product_id}, :filters #{%reviews.rating}}}+ (set (join-summaries query))))+ (is (nil? (lib/filters query)))+ (is (= {:values [["Doohickey"] ["Widget"]], :has_more_values false}+ (chain-filter products.category {orders.user_id 1, reviews.rating 1}))))))))++(deftest ^:parallel one-to-many-constraints-shared-join-test+ (testing (str "Constraints whose paths share their first one-to-many join are applied in one sub-select, so they "+ "still have to match the same row of it")+ (mt/dataset test-data+ (mt/$ids+ (let [query (#'chain-filter/chain-filter-mbql-query+ %venues.name+ [(shorthand->constraint %users.name "Plato Yeshua")+ (shorthand->constraint %checkins.date [:between "2015-01-01" "2015-06-30"])]+ nil)]+ (is (= [#{%users.name %checkins.date}]+ (map :filters (join-summaries query))))+ (testing "venues with a check-in by Plato Yeshua in the first half of 2015"+ (is (= {:values [["Barney's Beanery"]+ ["Cha Cha Chicken"]+ ["Don Day Korean Restaurant"]+ ["Red Medicine"]+ ["Shanghai Dumpling King"]]+ :has_more_values false}+ (chain-filter venues.name {users.name "Plato Yeshua"+ checkins.date [:between "2015-01-01" "2015-06-30"]})))))))))++(deftest ^:parallel one-to-many-constraints-range-test+ (testing "The range query applies constraints on Tables reached through one-to-many joins as semi-joins too"+ (mt/dataset test-data+ (mt/$ids+ (let [query (#'chain-filter/chain-filter-range-mbql-query %products.price+ [(shorthand->constraint %orders.user_id 1)])]+ (is (= [{:strategy :inner-join, :breakout #{%orders.product_id}, :filters #{%orders.user_id}}]+ (join-summaries query)))+ (is (nil? (lib/filters query)))+ (testing "min, max and distinct count of the prices of the Products that User 1 ordered"+ (is (= {:min 19.87, :max 85.25, :distinct-count 11}+ (chain-filter/chain-filter-range %products.price [(shorthand->constraint %orders.user_id 1)])))))))))+(deftest ^:parallel multiple-values-test(testing "Chain filtering should support multiple values for a single parameter (as a vector or set of values)"(testing "Show me restaurants with price = 1 or 2 with the word 'BBQ' in their name (case-sensitive)"@@ -644,21 +709,30 @@(lib.walk/walk-clause clause collect))@acc))+(defn- semi-join?+ "Whether `source-query`, the query a join joins, is a de-duplicated sub-select, whose only columns are its breakouts."+ [source-query]+ (boolean (seq (lib/breakouts source-query 0))))+(defn- inner-projection-field-ids- "Set of field-ids that `a-join`'s inner-stage `:fields` projects. Structural read: the projection list is the thing- under test."- [a-join]- (into #{} (mapcat lib/all-field-ids) (:fields (first (:stages a-join)))))+ "Set of field-ids that a join's `source-query` projects: its `:fields`, or its breakouts for a semi-join. Structural+ read: the projection list is the thing under test."+ [source-query]+ (into #{}+ (mapcat lib/all-field-ids)+ (if (semi-join? source-query)+ (lib/breakouts source-query 0)+ (lib/fields source-query 0))))(defn- inner-projection-by-join-alias"Return `{alias #{field-id ...}}` — the field-ids each join's inner-stage `:fields` projects, keyed by the join'salias."[query](into {}- (for [a-join (lib/joins query)+ (for [[a-join source-query] (map vector (lib/joins query 0) (join-source-queries query)):let [a-alias (lib/current-join-alias a-join)]:when a-alias]- [a-alias (inner-projection-field-ids a-join)])))+ [a-alias (inner-projection-field-ids source-query)])))(defn- projection-violations"Return a seq of diagnostic maps (one per offending join's alias) or nil if the invariant holds: each join's@@ -703,11 +777,11 @@(update m table-id (fnil conj #{}) id)){}(sql-tools/referenced-fields driver nq))- mbql-proj (reduce (fn [m a-join]+ mbql-proj (reduce (fn [m [a-join source-query]](let [tid (:id (lib/joined-thing query a-join))](cond-> m- tid (update tid (fnil into #{}) (inner-projection-field-ids a-join)))))- {} (lib/joins query))]+ tid (update tid (fnil into #{}) (inner-projection-field-ids source-query)))))+ {} (map vector (lib/joins query 0) (join-source-queries query)))](not-empty(into {}(for [[tid declared] mbql-proj@@ -717,12 +791,20 @@[tid {:declared declared, :actual actual, :extra extra}])))))(defn- check-tight-projections- "Run both the MBQL-level and SQL-level invariants on a chain-filter query."+ "Run both the MBQL-level and SQL-level invariants on a chain-filter query, and on the sub-select of each of its+ semi-joins. The SQL-level check of the outer query leaves out the semi-joins, whose sub-selects reference columns of+ their own."[query]- (testing "MBQL: each join's inner-stage :fields equals the fields referenced on its source table"- (is (nil? (projection-violations query))))- (testing "SQL: the compiled query references no joined-Table columns outside MBQL :fields"- (is (nil? (sql-over-projections query)))))+ (let [semi-joins (for [[a-join source-query] (map vector (lib/joins query 0) (join-source-queries query))+ :when (semi-join? source-query)]+ [a-join source-query])]+ (testing "MBQL: each join's inner-stage :fields equals the fields referenced on its source table"+ (is (nil? (projection-violations query))))+ (testing "SQL: the compiled query references no joined-Table columns outside MBQL :fields"+ (is (nil? (sql-over-projections (reduce #(lib/remove-join %1 0 %2) query (map first semi-joins))))))+ (doseq [[a-join source-query] semi-joins]+ (testing (str "semi-join " (lib/current-join-alias a-join))+ (check-tight-projections source-query)))));;; Scenarios that exercise the join-building paths in `chain-filter-mbql-query`. Each;;; `:build` thunk runs inside `mt/dataset test-data` so `mt/id` resolves correctly.@@ -799,7 +881,7 @@;; elides the partition filter, because the joined-Table column it would reference is no longer in;; `visible-columns`. That's a behavior regression on BigQuery.;;- ;; The query below is chosen so venues becomes a JOIN target (source = categories, constraint on venues.id), and+ ;; The query below is chosen so venues becomes a JOIN target (source = checkins, constraint on venues.id), and;; the partition column (venues.price) is not referenced by anything the user wrote. So if tighten-join-projections;; correctly survives the late-added partition filter, venues.price ends up in the venues join's inner stage;; *only because of the partition filter middleware*.@@ -808,7 +890,7 @@partition-fid (mt/id :venues :price)](mt/with-temp-vals-in-db :model/Table venues-id {:database_require_filter true}(mt/with-temp-vals-in-db :model/Field partition-fid {:database_partitioned true}- (let [q (mbql-for (mt/id :categories :name)+ (let [q (mbql-for (mt/id :checkins :user_id)nil{(mt/id :venues :id) 1})venues-alias (some (fn [j]@@ -821,7 +903,17 @@"in :filters; got: " (pr-str (-> q :stages first :filters)))))(testing "venues join projects partition column in its inner stage"(is (contains? (get (inner-projection-by-join-alias q) venues-alias) partition-fid)- (str "expected " partition-fid " in inner stage of venues join (" (pr-str venues-alias) ")")))))))))+ (str "expected " partition-fid " in inner stage of venues join (" (pr-str venues-alias) ")"))))+ (testing "a semi-join's sub-select gets the partition filter on its own source table"+ ;; source = categories reaches venues through a one-to-many join+ (let [q (mbql-for (mt/id :categories :name)+ nil+ {(mt/id :venues :id) 1})+ [sub-select] (join-source-queries q)]+ (is (semi-join? sub-select))+ (is (some? (find-partition-filter sub-select partition-fid nil))+ (str "expected a [:> [:field {} " partition-fid "] _] clause in the sub-select's :filters; got: "+ (pr-str (-> sub-select :stages first :filters)))))))))));; Detail: Key (entity_id)=(6nmVTpCpKFRkZJigvqSVm) already exists.(deftest use-cached-field-values-test