Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Apply chain filter constraints behind one-to-many joins as semi-joins (#83955)

Christoph Deil1f48f93db277
diff --git a/src/metabase/parameters/chain_filter.clj b/src/metabase/parameters/chain_filter.clj
index dedd6bfdf9c..0a03948e87c 100644
--- a/src/metabase/parameters/chain_filter.clj
+++ b/src/metabase/parameters/chain_filter.clj
@@ -74,7 +74,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.types.isa :as lib.types.isa]
- [metabase.lib.util :as lib.util]
+ [metabase.lib.walk :as lib.walk]
[metabase.parameters.chain-filter.dedupe-joins :as dedupe]
[metabase.parameters.db :as parameters.db]
[metabase.parameters.field-values :as params.field-values]
@@ -415,7 +415,9 @@
join's source table.
Assumes no two joins in stage 0 target the same `:table-id` (true for chain-filter's `joined-table-alias`
- convention). If that ever breaks, both joins get the union of fields — still correct, only over-projection."
+ convention). If that ever breaks, both joins get the union of fields — still correct, only over-projection.
+
+ Semi-joins (see [[add-semi-joins]]) are left alone: their sub-select returns only its breakout column already."
[query :- ::lib.schema/query]
(if (empty? (lib/joins query))
query
@@ -424,17 +426,170 @@
(let [field-ids (lib/all-field-ids query)
cols (lib.metadata/bulk-metadata query :metadata/column field-ids)
cols-by-tid (group-by :table-id cols)]
- (lib.util/update-query-stage
- query 0
- update :joins
- (fn [the-joins]
- (mapv (fn [a-join]
- (let [thing (lib/joined-thing query a-join)
- tid (when (= :metadata/table (:lib/type thing))
+ (lib.walk/walk
+ query
+ (fn [query path-type path a-join]
+ (when (and (= path-type :lib.walk/join)
+ ;; only the joins of the first stage: the joins inside a semi-join's sub-select were tightened
+ ;; when the sub-select was built
+ (= (lib.walk/join-parent-stage-path path) [:stages 0]))
+ (let [thing (lib/joined-thing query a-join)
+ summarized? (seq (lib.walk/apply-f-for-stage-at-path
+ lib/breakouts query (lib.walk/join-last-stage-path path a-join)))
+ tid (when (and (= :metadata/table (:lib/type thing))
+ (not summarized?))
(:id thing))]
- (cond-> a-join
- tid (lib/with-join-source-fields (get cols-by-tid tid)))))
- the-joins))))))
+ (when tid
+ (lib/with-join-source-fields a-join (get cols-by-tid tid))))))))))
+
+(defn- one-to-many-join?
+ "Whether `join-info` follows an FK backwards, from the Field it references to the referencing Field, so that one row
+ on the left-hand side can match many rows on the right-hand side."
+ [id->field {{lhs-field-id :field} :lhs, {rhs-field-id :field} :rhs}]
+ (not= rhs-field-id (:fk-target-field-id (id->field lhs-field-id))))
+
+(defn- join-path
+ "The joins that lead from `source-table-id` to `table-id`, in order, or nil if there is no path. Where several FKs
+ link two Tables, prefer one of `field-ids`."
+ [database-id source-table-id table-id field-ids]
+ (when-let [joins (not-empty (find-joins database-id source-table-id table-id))]
+ (dedupe/dedupe-joins source-table-id field-ids joins #{table-id})))
+
+(mr/def ::one-to-many-path
+ "A join path split after its first one-to-many join."
+ [:map {:closed true}
+ [:prefix [:sequential ::join-info]]
+ [:suffix [:sequential ::join-info]]])
+
+(mu/defn- one-to-many-paths :- [:map-of ::lib.schema.id/table ::one-to-many-path]
+ "The join paths from `source-table-id` to those Tables of `field-ids` that can only be reached by following an FK
+ backwards somewhere, keyed by Table ID."
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ database-id :- ::lib.schema.id/database
+ source-table-id :- ::lib.schema.id/table
+ field-ids :- [:set ::lib.schema.id/field]]
+ (let [table-ids (when (seq field-ids)
+ (disj (into #{}
+ (map :table-id)
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/column field-ids))
+ source-table-id))
+ paths (into {}
+ (keep (fn [table-id]
+ (when-let [path (join-path database-id source-table-id table-id field-ids)]
+ [table-id path])))
+ table-ids)
+ id->field (when (seq paths)
+ (u/index-by :id (lib.metadata/bulk-metadata metadata-providerable :metadata/column
+ (into #{}
+ (comp cat
+ (mapcat (juxt #(get-in % [:lhs :field])
+ #(get-in % [:rhs :field]))))
+ (vals paths)))))]
+ (into {}
+ (keep (fn [[table-id path]]
+ (let [[to-one [one-to-many & suffix]] (split-with #(not (one-to-many-join? id->field %)) path)]
+ (when one-to-many
+ [table-id {:prefix (conj (vec to-one) one-to-many)
+ :suffix (vec suffix)}]))))
+ paths)))
+
+(mu/defn- semi-join-clause :- :metabase.lib.schema.join/join
+ "A join against a sub-select that applies the `constraints` on the Tables with `table-ids` and returns the distinct
+ values of the column the source Table joins against. `paths` lead from the source Table to these Tables and share
+ their `:prefix`."
+ [query :- ::lib.schema/query
+ join-alias :- ::lib.schema.common/non-blank-string
+ paths :- [:sequential ::one-to-many-path]
+ table-ids :- [:set ::lib.schema.id/table]
+ constraints :- [:maybe ::constraints]]
+ (let [{{lhs-field-id :field} :lhs
+ {rhs-table-id :table, rhs-field-id :field} :rhs} (first (:prefix (first paths)))
+ field-ids (into #{} (map :field-id) constraints)
+ ;; the sub-select starts at the Table the first join leads to
+ joins (dedupe/dedupe-joins rhs-table-id
+ field-ids
+ (into [] (comp (mapcat #(concat (rest (:prefix %)) (:suffix %)))
+ (distinct))
+ paths)
+ (disj table-ids rhs-table-id))
+ sub-select (-> (lib/query query (lib.metadata/table query rhs-table-id))
+ (add-joins rhs-table-id joins)
+ (lib/breakout (lib.metadata/field query rhs-field-id))
+ (add-filters rhs-table-id (set (map #(get-in % [:rhs :table]) joins)) constraints)
+ schema.metadata-queries/add-required-filters-if-needed
+ tighten-join-projections)
+ rhs-field (lib/with-join-alias (lib.metadata/field query rhs-field-id) join-alias)]
+ (log/tracef "Adding semi-join %s against %s for %s"
+ join-alias
+ (name-for-logging :model/Table rhs-table-id)
+ (str/join ", " (map (partial name-for-logging :model/Table) (sort table-ids))))
+ (-> (lib/join-clause sub-select)
+ (lib/with-join-alias join-alias)
+ (lib/with-join-conditions [(lib/= (lib.metadata/field query lhs-field-id) rhs-field)])
+ (lib/with-join-strategy :inner-join))))
+
+(mu/defn- add-semi-joins :- ::lib.schema/query
+ "Constrain `query` by the `constraints` on the Tables in `paths` through semi-joins.
+
+ Joining a Table through a one-to-many join repeats each row of the source Table once per matching row, and joining
+ several such Tables multiplies these repetitions. With a locked embedding parameter that is mapped to a column of
+ every card's Table, this cross product can grow to billions of rows before `GROUP BY` reduces it to at most
+ [[max-results]] values.
+
+ So each of these Tables is joined in a sub-select instead, which applies its constraints and returns the distinct
+ values of the column the source Table joins against, e.g.
+
+ INNER JOIN (SELECT venue_id FROM checkins WHERE ... GROUP BY venue_id) semi_join_1
+ ON venues.id = semi_join_1.venue_id
+
+ Such a join matches each source row at most once. Tables whose paths share everything up to and including their
+ first one-to-many join are constrained in the same sub-select, so they still have to match through the same row, as
+ in a single query. Paths that part before that join meet only in rows the source row determines, so constraining
+ them separately returns the same values."
+ [query :- ::lib.schema/query
+ paths :- [:map-of ::lib.schema.id/table ::one-to-many-path]
+ constraints :- [:maybe ::constraints]]
+ (if (empty? paths)
+ query
+ (let [field-id->table-id (into {}
+ (map (juxt :id :table-id))
+ (lib.metadata/bulk-metadata query :metadata/column
+ (into #{} (map :field-id) constraints)))
+ table-id-groups (->> (group-by (comp :prefix paths) (keys paths))
+ vals
+ (map set)
+ (sort-by (comp vec sort)))]
+ (reduce (fn [query [i table-ids]]
+ (lib/join query (semi-join-clause query
+ (format "semi_join_%d" (inc i))
+ (map paths table-ids)
+ table-ids
+ (filter #(contains? table-ids (field-id->table-id (:field-id %)))
+ constraints))))
+ query
+ (map-indexed vector table-id-groups)))))
+
+(mu/defn- constraint-joins :- [:map {:closed true}
+ [:joins [:maybe [:sequential ::join-info]]]
+ [:semi-join-paths [:map-of ::lib.schema.id/table ::one-to-many-path]]]
+ "Plan how a query on `source-table-id` reaches the Fields of `constraints` and the additional `field-ids`. `:joins`
+ are ordinary joins for [[add-joins]]; `:semi-join-paths` lead to the Tables whose constraints [[add-semi-joins]]
+ applies, i.e. those reached only through a one-to-many join and not joined anyway."
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ database-id :- ::lib.schema.id/database
+ source-table-id :- ::lib.schema.id/table
+ constraints :- [:maybe ::constraints]
+ field-ids :- [:set ::lib.schema.id/field]]
+ (let [constraint-field-ids (into #{} (map :field-id) constraints)
+ paths (one-to-many-paths metadata-providerable database-id source-table-id constraint-field-ids)
+ direct-field-ids (into field-ids
+ (comp (remove #(contains? paths (:table-id %)))
+ (map :id))
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/column
+ constraint-field-ids))
+ joins (find-all-joins metadata-providerable database-id source-table-id direct-field-ids)]
+ {:joins joins
+ :semi-join-paths (apply dissoc paths (map #(get-in % [:rhs :table]) joins))}))
(mr/def ::options
;; if original-field-id is specified, we'll include this in the results. For Field->Field remapping.
@@ -463,10 +618,10 @@
;; engines like ClickHouse that materialize the right side in memory.
reversed? (and original-table-id (not= original-table-id field-table-id))
source-table-id (if reversed? original-table-id field-table-id)
- joins (find-all-joins mp database-id source-table-id
- (cond-> (set (map :field-id constraints))
- reversed? (conj field-id)
- (not reversed?) (cond-> original-field-id (conj original-field-id))))
+ remap-field-ids (cond-> #{}
+ reversed? (conj field-id)
+ (not reversed?) (cond-> original-field-id (conj original-field-id)))
+ {:keys [joins semi-join-paths]} (constraint-joins mp database-id source-table-id constraints remap-field-ids)
joined-table-ids (set (map #(get-in % [:rhs :table]) joins))
field (cond-> (lib.metadata/field mp field-id)
reversed? (lib/with-join-alias (joined-table-alias field-table-id)))
@@ -489,6 +644,7 @@
(lib/limit ((fnil min Integer/MAX_VALUE) limit max-results))
(assoc-in [:middleware :disable-remaps?] true)
(add-joins source-table-id joins)
+ (add-semi-joins semi-join-paths constraints)
(cond-> original-field (->
;; don't return rows that don't have values for the original Field. e.g. if
;; venues.category_id is remapped to categories.name and we do a search with query
@@ -687,7 +843,7 @@
(let [database-id (field/field-id->database-id field-id)
mp (lib-be/application-database-metadata-provider database-id)
source-table-id (:table-id (lib.metadata/field mp field-id))
- joins (find-all-joins mp database-id source-table-id (set (map :field-id constraints)))
+ {:keys [joins semi-join-paths]} (constraint-joins mp database-id source-table-id constraints #{})
joined-table-ids (set (map #(get-in % [:rhs :table]) joins))
field (lib.metadata/field mp field-id)]
(when (seq joins)
@@ -696,6 +852,7 @@
(-> (lib/query mp (lib.metadata/table mp source-table-id))
(assoc-in [:middleware :disable-remaps?] true)
(add-joins source-table-id joins)
+ (add-semi-joins semi-join-paths constraints)
(lib/aggregate (lib/min field))
(lib/aggregate (lib/max field))
(lib/aggregate (lib/distinct field))
@@ -860,5 +1017,12 @@
(let [mbql-query (chain-filter-mbql-query field-id
(for [id filter-field-ids]
{:field-id id, :op :=, :value nil})
- nil)]
- (into #{} (mapcat lib/all-field-ids) (lib/filters mbql-query)))))
+ nil)
+ ;; constraints on Tables behind one-to-many joins are filters of the semi-joins' sub-selects, so collect the
+ ;; filters of every stage, including the stages inside joins
+ filters (volatile! [])]
+ (lib.walk/walk-stages mbql-query
+ (fn [query path _stage]
+ (vswap! filters into (lib.walk/apply-f-for-stage-at-path lib/filters query path))
+ nil))
+ (into #{} (mapcat lib/all-field-ids) @filters))))
diff --git a/test/metabase/parameters/chain_filter_test.clj b/test/metabase/parameters/chain_filter_test.clj
index 25bd223cebb..fa206366c00 100644
--- a/test/metabase/parameters/chain_filter_test.clj
+++ b/test/metabase/parameters/chain_filter_test.clj
@@ -57,6 +57,20 @@
[n result]
(update result :values #(take n %)))
+(defn- join-source-queries
+ "The queries joined by the joins of `query`'s first stage, in join order, each as a query of its own."
+ [query]
+ (for [join-number (range (count (lib/joins query 0)))]
+ (:query (lib.walk/query-for-path query [:stages 0 :joins join-number]))))
+
+(defn- join-summaries
+ "For each join of `query`'s first stage: its strategy, and the field IDs its source query breaks out and filters by."
+ [query]
+ (for [[a-join source-query] (map vector (lib/joins query 0) (join-source-queries query))]
+ {:strategy (lib/raw-join-strategy a-join)
+ :breakout (into #{} (mapcat lib/all-field-ids) (lib/breakouts source-query 0))
+ :filters (into #{} (mapcat lib/all-field-ids) (lib/filters source-query 0))}))
+
(deftest ^:parallel special-form-filter-clause-test
(testing "Can handle multi-arg string filters when chaining (#57287)"
(let [mp (mt/metadata-provider)]
@@ -138,6 +152,57 @@
(is (= {:values [["Festa"] ["Fred 62"]], :has_more_values false}
(chain-filter venues.name {venues.price [:between 2 3], venues.name [:starts-with "f" {:case-sensitive false}]})))))
+(deftest ^:parallel one-to-many-constraints-semi-join-test
+ (testing (str "Constraints on Tables reached through one-to-many joins are applied in de-duplicated sub-selects, "
+ "so the joins can't multiply the source Table's rows")
+ (mt/dataset test-data
+ (mt/$ids
+ (let [query (#'chain-filter/chain-filter-mbql-query %products.category
+ [(shorthand->constraint %orders.user_id 1)
+ (shorthand->constraint %reviews.rating 1)]
+ nil)]
+ (is (= #{{:strategy :inner-join, :breakout #{%orders.product_id}, :filters #{%orders.user_id}}
+ {:strategy :inner-join, :breakout #{%reviews.product_id}, :filters #{%reviews.rating}}}
+ (set (join-summaries query))))
+ (is (nil? (lib/filters query)))
+ (is (= {:values [["Doohickey"] ["Widget"]], :has_more_values false}
+ (chain-filter products.category {orders.user_id 1, reviews.rating 1}))))))))
+
+(deftest ^:parallel one-to-many-constraints-shared-join-test
+ (testing (str "Constraints whose paths share their first one-to-many join are applied in one sub-select, so they "
+ "still have to match the same row of it")
+ (mt/dataset test-data
+ (mt/$ids
+ (let [query (#'chain-filter/chain-filter-mbql-query
+ %venues.name
+ [(shorthand->constraint %users.name "Plato Yeshua")
+ (shorthand->constraint %checkins.date [:between "2015-01-01" "2015-06-30"])]
+ nil)]
+ (is (= [#{%users.name %checkins.date}]
+ (map :filters (join-summaries query))))
+ (testing "venues with a check-in by Plato Yeshua in the first half of 2015"
+ (is (= {:values [["Barney's Beanery"]
+ ["Cha Cha Chicken"]
+ ["Don Day Korean Restaurant"]
+ ["Red Medicine"]
+ ["Shanghai Dumpling King"]]
+ :has_more_values false}
+ (chain-filter venues.name {users.name "Plato Yeshua"
+ checkins.date [:between "2015-01-01" "2015-06-30"]})))))))))
+
+(deftest ^:parallel one-to-many-constraints-range-test
+ (testing "The range query applies constraints on Tables reached through one-to-many joins as semi-joins too"
+ (mt/dataset test-data
+ (mt/$ids
+ (let [query (#'chain-filter/chain-filter-range-mbql-query %products.price
+ [(shorthand->constraint %orders.user_id 1)])]
+ (is (= [{:strategy :inner-join, :breakout #{%orders.product_id}, :filters #{%orders.user_id}}]
+ (join-summaries query)))
+ (is (nil? (lib/filters query)))
+ (testing "min, max and distinct count of the prices of the Products that User 1 ordered"
+ (is (= {:min 19.87, :max 85.25, :distinct-count 11}
+ (chain-filter/chain-filter-range %products.price [(shorthand->constraint %orders.user_id 1)])))))))))
+
(deftest ^:parallel multiple-values-test
(testing "Chain filtering should support multiple values for a single parameter (as a vector or set of values)"
(testing "Show me restaurants with price = 1 or 2 with the word 'BBQ' in their name (case-sensitive)"
@@ -644,21 +709,30 @@
(lib.walk/walk-clause clause collect))
@acc))
+(defn- semi-join?
+ "Whether `source-query`, the query a join joins, is a de-duplicated sub-select, whose only columns are its breakouts."
+ [source-query]
+ (boolean (seq (lib/breakouts source-query 0))))
+
(defn- inner-projection-field-ids
- "Set of field-ids that `a-join`'s inner-stage `:fields` projects. Structural read: the projection list is the thing
- under test."
- [a-join]
- (into #{} (mapcat lib/all-field-ids) (:fields (first (:stages a-join)))))
+ "Set of field-ids that a join's `source-query` projects: its `:fields`, or its breakouts for a semi-join. Structural
+ read: the projection list is the thing under test."
+ [source-query]
+ (into #{}
+ (mapcat lib/all-field-ids)
+ (if (semi-join? source-query)
+ (lib/breakouts source-query 0)
+ (lib/fields source-query 0))))
(defn- inner-projection-by-join-alias
"Return `{alias #{field-id ...}}` — the field-ids each join's inner-stage `:fields` projects, keyed by the join's
alias."
[query]
(into {}
- (for [a-join (lib/joins query)
+ (for [[a-join source-query] (map vector (lib/joins query 0) (join-source-queries query))
:let [a-alias (lib/current-join-alias a-join)]
:when a-alias]
- [a-alias (inner-projection-field-ids a-join)])))
+ [a-alias (inner-projection-field-ids source-query)])))
(defn- projection-violations
"Return a seq of diagnostic maps (one per offending join's alias) or nil if the invariant holds: each join's
@@ -703,11 +777,11 @@
(update m table-id (fnil conj #{}) id))
{}
(sql-tools/referenced-fields driver nq))
- mbql-proj (reduce (fn [m a-join]
+ mbql-proj (reduce (fn [m [a-join source-query]]
(let [tid (:id (lib/joined-thing query a-join))]
(cond-> m
- tid (update tid (fnil into #{}) (inner-projection-field-ids a-join)))))
- {} (lib/joins query))]
+ tid (update tid (fnil into #{}) (inner-projection-field-ids source-query)))))
+ {} (map vector (lib/joins query 0) (join-source-queries query)))]
(not-empty
(into {}
(for [[tid declared] mbql-proj
@@ -717,12 +791,20 @@
[tid {:declared declared, :actual actual, :extra extra}])))))
(defn- check-tight-projections
- "Run both the MBQL-level and SQL-level invariants on a chain-filter query."
+ "Run both the MBQL-level and SQL-level invariants on a chain-filter query, and on the sub-select of each of its
+ semi-joins. The SQL-level check of the outer query leaves out the semi-joins, whose sub-selects reference columns of
+ their own."
[query]
- (testing "MBQL: each join's inner-stage :fields equals the fields referenced on its source table"
- (is (nil? (projection-violations query))))
- (testing "SQL: the compiled query references no joined-Table columns outside MBQL :fields"
- (is (nil? (sql-over-projections query)))))
+ (let [semi-joins (for [[a-join source-query] (map vector (lib/joins query 0) (join-source-queries query))
+ :when (semi-join? source-query)]
+ [a-join source-query])]
+ (testing "MBQL: each join's inner-stage :fields equals the fields referenced on its source table"
+ (is (nil? (projection-violations query))))
+ (testing "SQL: the compiled query references no joined-Table columns outside MBQL :fields"
+ (is (nil? (sql-over-projections (reduce #(lib/remove-join %1 0 %2) query (map first semi-joins))))))
+ (doseq [[a-join source-query] semi-joins]
+ (testing (str "semi-join " (lib/current-join-alias a-join))
+ (check-tight-projections source-query)))))
;;; Scenarios that exercise the join-building paths in `chain-filter-mbql-query`. Each
;;; `:build` thunk runs inside `mt/dataset test-data` so `mt/id` resolves correctly.
@@ -799,7 +881,7 @@
;; elides the partition filter, because the joined-Table column it would reference is no longer in
;; `visible-columns`. That's a behavior regression on BigQuery.
;;
- ;; The query below is chosen so venues becomes a JOIN target (source = categories, constraint on venues.id), and
+ ;; The query below is chosen so venues becomes a JOIN target (source = checkins, constraint on venues.id), and
;; the partition column (venues.price) is not referenced by anything the user wrote. So if tighten-join-projections
;; correctly survives the late-added partition filter, venues.price ends up in the venues join's inner stage
;; *only because of the partition filter middleware*.
@@ -808,7 +890,7 @@
partition-fid (mt/id :venues :price)]
(mt/with-temp-vals-in-db :model/Table venues-id {:database_require_filter true}
(mt/with-temp-vals-in-db :model/Field partition-fid {:database_partitioned true}
- (let [q (mbql-for (mt/id :categories :name)
+ (let [q (mbql-for (mt/id :checkins :user_id)
nil
{(mt/id :venues :id) 1})
venues-alias (some (fn [j]
@@ -821,7 +903,17 @@
"in :filters; got: " (pr-str (-> q :stages first :filters)))))
(testing "venues join projects partition column in its inner stage"
(is (contains? (get (inner-projection-by-join-alias q) venues-alias) partition-fid)
- (str "expected " partition-fid " in inner stage of venues join (" (pr-str venues-alias) ")")))))))))
+ (str "expected " partition-fid " in inner stage of venues join (" (pr-str venues-alias) ")"))))
+ (testing "a semi-join's sub-select gets the partition filter on its own source table"
+ ;; source = categories reaches venues through a one-to-many join
+ (let [q (mbql-for (mt/id :categories :name)
+ nil
+ {(mt/id :venues :id) 1})
+ [sub-select] (join-source-queries q)]
+ (is (semi-join? sub-select))
+ (is (some? (find-partition-filter sub-select partition-fid nil))
+ (str "expected a [:> [:field {} " partition-fid "] _] clause in the sub-select's :filters; got: "
+ (pr-str (-> sub-select :stages first :filters)))))))))))
;; Detail: Key (entity_id)=(6nmVTpCpKFRkZJigvqSVm) already exists.
(deftest use-cached-field-values-test