Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Optimize chain filter inner field projections (#75209)

Timothy S. Deanb1b5d4906bd6
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index dfe19f5e3fa..841d85837e8 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -796,6 +796,14 @@
stage-number :- :int]
(lib.join/joins a-query stage-number)))
+(mu/defn joined-thing :- [:maybe ::lib.join/joinable]
+ "Return metadata about the origin of `a-join` (the joined Table or Card).
+
+ **Code Health:** Healthy. This is a core API."
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ a-join :- ::lib.join.util/partial-join]
+ (lib.join/joined-thing metadata-providerable a-join))
+
;;; ### Join Strategies
(mu/defn join-strategy :- ::lib.schema.join/strategy.option
@@ -1018,6 +1026,20 @@
fields :- [:maybe [:or [:enum :all :none] [:sequential some?]]]] ;; TODO: More precise schema.
(lib.join/with-join-fields a-join fields))
+(mu/defn with-join-source-fields :- ::lib.join.util/partial-join
+ "Set the `:fields` projection on the join's source subquery (the first stage of `a-join`). `cols` is a coll of
+ column metadatas from the source Table or Card; `nil`/empty dissocs, reverting to implicit-all.
+
+ Throws if the join's first stage is not an MBQL stage, or if narrowing the source to `cols` would strand a column the
+ join still references in its conditions or its exposed `:fields`.
+
+ For what the join EXPOSES to its outer stage, see [[with-join-fields]].
+
+ **Code Health:** Healthy. This is a core API."
+ [a-join :- ::lib.join.util/partial-join
+ cols :- [:maybe [:sequential some?]]] ; ideally [:sequential ::lib.schema.metadata/column]
+ (lib.join/with-join-source-fields a-join cols))
+
(mu/defn join-fieldable-columns :- ::lib.metadata.calculation/visible-columns
"Returns the list of column metadata for the columns which are *visible* on the RHS of `a-joinable`, such as a table,
card or join clause.
diff --git a/src/metabase/lib/field.cljc b/src/metabase/lib/field.cljc
index 0cfb4f8b677..dc41ec91326 100644
--- a/src/metabase/lib/field.cljc
+++ b/src/metabase/lib/field.cljc
@@ -492,8 +492,36 @@
:include-expressions? true
:include-implicitly-joinable? false})))
+(defn- newly-stranded-downstream-refs
+ "Field refs in stages after `stage-number` that resolve against `before` but not against `after`. Non-empty means
+ narrowing a stage's `:fields` from `before` to `after` left a later stage referencing a column that stage no longer
+ returns."
+ [before after stage-number]
+ (let [stage-number (lib.util/canonical-stage-index before stage-number)
+ bad-refs (fn [query]
+ (let [bad (volatile! #{})]
+ (lib.walk/walk-clauses
+ query
+ (fn [query path-type path clause]
+ (when (and (= path-type :lib.walk/stage)
+ (> (second path) stage-number)
+ (vector? clause)
+ (= (first clause) :field))
+ (let [col (lib.walk/apply-f-for-stage-at-path
+ lib.field.resolution/resolve-field-ref query path clause)]
+ (when (or (not col)
+ (::lib.field.resolution/fallback-metadata? col)
+ (not (:active col true)))
+ (vswap! bad conj clause))))
+ nil))
+ @bad))]
+ (set/difference (bad-refs after) (bad-refs before))))
+
(mu/defn with-fields :- ::lib.schema/query
- "Specify the `:fields` for a query. Pass `nil` or an empty sequence to remove `:fields`."
+ "Specify the `:fields` for a query. Pass `nil` or an empty sequence to remove `:fields`.
+
+ Throws if narrowing the projection would strand a reference in a later stage (a column that stage no longer
+ returns)."
([xs]
(fn [query stage-number]
(with-fields query stage-number xs)))
@@ -513,8 +541,13 @@
(or xs []))
;; Those expr-refs which must still be included.
to-add (remove included expr-cols)
- xs (when xs (into xs (map lib.ref/ref) to-add))]
- (lib.util/update-query-stage query stage-number u/assoc-dissoc :fields xs))))
+ xs (when xs (into xs (map lib.ref/ref) to-add))
+ result (lib.util/update-query-stage query stage-number u/assoc-dissoc :fields xs)]
+ (when (and xs (lib.util/next-stage-number query stage-number))
+ (when-let [stranded (not-empty (newly-stranded-downstream-refs query result stage-number))]
+ (throw (ex-info "with-fields would strand downstream references to dropped columns"
+ {:stage-number stage-number, :stranded stranded}))))
+ result)))
(mu/defn fields :- [:maybe [:ref ::lib.schema/fields]]
"Fetches the `:fields` for a query. Returns `nil` if there are no `:fields`. `:fields` should never be empty; this is
diff --git a/src/metabase/lib/join.cljc b/src/metabase/lib/join.cljc
index bfcbd916342..9c3a94f8e4f 100644
--- a/src/metabase/lib/join.cljc
+++ b/src/metabase/lib/join.cljc
@@ -515,6 +515,45 @@
fields)))]
(u/assoc-dissoc joinable :fields fields)))
+(defn- stranded-join-refs
+ "Refs in `a-join`'s conditions or exposed `:fields` that point at source columns missing from `cols` (a proposed new
+ source projection). Non-empty means narrowing the source to `cols` would leave the join referencing columns its
+ subquery no longer returns."
+ [a-join cols]
+ (let [join-alias (:alias a-join)
+ field-refs (when (sequential? (:fields a-join))
+ (:fields a-join))
+ cond-refs (when join-alias
+ (match/match-many (:conditions a-join)
+ [:field (opts :guard (= (:join-alias opts) join-alias)) _] &match))]
+ (into []
+ (remove (fn [a-ref]
+ (lib.equality/find-matching-column
+ (lib.options/update-options a-ref dissoc :join-alias :source-field) cols)))
+ (concat field-refs cond-refs))))
+
+(mu/defn with-join-source-fields :- ::lib.join.util/partial-join
+ "Set the `:fields` projection on the join's source subquery (the first stage of `a-join`). `cols` is a coll of
+ column metadatas from the source Table or Card; `nil`/empty dissocs, reverting to implicit-all.
+
+ Throws if the join's first stage is not an MBQL stage, or if narrowing the source to `cols` would strand a column the
+ join still references in its conditions or its exposed `:fields`.
+
+ For what the join EXPOSES to its outer stage, see [[with-join-fields]]."
+ [a-join :- ::lib.join.util/partial-join
+ cols :- [:maybe [:sequential some?]]] ; ideally [:sequential ::lib.schema.metadata/column]
+ (let [first-stage-type (-> a-join :stages first :lib/type)]
+ (when-not (= :mbql.stage/mbql first-stage-type)
+ (throw (ex-info "with-join-source-fields requires the join's first stage to be an MBQL stage"
+ {:first-stage-type first-stage-type
+ :join a-join}))))
+ (let [refs (not-empty (mapv lib.ref/ref cols))]
+ (when refs
+ (when-let [stranded (not-empty (stranded-join-refs a-join cols))]
+ (throw (ex-info "with-join-source-fields would strand references the join still needs"
+ {:join a-join, :cols cols, :stranded stranded}))))
+ (update-in a-join [:stages 0] u/assoc-dissoc :fields refs)))
+
(defn- select-home-column
[home-cols cond-fields]
(when (seq cond-fields)
diff --git a/src/metabase/parameters/chain_filter.clj b/src/metabase/parameters/chain_filter.clj
index cac486e1afa..e2f9a9fa0c1 100644
--- a/src/metabase/parameters/chain_filter.clj
+++ b/src/metabase/parameters/chain_filter.clj
@@ -73,6 +73,7 @@
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.types.isa :as lib.types.isa]
+ [metabase.lib.util :as lib.util]
[metabase.parameters.chain-filter.dedupe-joins :as dedupe]
[metabase.parameters.field-values :as params.field-values]
[metabase.parameters.params :as params]
@@ -403,6 +404,32 @@
query
joins))
+(mu/defn- tighten-join-projections :- ::lib.schema/query
+ "Narrow each join's inner-stage `:fields` to exactly the field-ids `query` references whose `:table-id` matches the
+ join's source table.
+
+ Assumes no two joins in stage 0 target the same `:table-id` (true for chain-filter's `joined-table-alias`
+ convention). If that ever breaks, both joins get the union of fields — still correct, only over-projection."
+ [query :- ::lib.schema/query]
+ (if (empty? (lib/joins query))
+ query
+ ;; Bucket by `:table-id`, not by join alias: aliases are stage-scoped and the same string can recur in nested
+ ;; scopes referring to different things. Table-id is the field's natural grain.
+ (let [field-ids (lib/all-field-ids query)
+ cols (lib.metadata/bulk-metadata query :metadata/column field-ids)
+ cols-by-tid (group-by :table-id cols)]
+ (lib.util/update-query-stage
+ query 0
+ update :joins
+ (fn [the-joins]
+ (mapv (fn [a-join]
+ (let [thing (lib/joined-thing query a-join)
+ tid (when (= :metadata/table (:lib/type thing))
+ (:id thing))]
+ (cond-> a-join
+ tid (lib/with-join-source-fields (get cols-by-tid tid)))))
+ the-joins))))))
+
(mr/def ::options
;; if original-field-id is specified, we'll include this in the results. For Field->Field remapping.
[:map {:closed true}
@@ -473,7 +500,11 @@
(lib/order-by field))
(not original-field) (lib/breakout field))
(add-filters source-table-id joined-table-ids constraints)
- schema.metadata-queries/add-required-filters-if-needed)))
+ schema.metadata-queries/add-required-filters-if-needed
+ ;; Runs LAST so it picks up joined-table field refs injected by middleware above (e.g. BigQuery partition
+ ;; filters). Without an explicit projection here the join's inner stage gets expanded to every column on the
+ ;; joined Table by add-implicit-clauses, OOM'ing on wide fact tables.
+ tighten-join-projections)))
;;; ------------------------ Chain filter (powers GET /api/dashboard/:id/params/:key/values) -------------------------
diff --git a/test/metabase/lib/field_test.cljc b/test/metabase/lib/field_test.cljc
index 3f2dd9f0fde..4831a738aa7 100644
--- a/test/metabase/lib/field_test.cljc
+++ b/test/metabase/lib/field_test.cljc
@@ -25,6 +25,7 @@
[metabase.lib.test-util.notebook-helpers :as lib.tu.notebook]
[metabase.lib.types.isa :as lib.types.isa]
[metabase.lib.util :as lib.util]
+ [metabase.lib.validate :as lib.validate]
[metabase.util :as u]
[metabase.util.malli :as mu]))
@@ -803,6 +804,50 @@
{:name "myadd"}]
metadatas)))))
+(deftest ^:parallel with-fields-throws-on-stranded-downstream-refs-test
+ (testing "narrowing a stage's :fields throws if it strands a later-stage reference"
+ (let [base (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ lib/append-stage)
+ total (->> (lib.metadata.calculation/visible-columns base 1)
+ (m/find-first (comp #{"TOTAL"} :name)))
+ base (lib/order-by base 1 total :asc)
+ id-col (->> (lib.metadata.calculation/returned-columns base 0)
+ (m/find-first (comp #{"ID"} :name)))]
+ (is (thrown? #?(:clj clojure.lang.ExceptionInfo :cljs js/Error)
+ (lib/with-fields base 0 [id-col]))))))
+
+(deftest ^:parallel with-fields-allows-safe-narrowing-test
+ (testing "narrowing that keeps every referenced column does not throw"
+ (let [base (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ lib/append-stage)
+ total (->> (lib.metadata.calculation/visible-columns base 1)
+ (m/find-first (comp #{"TOTAL"} :name)))
+ base (lib/order-by base 1 total :asc)
+ kept (->> (lib.metadata.calculation/returned-columns base 0)
+ (filter (comp #{"ID" "TOTAL"} :name)))
+ narrowed (lib/with-fields base 0 kept)]
+ (is (empty? (lib.validate/find-bad-refs narrowed))))))
+
+(deftest ^:parallel newly-stranded-downstream-refs-test
+ (let [before (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ lib/append-stage)
+ total (->> (lib.metadata.calculation/visible-columns before 1)
+ (m/find-first (comp #{"TOTAL"} :name)))
+ before (lib/order-by before 1 total :asc)
+ id-ref (lib/ref (->> (lib.metadata.calculation/returned-columns before 0)
+ (m/find-first (comp #{"ID"} :name))))
+ total-ref (lib/ref (->> (lib.metadata.calculation/returned-columns before 0)
+ (m/find-first (comp #{"TOTAL"} :name))))
+ narrow (fn [refs] (lib.util/update-query-stage before 0 assoc :fields refs))]
+ (testing "reports a later-stage ref the narrowing drops"
+ (is (= #{"TOTAL"}
+ (into #{} (map peek)
+ (#'lib.field/newly-stranded-downstream-refs before (narrow [id-ref]) 0)))))
+ (testing "reports nothing when the narrowing keeps the referenced column"
+ (is (empty? (#'lib.field/newly-stranded-downstream-refs before (narrow [id-ref total-ref]) 0))))
+ (testing "considers only stages after stage-number"
+ (is (empty? (#'lib.field/newly-stranded-downstream-refs before (narrow [id-ref]) 1))))))
+
(deftest ^:parallel fieldable-columns-test
(testing "query with no :fields"
(is (=? [{:lib/source-column-alias "ID", :selected? true}
diff --git a/test/metabase/lib/join_test.cljc b/test/metabase/lib/join_test.cljc
index 669fe900c8a..3bf8cdb0665 100644
--- a/test/metabase/lib/join_test.cljc
+++ b/test/metabase/lib/join_test.cljc
@@ -4,6 +4,7 @@
[clojure.test :refer [are deftest is testing]]
[medley.core :as m]
[metabase.lib.core :as lib]
+ [metabase.lib.equality :as lib.equality]
[metabase.lib.join :as lib.join]
[metabase.lib.join.util :as lib.join.util]
[metabase.lib.metadata :as lib.metadata]
@@ -16,7 +17,8 @@
[metabase.lib.test-util.mocks-31769 :as lib.tu.mocks-31769]
[metabase.lib.test-util.notebook-helpers :as lib.tu.notebook]
[metabase.lib.util :as lib.util]
- [metabase.util :as u]))
+ [metabase.util :as u]
+ [metabase.util.match :as lib.util.match]))
#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))
@@ -62,6 +64,96 @@
:args [(lib/ref (meta/field-metadata :venues :category-id))
(lib/ref (meta/field-metadata :categories :id))]}]))))))
+(deftest ^:parallel with-join-source-fields-test
+ (let [base-join (lib/join-clause (meta/table-metadata :categories)
+ [(lib/= (meta/field-metadata :venues :category-id)
+ (meta/field-metadata :categories :id))])]
+ (testing "Sets :fields on the join's first (source) stage from column metadatas"
+ (let [result (lib/with-join-source-fields base-join [(meta/field-metadata :categories :id)])]
+ (is (= 1 (count (:fields (first (:stages result))))))))
+ (testing "nil or empty cols dissocs :fields (reverts to implicit-all)"
+ (let [with-it (lib/with-join-source-fields base-join [(meta/field-metadata :categories :id)])]
+ (is (not (contains? (first (:stages (lib/with-join-source-fields with-it nil))) :fields)))
+ (is (not (contains? (first (:stages (lib/with-join-source-fields with-it []))) :fields)))))
+ (testing "Rejects a native-stage inner with a clear error (not a malli output-schema mess)"
+ (let [native-join (assoc base-join :stages [{:lib/type :mbql.stage/native
+ :lib/options {:lib/uuid (str (random-uuid))}
+ :native "SELECT 1"}])]
+ #?(:clj (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"requires the join's first stage to be an MBQL stage"
+ (lib/with-join-source-fields native-join
+ [(meta/field-metadata :categories :id)])))
+ :cljs (is (thrown-with-msg? js/Error
+ #"requires the join's first stage to be an MBQL stage"
+ (lib/with-join-source-fields native-join
+ [(meta/field-metadata :categories :id)]))))))))
+
+(defn- dangling-join-refs
+ "Refs in `query` that point into `a-join` but don't resolve to a column its inner pipeline returns.
+ Empty means the join's projection is coherent."
+ [query a-join]
+ (let [join-alias (:alias a-join)
+ inner (lib.metadata.calculation/returned-columns (assoc query :stages (:stages a-join)))
+ into-join (lib.util.match/match-many query
+ [:field (o :guard (= (:join-alias o) join-alias)) _] &match)]
+ (into #{}
+ (remove (fn [r]
+ (lib.equality/find-matching-column
+ query -1 (lib.options/update-options r dissoc :join-alias :source-field) inner)))
+ into-join)))
+
+(deftest ^:parallel with-join-source-fields-throws-on-stranded-refs-test
+ (let [base (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ a-join (-> (lib/join-clause (meta/table-metadata :products)
+ [(lib/= (meta/field-metadata :orders :product-id)
+ (meta/field-metadata :products :id))])
+ (lib/with-join-fields [(meta/field-metadata :products :category)]))
+ query (lib/join base a-join)
+ the-join (first (lib/joins query))]
+ (testing "throws when narrowing drops a column the join condition references"
+ (is (thrown? #?(:clj clojure.lang.ExceptionInfo :cljs js/Error)
+ (lib/with-join-source-fields the-join [(meta/field-metadata :products :category)]))))
+ (testing "throws when narrowing drops a column the join exposes"
+ (is (thrown? #?(:clj clojure.lang.ExceptionInfo :cljs js/Error)
+ (lib/with-join-source-fields the-join [(meta/field-metadata :products :id)]))))))
+
+(deftest ^:parallel with-join-source-fields-allows-safe-narrowing-test
+ (testing "narrowing that keeps the condition and exposed columns does not throw"
+ (let [base (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ a-join (-> (lib/join-clause (meta/table-metadata :products)
+ [(lib/= (meta/field-metadata :orders :product-id)
+ (meta/field-metadata :products :id))])
+ (lib/with-join-fields [(meta/field-metadata :products :category)]))
+ query (lib/join base a-join)
+ the-join (first (lib/joins query))
+ narrowed (lib/with-join-source-fields the-join [(meta/field-metadata :products :id)
+ (meta/field-metadata :products :category)])
+ query' (assoc-in query [:stages 0 :joins 0] narrowed)]
+ (is (empty? (dangling-join-refs query' narrowed))))))
+
+(deftest ^:parallel stranded-join-refs-test
+ (let [query (lib/join (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (-> (lib/join-clause (meta/table-metadata :products)
+ [(lib/= (meta/field-metadata :orders :product-id)
+ (meta/field-metadata :products :id))])
+ (lib/with-join-fields [(meta/field-metadata :products :category)])))
+ the-join (first (lib/joins query))
+ id-col (meta/field-metadata :products :id)
+ cat-col (meta/field-metadata :products :category)]
+ (testing "nothing stranded when cols cover the condition and exposed columns"
+ (is (empty? (#'lib.join/stranded-join-refs the-join [id-col cat-col]))))
+ (testing "reports the condition column when cols drop it"
+ (is (= [(:id id-col)]
+ (mapv peek (#'lib.join/stranded-join-refs the-join [cat-col])))))
+ (testing "reports an exposed column when cols drop it"
+ (is (= [(:id cat-col)]
+ (mapv peek (#'lib.join/stranded-join-refs the-join [id-col])))))
+ (testing "a join with no alias and :fields :all has nothing to strand"
+ (let [raw (lib/join-clause (meta/table-metadata :categories)
+ [(lib/= (meta/field-metadata :venues :category-id)
+ (meta/field-metadata :categories :id))])]
+ (is (empty? (#'lib.join/stranded-join-refs raw [(meta/field-metadata :categories :id)])))))))
+
(deftest ^:parallel join-clause-test
(testing "Should have :fields :all by default (#32419)"
(is (=? {:lib/type :mbql/join
diff --git a/test/metabase/parameters/chain_filter_test.clj b/test/metabase/parameters/chain_filter_test.clj
index ade0631c4b3..159c7be7a23 100644
--- a/test/metabase/parameters/chain_filter_test.clj
+++ b/test/metabase/parameters/chain_filter_test.clj
@@ -1,10 +1,15 @@
(ns metabase.parameters.chain-filter-test
(:require
+ [clojure.set :as set]
[clojure.test :refer :all]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.util :as lib.util]
+ [metabase.lib.walk :as lib.walk]
[metabase.parameters.chain-filter :as chain-filter]
[metabase.parameters.field-values :as params.field-values]
+ [metabase.query-processor.compile :as qp.compile]
+ [metabase.sql-tools.core :as sql-tools]
[metabase.test :as mt]
[metabase.util :as u]
[metabase.util.json :as json]
@@ -582,6 +587,219 @@
:has_more_values false}
(chain-filter-search venues.category_id {venues.price 4} "zzzzz"))))))
+;;; ------------ Structural invariant: tight inner-stage :fields per join ------------
+;;;
+;;; Two failure modes the invariant rules out:
+;;;
+;;; - UNDER-projection — a column the query references via this join is missing from the
+;;; inner :fields. SQL won't compile (or projects the wrong shape) and the query returns
+;;; wrong/missing values.
+;;;
+;;; - OVER-projection — the inner :fields contains a column nothing references. SQL
+;;; compiles fine and values are correct, but we're materializing columns we don't need,
+;;; which on wide tables could OOM an engine.
+
+(defn- referenced-field-ids-by-join-alias
+ "Return `{alias #{field-id ...}}` — for every `[:field {:join-alias A} id]` ref in `query`'s outer stage (including
+ refs in each outer-stage join's `:conditions` and outer `:fields`), bucket by `A`. Does NOT recurse into joins'
+ inner stages or any nested query — aliases are stage-scoped, so merging refs across scopes would silently
+ mis-attribute fields to the wrong join."
+ [query]
+ (let [acc (volatile! {})
+ collect (fn [clause]
+ (when (lib.util/clause-of-type? clause :field)
+ (let [[_ opts id] clause
+ alias (:join-alias opts)]
+ (when (and alias (pos-int? id))
+ (vswap! acc update alias (fnil conj #{}) id))))
+ nil)]
+ (lib.walk/walk-clauses-in-stage (lib.util/query-stage query -1) collect)
+ (doseq [a-join (lib/joins query)
+ clause (concat (lib/join-conditions a-join)
+ (let [outer-fields (lib/join-fields a-join)]
+ (when (sequential? outer-fields) outer-fields)))]
+ (lib.walk/walk-clause clause collect))
+ @acc))
+
+(defn- inner-projection-field-ids
+ "Set of field-ids that `a-join`'s inner-stage `:fields` projects. Structural read: the projection list is the thing
+ under test."
+ [a-join]
+ (into #{} (mapcat lib/all-field-ids) (:fields (first (:stages a-join)))))
+
+(defn- inner-projection-by-join-alias
+ "Return `{alias #{field-id ...}}` — the field-ids each join's inner-stage `:fields` projects, keyed by the join's
+ alias."
+ [query]
+ (into {}
+ (for [a-join (lib/joins query)
+ :let [a-alias (lib/current-join-alias a-join)]
+ :when a-alias]
+ [a-alias (inner-projection-field-ids a-join)])))
+
+(defn- projection-violations
+ "Return a seq of diagnostic maps (one per offending join's alias) or nil if the invariant holds: each join's
+ inner-stage `:fields` equals exactly the field-ids the rest of the query references through that join's alias."
+ [query]
+ (let [refs (referenced-field-ids-by-join-alias query)
+ proj (inner-projection-by-join-alias query)]
+ (not-empty
+ (vec
+ (for [a-alias (sort (into (set (keys refs)) (keys proj)))
+ :let [r (get refs a-alias #{})
+ p (get proj a-alias #{})]
+ :when (not= r p)]
+ {:alias a-alias
+ :referenced r
+ :projected p
+ :missing-from-projection (set/difference r p)
+ :extra-in-projection (set/difference p r)})))))
+
+(defn- mbql-for
+ "Build the chain-filter MBQL query for a given field/original/constraints triple."
+ [field-id original-field-id constraints]
+ (#'chain-filter/chain-filter-mbql-query
+ field-id
+ (when (seq constraints)
+ (vec (for [[fid v] constraints]
+ (shorthand->constraint fid v))))
+ (when original-field-id {:original-field-id original-field-id})))
+
+(defn- sql-over-projections
+ "Return a map of `{table-id {:declared #{…} :actual #{…} :extra #{…}}}` for any joined Table whose compiled-SQL
+ field references aren't a subset of the MBQL inner-stage `:fields` for that join. Returns nil if the SQL is tight.
+
+ Keyed by table-id (not alias) because `sql-tools/referenced-fields` reports columns by `:table-id`, and the SQL
+ emitter aggregates across joins to the same table. When multiple joins target the same table-id, `:declared` is the
+ union of their inner-stage projections."
+ [query]
+ (let [driver (t2/select-one-fn :engine :model/Database :id (:database query))
+ sql (:query (qp.compile/compile query))
+ nq (lib/native-query query sql)
+ sql-refs (reduce (fn [m {:keys [table-id id]}]
+ (update m table-id (fnil conj #{}) id))
+ {}
+ (sql-tools/referenced-fields driver nq))
+ mbql-proj (reduce (fn [m a-join]
+ (let [tid (:id (lib/joined-thing query a-join))]
+ (cond-> m
+ tid (update tid (fnil into #{}) (inner-projection-field-ids a-join)))))
+ {} (lib/joins query))]
+ (not-empty
+ (into {}
+ (for [[tid declared] mbql-proj
+ :let [actual (get sql-refs tid #{})
+ extra (set/difference actual declared)]
+ :when (seq extra)]
+ [tid {:declared declared, :actual actual, :extra extra}])))))
+
+(defn- check-tight-projections
+ "Run both the MBQL-level and SQL-level invariants on a chain-filter query."
+ [query]
+ (testing "MBQL: each join's inner-stage :fields equals the fields referenced on its source table"
+ (is (nil? (projection-violations query))))
+ (testing "SQL: the compiled query references no joined-Table columns outside MBQL :fields"
+ (is (nil? (sql-over-projections query)))))
+
+;;; Scenarios that exercise the join-building paths in `chain-filter-mbql-query`. Each
+;;; `:build` thunk runs inside `mt/dataset test-data` so `mt/id` resolves correctly.
+(def ^:private projection-scenarios
+ [{:label "no-joins"
+ :build #(mbql-for (mt/id :categories :name) nil nil)}
+ {:label "fk-remap-only (#74154)"
+ :build #(mbql-for (mt/id :categories :name) (mt/id :venues :category_id) nil)}
+ {:label "fk-remap + same-table constraint"
+ :build #(mbql-for (mt/id :categories :name) (mt/id :venues :category_id)
+ {(mt/id :venues :price) 4})}
+ {:label "constraint only, reverse-direction join"
+ :build #(mbql-for (mt/id :categories :name) nil {(mt/id :venues :price) 4})}
+ {:label "constraint on a different joined table"
+ :build #(mbql-for (mt/id :venues :name) nil {(mt/id :categories :name) "BBQ"})}
+ {:label "multi-hop chain (categories→venues→checkins→users)"
+ :build #(mbql-for (mt/id :categories :name) nil
+ {(mt/id :users :name) "Charles Lindbergh"})}
+ {:label "multi-hop + multi-table constraints"
+ :build #(mbql-for (mt/id :categories :name) nil
+ {(mt/id :venues :price) 4
+ (mt/id :users :name) "Charles Lindbergh"})}])
+
+(deftest ^:parallel tight-projections-test
+ (mt/dataset test-data
+ (doseq [{:keys [label build]} projection-scenarios]
+ (testing label
+ (check-tight-projections (build))))))
+
+(deftest ^:parallel tighten-leaves-card-source-joins-untouched-test
+ ;; chain-filter never produces card-source joins today; this test guards against silent breakage if a future caller
+ ;; uses tighten-join-projections on a query that does. Without the `:metadata/table` check in tighten, the join's
+ ;; inner-stage `:fields` would be dissoc'd (because `:id` of a card metadata won't match any field's `:table-id`),
+ ;; re-exposing the original add-implicit-clauses expansion bug for that join.
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card temp-card {:dataset_query {:database (mt/id)
+ :type :query
+ :query {:source-table (mt/id :venues)}}}]
+ (let [;; Build a real chain-filter query, then synthesize a card-source join by swapping `:source-table` for
+ ;; `:source-card` on the venues join's inner stage. The Card just needs to exist as a metadata-provider
+ ;; entry — its actual query doesn't matter for this structural check.
+ q (mbql-for (mt/id :categories :name) (mt/id :venues :category_id) nil)
+ q-mocked (-> q
+ (update-in [:stages 0 :joins 0 :stages 0] dissoc :source-table)
+ (assoc-in [:stages 0 :joins 0 :stages 0 :source-card] (:id temp-card)))
+ inner-before (first (:stages (first (lib/joins q-mocked))))
+ q-tightened (#'chain-filter/tighten-join-projections q-mocked)
+ inner-after (first (:stages (first (lib/joins q-tightened))))]
+ (is (= (:fields inner-before) (:fields inner-after))
+ "tighten must not modify :fields on a card-source join's inner stage")))))
+
+(defn- find-partition-filter
+ "Return the `[:> [:field {:join-alias join-alias} partition-fid] _]` clause from `query`'s outer-stage `:filters`,
+ or nil if none. `:join-alias` on the field ref must equal `join-alias` exactly (pass `nil` to match a source-table
+ ref with no `:join-alias`)."
+ [query partition-fid join-alias]
+ (some (fn [clause]
+ (and (vector? clause)
+ (= :> (first clause))
+ (let [field (nth clause 2 nil)]
+ (and (vector? field)
+ (= :field (first field))
+ (= partition-fid (last field))
+ (= join-alias (:join-alias (second field)))
+ clause))))
+ (-> query :stages first :filters)))
+
+(deftest ^:sequential chain-filter-preserves-required-partition-filter-on-joined-table-test
+ ;; `add-required-filters-if-needed` runs near the end of `chain-filter-mbql-query`, after joins are built. For
+ ;; tables that require a partition filter (currently BigQuery partitioned tables), it adds a `[:> partition-col _]`
+ ;; clause; when the partition column lives on a *joined* table, the clause references it through the join's alias.
+ ;;
+ ;; A fix that narrows the join's projection up front — and doesn't account for filters added later — silently
+ ;; elides the partition filter, because the joined-Table column it would reference is no longer in
+ ;; `visible-columns`. That's a behavior regression on BigQuery.
+ ;;
+ ;; The query below is chosen so venues becomes a JOIN target (source = categories, constraint on venues.id), and
+ ;; the partition column (venues.price) is not referenced by anything the user wrote. So if tighten-join-projections
+ ;; correctly survives the late-added partition filter, venues.price ends up in the venues join's inner stage
+ ;; *only because of the partition filter middleware*.
+ (mt/dataset test-data
+ (let [venues-id (mt/id :venues)
+ partition-fid (mt/id :venues :price)]
+ (mt/with-temp-vals-in-db :model/Table venues-id {:database_require_filter true}
+ (mt/with-temp-vals-in-db :model/Field partition-fid {:database_partitioned true}
+ (let [q (mbql-for (mt/id :categories :name)
+ nil
+ {(mt/id :venues :id) 1})
+ venues-alias (some (fn [j]
+ (when (= venues-id (:id (lib/joined-thing q j)))
+ (lib/current-join-alias j)))
+ (lib/joins q))]
+ (testing "partition filter clause is present in :filters, referencing the joined-table alias"
+ (is (some? (find-partition-filter q partition-fid venues-alias))
+ (str "expected a [:> [:field {:join-alias " (pr-str venues-alias) "} " partition-fid "] _] clause "
+ "in :filters; got: " (pr-str (-> q :stages first :filters)))))
+ (testing "venues join projects partition column in its inner stage"
+ (is (contains? (get (inner-projection-by-join-alias q) venues-alias) partition-fid)
+ (str "expected " partition-fid " in inner stage of venues join (" (pr-str venues-alias) ")")))))))))
+
;; Detail: Key (entity_id)=(6nmVTpCpKFRkZJigvqSVm) already exists.
(deftest use-cached-field-values-test
(testing "chain-filter should use cached FieldValues if applicable (#13832)"