The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Optimize chain filter inner field projections (#75209)
Timothy S. Deanb1b5d4906bd6
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex dfe19f5e3fa..841d85837e8 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -796,6 +796,14 @@stage-number :- :int](lib.join/joins a-query stage-number)))+(mu/defn joined-thing :- [:maybe ::lib.join/joinable]+ "Return metadata about the origin of `a-join` (the joined Table or Card).++ **Code Health:** Healthy. This is a core API."+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ a-join :- ::lib.join.util/partial-join]+ (lib.join/joined-thing metadata-providerable a-join))+;;; ### Join Strategies(mu/defn join-strategy :- ::lib.schema.join/strategy.option@@ -1018,6 +1026,20 @@fields :- [:maybe [:or [:enum :all :none] [:sequential some?]]]] ;; TODO: More precise schema.(lib.join/with-join-fields a-join fields))+(mu/defn with-join-source-fields :- ::lib.join.util/partial-join+ "Set the `:fields` projection on the join's source subquery (the first stage of `a-join`). `cols` is a coll of+ column metadatas from the source Table or Card; `nil`/empty dissocs, reverting to implicit-all.++ Throws if the join's first stage is not an MBQL stage, or if narrowing the source to `cols` would strand a column the+ join still references in its conditions or its exposed `:fields`.++ For what the join EXPOSES to its outer stage, see [[with-join-fields]].++ **Code Health:** Healthy. This is a core API."+ [a-join :- ::lib.join.util/partial-join+ cols :- [:maybe [:sequential some?]]] ; ideally [:sequential ::lib.schema.metadata/column]+ (lib.join/with-join-source-fields a-join cols))+(mu/defn join-fieldable-columns :- ::lib.metadata.calculation/visible-columns"Returns the list of column metadata for the columns which are *visible* on the RHS of `a-joinable`, such as a table,card or join clause.diff --git a/src/metabase/lib/field.cljc b/src/metabase/lib/field.cljcindex 0cfb4f8b677..dc41ec91326 100644--- a/src/metabase/lib/field.cljc+++ b/src/metabase/lib/field.cljc@@ -492,8 +492,36 @@:include-expressions? true:include-implicitly-joinable? false})))+(defn- newly-stranded-downstream-refs+ "Field refs in stages after `stage-number` that resolve against `before` but not against `after`. Non-empty means+ narrowing a stage's `:fields` from `before` to `after` left a later stage referencing a column that stage no longer+ returns."+ [before after stage-number]+ (let [stage-number (lib.util/canonical-stage-index before stage-number)+ bad-refs (fn [query]+ (let [bad (volatile! #{})]+ (lib.walk/walk-clauses+ query+ (fn [query path-type path clause]+ (when (and (= path-type :lib.walk/stage)+ (> (second path) stage-number)+ (vector? clause)+ (= (first clause) :field))+ (let [col (lib.walk/apply-f-for-stage-at-path+ lib.field.resolution/resolve-field-ref query path clause)]+ (when (or (not col)+ (::lib.field.resolution/fallback-metadata? col)+ (not (:active col true)))+ (vswap! bad conj clause))))+ nil))+ @bad))]+ (set/difference (bad-refs after) (bad-refs before))))+(mu/defn with-fields :- ::lib.schema/query- "Specify the `:fields` for a query. Pass `nil` or an empty sequence to remove `:fields`."+ "Specify the `:fields` for a query. Pass `nil` or an empty sequence to remove `:fields`.++ Throws if narrowing the projection would strand a reference in a later stage (a column that stage no longer+ returns)."([xs](fn [query stage-number](with-fields query stage-number xs)))@@ -513,8 +541,13 @@(or xs []));; Those expr-refs which must still be included.to-add (remove included expr-cols)- xs (when xs (into xs (map lib.ref/ref) to-add))]- (lib.util/update-query-stage query stage-number u/assoc-dissoc :fields xs))))+ xs (when xs (into xs (map lib.ref/ref) to-add))+ result (lib.util/update-query-stage query stage-number u/assoc-dissoc :fields xs)]+ (when (and xs (lib.util/next-stage-number query stage-number))+ (when-let [stranded (not-empty (newly-stranded-downstream-refs query result stage-number))]+ (throw (ex-info "with-fields would strand downstream references to dropped columns"+ {:stage-number stage-number, :stranded stranded}))))+ result)))(mu/defn fields :- [:maybe [:ref ::lib.schema/fields]]"Fetches the `:fields` for a query. Returns `nil` if there are no `:fields`. `:fields` should never be empty; this isdiff --git a/src/metabase/lib/join.cljc b/src/metabase/lib/join.cljcindex bfcbd916342..9c3a94f8e4f 100644--- a/src/metabase/lib/join.cljc+++ b/src/metabase/lib/join.cljc@@ -515,6 +515,45 @@fields)))](u/assoc-dissoc joinable :fields fields)))+(defn- stranded-join-refs+ "Refs in `a-join`'s conditions or exposed `:fields` that point at source columns missing from `cols` (a proposed new+ source projection). Non-empty means narrowing the source to `cols` would leave the join referencing columns its+ subquery no longer returns."+ [a-join cols]+ (let [join-alias (:alias a-join)+ field-refs (when (sequential? (:fields a-join))+ (:fields a-join))+ cond-refs (when join-alias+ (match/match-many (:conditions a-join)+ [:field (opts :guard (= (:join-alias opts) join-alias)) _] &match))]+ (into []+ (remove (fn [a-ref]+ (lib.equality/find-matching-column+ (lib.options/update-options a-ref dissoc :join-alias :source-field) cols)))+ (concat field-refs cond-refs))))++(mu/defn with-join-source-fields :- ::lib.join.util/partial-join+ "Set the `:fields` projection on the join's source subquery (the first stage of `a-join`). `cols` is a coll of+ column metadatas from the source Table or Card; `nil`/empty dissocs, reverting to implicit-all.++ Throws if the join's first stage is not an MBQL stage, or if narrowing the source to `cols` would strand a column the+ join still references in its conditions or its exposed `:fields`.++ For what the join EXPOSES to its outer stage, see [[with-join-fields]]."+ [a-join :- ::lib.join.util/partial-join+ cols :- [:maybe [:sequential some?]]] ; ideally [:sequential ::lib.schema.metadata/column]+ (let [first-stage-type (-> a-join :stages first :lib/type)]+ (when-not (= :mbql.stage/mbql first-stage-type)+ (throw (ex-info "with-join-source-fields requires the join's first stage to be an MBQL stage"+ {:first-stage-type first-stage-type+ :join a-join}))))+ (let [refs (not-empty (mapv lib.ref/ref cols))]+ (when refs+ (when-let [stranded (not-empty (stranded-join-refs a-join cols))]+ (throw (ex-info "with-join-source-fields would strand references the join still needs"+ {:join a-join, :cols cols, :stranded stranded}))))+ (update-in a-join [:stages 0] u/assoc-dissoc :fields refs)))+(defn- select-home-column[home-cols cond-fields](when (seq cond-fields)diff --git a/src/metabase/parameters/chain_filter.clj b/src/metabase/parameters/chain_filter.cljindex cac486e1afa..e2f9a9fa0c1 100644--- a/src/metabase/parameters/chain_filter.clj+++ b/src/metabase/parameters/chain_filter.clj@@ -73,6 +73,7 @@[metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.lib.types.isa :as lib.types.isa]+ [metabase.lib.util :as lib.util][metabase.parameters.chain-filter.dedupe-joins :as dedupe][metabase.parameters.field-values :as params.field-values][metabase.parameters.params :as params]@@ -403,6 +404,32 @@queryjoins))+(mu/defn- tighten-join-projections :- ::lib.schema/query+ "Narrow each join's inner-stage `:fields` to exactly the field-ids `query` references whose `:table-id` matches the+ join's source table.++ Assumes no two joins in stage 0 target the same `:table-id` (true for chain-filter's `joined-table-alias`+ convention). If that ever breaks, both joins get the union of fields — still correct, only over-projection."+ [query :- ::lib.schema/query]+ (if (empty? (lib/joins query))+ query+ ;; Bucket by `:table-id`, not by join alias: aliases are stage-scoped and the same string can recur in nested+ ;; scopes referring to different things. Table-id is the field's natural grain.+ (let [field-ids (lib/all-field-ids query)+ cols (lib.metadata/bulk-metadata query :metadata/column field-ids)+ cols-by-tid (group-by :table-id cols)]+ (lib.util/update-query-stage+ query 0+ update :joins+ (fn [the-joins]+ (mapv (fn [a-join]+ (let [thing (lib/joined-thing query a-join)+ tid (when (= :metadata/table (:lib/type thing))+ (:id thing))]+ (cond-> a-join+ tid (lib/with-join-source-fields (get cols-by-tid tid)))))+ the-joins))))))+(mr/def ::options;; if original-field-id is specified, we'll include this in the results. For Field->Field remapping.[:map {:closed true}@@ -473,7 +500,11 @@(lib/order-by field))(not original-field) (lib/breakout field))(add-filters source-table-id joined-table-ids constraints)- schema.metadata-queries/add-required-filters-if-needed)))+ schema.metadata-queries/add-required-filters-if-needed+ ;; Runs LAST so it picks up joined-table field refs injected by middleware above (e.g. BigQuery partition+ ;; filters). Without an explicit projection here the join's inner stage gets expanded to every column on the+ ;; joined Table by add-implicit-clauses, OOM'ing on wide fact tables.+ tighten-join-projections)));;; ------------------------ Chain filter (powers GET /api/dashboard/:id/params/:key/values) -------------------------diff --git a/test/metabase/lib/field_test.cljc b/test/metabase/lib/field_test.cljcindex 3f2dd9f0fde..4831a738aa7 100644--- a/test/metabase/lib/field_test.cljc+++ b/test/metabase/lib/field_test.cljc@@ -25,6 +25,7 @@[metabase.lib.test-util.notebook-helpers :as lib.tu.notebook][metabase.lib.types.isa :as lib.types.isa][metabase.lib.util :as lib.util]+ [metabase.lib.validate :as lib.validate][metabase.util :as u][metabase.util.malli :as mu]))@@ -803,6 +804,50 @@{:name "myadd"}]metadatas)))))+(deftest ^:parallel with-fields-throws-on-stranded-downstream-refs-test+ (testing "narrowing a stage's :fields throws if it strands a later-stage reference"+ (let [base (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ lib/append-stage)+ total (->> (lib.metadata.calculation/visible-columns base 1)+ (m/find-first (comp #{"TOTAL"} :name)))+ base (lib/order-by base 1 total :asc)+ id-col (->> (lib.metadata.calculation/returned-columns base 0)+ (m/find-first (comp #{"ID"} :name)))]+ (is (thrown? #?(:clj clojure.lang.ExceptionInfo :cljs js/Error)+ (lib/with-fields base 0 [id-col]))))))++(deftest ^:parallel with-fields-allows-safe-narrowing-test+ (testing "narrowing that keeps every referenced column does not throw"+ (let [base (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ lib/append-stage)+ total (->> (lib.metadata.calculation/visible-columns base 1)+ (m/find-first (comp #{"TOTAL"} :name)))+ base (lib/order-by base 1 total :asc)+ kept (->> (lib.metadata.calculation/returned-columns base 0)+ (filter (comp #{"ID" "TOTAL"} :name)))+ narrowed (lib/with-fields base 0 kept)]+ (is (empty? (lib.validate/find-bad-refs narrowed))))))++(deftest ^:parallel newly-stranded-downstream-refs-test+ (let [before (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ lib/append-stage)+ total (->> (lib.metadata.calculation/visible-columns before 1)+ (m/find-first (comp #{"TOTAL"} :name)))+ before (lib/order-by before 1 total :asc)+ id-ref (lib/ref (->> (lib.metadata.calculation/returned-columns before 0)+ (m/find-first (comp #{"ID"} :name))))+ total-ref (lib/ref (->> (lib.metadata.calculation/returned-columns before 0)+ (m/find-first (comp #{"TOTAL"} :name))))+ narrow (fn [refs] (lib.util/update-query-stage before 0 assoc :fields refs))]+ (testing "reports a later-stage ref the narrowing drops"+ (is (= #{"TOTAL"}+ (into #{} (map peek)+ (#'lib.field/newly-stranded-downstream-refs before (narrow [id-ref]) 0)))))+ (testing "reports nothing when the narrowing keeps the referenced column"+ (is (empty? (#'lib.field/newly-stranded-downstream-refs before (narrow [id-ref total-ref]) 0))))+ (testing "considers only stages after stage-number"+ (is (empty? (#'lib.field/newly-stranded-downstream-refs before (narrow [id-ref]) 1))))))+(deftest ^:parallel fieldable-columns-test(testing "query with no :fields"(is (=? [{:lib/source-column-alias "ID", :selected? true}diff --git a/test/metabase/lib/join_test.cljc b/test/metabase/lib/join_test.cljcindex 669fe900c8a..3bf8cdb0665 100644--- a/test/metabase/lib/join_test.cljc+++ b/test/metabase/lib/join_test.cljc@@ -4,6 +4,7 @@[clojure.test :refer [are deftest is testing]][medley.core :as m][metabase.lib.core :as lib]+ [metabase.lib.equality :as lib.equality][metabase.lib.join :as lib.join][metabase.lib.join.util :as lib.join.util][metabase.lib.metadata :as lib.metadata]@@ -16,7 +17,8 @@[metabase.lib.test-util.mocks-31769 :as lib.tu.mocks-31769][metabase.lib.test-util.notebook-helpers :as lib.tu.notebook][metabase.lib.util :as lib.util]- [metabase.util :as u]))+ [metabase.util :as u]+ [metabase.util.match :as lib.util.match]))#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))@@ -62,6 +64,96 @@:args [(lib/ref (meta/field-metadata :venues :category-id))(lib/ref (meta/field-metadata :categories :id))]}]))))))+(deftest ^:parallel with-join-source-fields-test+ (let [base-join (lib/join-clause (meta/table-metadata :categories)+ [(lib/= (meta/field-metadata :venues :category-id)+ (meta/field-metadata :categories :id))])]+ (testing "Sets :fields on the join's first (source) stage from column metadatas"+ (let [result (lib/with-join-source-fields base-join [(meta/field-metadata :categories :id)])]+ (is (= 1 (count (:fields (first (:stages result))))))))+ (testing "nil or empty cols dissocs :fields (reverts to implicit-all)"+ (let [with-it (lib/with-join-source-fields base-join [(meta/field-metadata :categories :id)])]+ (is (not (contains? (first (:stages (lib/with-join-source-fields with-it nil))) :fields)))+ (is (not (contains? (first (:stages (lib/with-join-source-fields with-it []))) :fields)))))+ (testing "Rejects a native-stage inner with a clear error (not a malli output-schema mess)"+ (let [native-join (assoc base-join :stages [{:lib/type :mbql.stage/native+ :lib/options {:lib/uuid (str (random-uuid))}+ :native "SELECT 1"}])]+ #?(:clj (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"requires the join's first stage to be an MBQL stage"+ (lib/with-join-source-fields native-join+ [(meta/field-metadata :categories :id)])))+ :cljs (is (thrown-with-msg? js/Error+ #"requires the join's first stage to be an MBQL stage"+ (lib/with-join-source-fields native-join+ [(meta/field-metadata :categories :id)]))))))))++(defn- dangling-join-refs+ "Refs in `query` that point into `a-join` but don't resolve to a column its inner pipeline returns.+ Empty means the join's projection is coherent."+ [query a-join]+ (let [join-alias (:alias a-join)+ inner (lib.metadata.calculation/returned-columns (assoc query :stages (:stages a-join)))+ into-join (lib.util.match/match-many query+ [:field (o :guard (= (:join-alias o) join-alias)) _] &match)]+ (into #{}+ (remove (fn [r]+ (lib.equality/find-matching-column+ query -1 (lib.options/update-options r dissoc :join-alias :source-field) inner)))+ into-join)))++(deftest ^:parallel with-join-source-fields-throws-on-stranded-refs-test+ (let [base (lib/query meta/metadata-provider (meta/table-metadata :orders))+ a-join (-> (lib/join-clause (meta/table-metadata :products)+ [(lib/= (meta/field-metadata :orders :product-id)+ (meta/field-metadata :products :id))])+ (lib/with-join-fields [(meta/field-metadata :products :category)]))+ query (lib/join base a-join)+ the-join (first (lib/joins query))]+ (testing "throws when narrowing drops a column the join condition references"+ (is (thrown? #?(:clj clojure.lang.ExceptionInfo :cljs js/Error)+ (lib/with-join-source-fields the-join [(meta/field-metadata :products :category)]))))+ (testing "throws when narrowing drops a column the join exposes"+ (is (thrown? #?(:clj clojure.lang.ExceptionInfo :cljs js/Error)+ (lib/with-join-source-fields the-join [(meta/field-metadata :products :id)]))))))++(deftest ^:parallel with-join-source-fields-allows-safe-narrowing-test+ (testing "narrowing that keeps the condition and exposed columns does not throw"+ (let [base (lib/query meta/metadata-provider (meta/table-metadata :orders))+ a-join (-> (lib/join-clause (meta/table-metadata :products)+ [(lib/= (meta/field-metadata :orders :product-id)+ (meta/field-metadata :products :id))])+ (lib/with-join-fields [(meta/field-metadata :products :category)]))+ query (lib/join base a-join)+ the-join (first (lib/joins query))+ narrowed (lib/with-join-source-fields the-join [(meta/field-metadata :products :id)+ (meta/field-metadata :products :category)])+ query' (assoc-in query [:stages 0 :joins 0] narrowed)]+ (is (empty? (dangling-join-refs query' narrowed))))))++(deftest ^:parallel stranded-join-refs-test+ (let [query (lib/join (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (-> (lib/join-clause (meta/table-metadata :products)+ [(lib/= (meta/field-metadata :orders :product-id)+ (meta/field-metadata :products :id))])+ (lib/with-join-fields [(meta/field-metadata :products :category)])))+ the-join (first (lib/joins query))+ id-col (meta/field-metadata :products :id)+ cat-col (meta/field-metadata :products :category)]+ (testing "nothing stranded when cols cover the condition and exposed columns"+ (is (empty? (#'lib.join/stranded-join-refs the-join [id-col cat-col]))))+ (testing "reports the condition column when cols drop it"+ (is (= [(:id id-col)]+ (mapv peek (#'lib.join/stranded-join-refs the-join [cat-col])))))+ (testing "reports an exposed column when cols drop it"+ (is (= [(:id cat-col)]+ (mapv peek (#'lib.join/stranded-join-refs the-join [id-col])))))+ (testing "a join with no alias and :fields :all has nothing to strand"+ (let [raw (lib/join-clause (meta/table-metadata :categories)+ [(lib/= (meta/field-metadata :venues :category-id)+ (meta/field-metadata :categories :id))])]+ (is (empty? (#'lib.join/stranded-join-refs raw [(meta/field-metadata :categories :id)])))))))+(deftest ^:parallel join-clause-test(testing "Should have :fields :all by default (#32419)"(is (=? {:lib/type :mbql/joindiff --git a/test/metabase/parameters/chain_filter_test.clj b/test/metabase/parameters/chain_filter_test.cljindex ade0631c4b3..159c7be7a23 100644--- a/test/metabase/parameters/chain_filter_test.clj+++ b/test/metabase/parameters/chain_filter_test.clj@@ -1,10 +1,15 @@(ns metabase.parameters.chain-filter-test(:require+ [clojure.set :as set][clojure.test :refer :all][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.util :as lib.util]+ [metabase.lib.walk :as lib.walk][metabase.parameters.chain-filter :as chain-filter][metabase.parameters.field-values :as params.field-values]+ [metabase.query-processor.compile :as qp.compile]+ [metabase.sql-tools.core :as sql-tools][metabase.test :as mt][metabase.util :as u][metabase.util.json :as json]@@ -582,6 +587,219 @@:has_more_values false}(chain-filter-search venues.category_id {venues.price 4} "zzzzz"))))))+;;; ------------ Structural invariant: tight inner-stage :fields per join ------------+;;;+;;; Two failure modes the invariant rules out:+;;;+;;; - UNDER-projection — a column the query references via this join is missing from the+;;; inner :fields. SQL won't compile (or projects the wrong shape) and the query returns+;;; wrong/missing values.+;;;+;;; - OVER-projection — the inner :fields contains a column nothing references. SQL+;;; compiles fine and values are correct, but we're materializing columns we don't need,+;;; which on wide tables could OOM an engine.++(defn- referenced-field-ids-by-join-alias+ "Return `{alias #{field-id ...}}` — for every `[:field {:join-alias A} id]` ref in `query`'s outer stage (including+ refs in each outer-stage join's `:conditions` and outer `:fields`), bucket by `A`. Does NOT recurse into joins'+ inner stages or any nested query — aliases are stage-scoped, so merging refs across scopes would silently+ mis-attribute fields to the wrong join."+ [query]+ (let [acc (volatile! {})+ collect (fn [clause]+ (when (lib.util/clause-of-type? clause :field)+ (let [[_ opts id] clause+ alias (:join-alias opts)]+ (when (and alias (pos-int? id))+ (vswap! acc update alias (fnil conj #{}) id))))+ nil)]+ (lib.walk/walk-clauses-in-stage (lib.util/query-stage query -1) collect)+ (doseq [a-join (lib/joins query)+ clause (concat (lib/join-conditions a-join)+ (let [outer-fields (lib/join-fields a-join)]+ (when (sequential? outer-fields) outer-fields)))]+ (lib.walk/walk-clause clause collect))+ @acc))++(defn- inner-projection-field-ids+ "Set of field-ids that `a-join`'s inner-stage `:fields` projects. Structural read: the projection list is the thing+ under test."+ [a-join]+ (into #{} (mapcat lib/all-field-ids) (:fields (first (:stages a-join)))))++(defn- inner-projection-by-join-alias+ "Return `{alias #{field-id ...}}` — the field-ids each join's inner-stage `:fields` projects, keyed by the join's+ alias."+ [query]+ (into {}+ (for [a-join (lib/joins query)+ :let [a-alias (lib/current-join-alias a-join)]+ :when a-alias]+ [a-alias (inner-projection-field-ids a-join)])))++(defn- projection-violations+ "Return a seq of diagnostic maps (one per offending join's alias) or nil if the invariant holds: each join's+ inner-stage `:fields` equals exactly the field-ids the rest of the query references through that join's alias."+ [query]+ (let [refs (referenced-field-ids-by-join-alias query)+ proj (inner-projection-by-join-alias query)]+ (not-empty+ (vec+ (for [a-alias (sort (into (set (keys refs)) (keys proj)))+ :let [r (get refs a-alias #{})+ p (get proj a-alias #{})]+ :when (not= r p)]+ {:alias a-alias+ :referenced r+ :projected p+ :missing-from-projection (set/difference r p)+ :extra-in-projection (set/difference p r)})))))++(defn- mbql-for+ "Build the chain-filter MBQL query for a given field/original/constraints triple."+ [field-id original-field-id constraints]+ (#'chain-filter/chain-filter-mbql-query+ field-id+ (when (seq constraints)+ (vec (for [[fid v] constraints]+ (shorthand->constraint fid v))))+ (when original-field-id {:original-field-id original-field-id})))++(defn- sql-over-projections+ "Return a map of `{table-id {:declared #{…} :actual #{…} :extra #{…}}}` for any joined Table whose compiled-SQL+ field references aren't a subset of the MBQL inner-stage `:fields` for that join. Returns nil if the SQL is tight.++ Keyed by table-id (not alias) because `sql-tools/referenced-fields` reports columns by `:table-id`, and the SQL+ emitter aggregates across joins to the same table. When multiple joins target the same table-id, `:declared` is the+ union of their inner-stage projections."+ [query]+ (let [driver (t2/select-one-fn :engine :model/Database :id (:database query))+ sql (:query (qp.compile/compile query))+ nq (lib/native-query query sql)+ sql-refs (reduce (fn [m {:keys [table-id id]}]+ (update m table-id (fnil conj #{}) id))+ {}+ (sql-tools/referenced-fields driver nq))+ mbql-proj (reduce (fn [m a-join]+ (let [tid (:id (lib/joined-thing query a-join))]+ (cond-> m+ tid (update tid (fnil into #{}) (inner-projection-field-ids a-join)))))+ {} (lib/joins query))]+ (not-empty+ (into {}+ (for [[tid declared] mbql-proj+ :let [actual (get sql-refs tid #{})+ extra (set/difference actual declared)]+ :when (seq extra)]+ [tid {:declared declared, :actual actual, :extra extra}])))))++(defn- check-tight-projections+ "Run both the MBQL-level and SQL-level invariants on a chain-filter query."+ [query]+ (testing "MBQL: each join's inner-stage :fields equals the fields referenced on its source table"+ (is (nil? (projection-violations query))))+ (testing "SQL: the compiled query references no joined-Table columns outside MBQL :fields"+ (is (nil? (sql-over-projections query)))))++;;; Scenarios that exercise the join-building paths in `chain-filter-mbql-query`. Each+;;; `:build` thunk runs inside `mt/dataset test-data` so `mt/id` resolves correctly.+(def ^:private projection-scenarios+ [{:label "no-joins"+ :build #(mbql-for (mt/id :categories :name) nil nil)}+ {:label "fk-remap-only (#74154)"+ :build #(mbql-for (mt/id :categories :name) (mt/id :venues :category_id) nil)}+ {:label "fk-remap + same-table constraint"+ :build #(mbql-for (mt/id :categories :name) (mt/id :venues :category_id)+ {(mt/id :venues :price) 4})}+ {:label "constraint only, reverse-direction join"+ :build #(mbql-for (mt/id :categories :name) nil {(mt/id :venues :price) 4})}+ {:label "constraint on a different joined table"+ :build #(mbql-for (mt/id :venues :name) nil {(mt/id :categories :name) "BBQ"})}+ {:label "multi-hop chain (categories→venues→checkins→users)"+ :build #(mbql-for (mt/id :categories :name) nil+ {(mt/id :users :name) "Charles Lindbergh"})}+ {:label "multi-hop + multi-table constraints"+ :build #(mbql-for (mt/id :categories :name) nil+ {(mt/id :venues :price) 4+ (mt/id :users :name) "Charles Lindbergh"})}])++(deftest ^:parallel tight-projections-test+ (mt/dataset test-data+ (doseq [{:keys [label build]} projection-scenarios]+ (testing label+ (check-tight-projections (build))))))++(deftest ^:parallel tighten-leaves-card-source-joins-untouched-test+ ;; chain-filter never produces card-source joins today; this test guards against silent breakage if a future caller+ ;; uses tighten-join-projections on a query that does. Without the `:metadata/table` check in tighten, the join's+ ;; inner-stage `:fields` would be dissoc'd (because `:id` of a card metadata won't match any field's `:table-id`),+ ;; re-exposing the original add-implicit-clauses expansion bug for that join.+ (mt/dataset test-data+ (mt/with-temp [:model/Card temp-card {:dataset_query {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :venues)}}}]+ (let [;; Build a real chain-filter query, then synthesize a card-source join by swapping `:source-table` for+ ;; `:source-card` on the venues join's inner stage. The Card just needs to exist as a metadata-provider+ ;; entry — its actual query doesn't matter for this structural check.+ q (mbql-for (mt/id :categories :name) (mt/id :venues :category_id) nil)+ q-mocked (-> q+ (update-in [:stages 0 :joins 0 :stages 0] dissoc :source-table)+ (assoc-in [:stages 0 :joins 0 :stages 0 :source-card] (:id temp-card)))+ inner-before (first (:stages (first (lib/joins q-mocked))))+ q-tightened (#'chain-filter/tighten-join-projections q-mocked)+ inner-after (first (:stages (first (lib/joins q-tightened))))]+ (is (= (:fields inner-before) (:fields inner-after))+ "tighten must not modify :fields on a card-source join's inner stage")))))++(defn- find-partition-filter+ "Return the `[:> [:field {:join-alias join-alias} partition-fid] _]` clause from `query`'s outer-stage `:filters`,+ or nil if none. `:join-alias` on the field ref must equal `join-alias` exactly (pass `nil` to match a source-table+ ref with no `:join-alias`)."+ [query partition-fid join-alias]+ (some (fn [clause]+ (and (vector? clause)+ (= :> (first clause))+ (let [field (nth clause 2 nil)]+ (and (vector? field)+ (= :field (first field))+ (= partition-fid (last field))+ (= join-alias (:join-alias (second field)))+ clause))))+ (-> query :stages first :filters)))++(deftest ^:sequential chain-filter-preserves-required-partition-filter-on-joined-table-test+ ;; `add-required-filters-if-needed` runs near the end of `chain-filter-mbql-query`, after joins are built. For+ ;; tables that require a partition filter (currently BigQuery partitioned tables), it adds a `[:> partition-col _]`+ ;; clause; when the partition column lives on a *joined* table, the clause references it through the join's alias.+ ;;+ ;; A fix that narrows the join's projection up front — and doesn't account for filters added later — silently+ ;; elides the partition filter, because the joined-Table column it would reference is no longer in+ ;; `visible-columns`. That's a behavior regression on BigQuery.+ ;;+ ;; The query below is chosen so venues becomes a JOIN target (source = categories, constraint on venues.id), and+ ;; the partition column (venues.price) is not referenced by anything the user wrote. So if tighten-join-projections+ ;; correctly survives the late-added partition filter, venues.price ends up in the venues join's inner stage+ ;; *only because of the partition filter middleware*.+ (mt/dataset test-data+ (let [venues-id (mt/id :venues)+ partition-fid (mt/id :venues :price)]+ (mt/with-temp-vals-in-db :model/Table venues-id {:database_require_filter true}+ (mt/with-temp-vals-in-db :model/Field partition-fid {:database_partitioned true}+ (let [q (mbql-for (mt/id :categories :name)+ nil+ {(mt/id :venues :id) 1})+ venues-alias (some (fn [j]+ (when (= venues-id (:id (lib/joined-thing q j)))+ (lib/current-join-alias j)))+ (lib/joins q))]+ (testing "partition filter clause is present in :filters, referencing the joined-table alias"+ (is (some? (find-partition-filter q partition-fid venues-alias))+ (str "expected a [:> [:field {:join-alias " (pr-str venues-alias) "} " partition-fid "] _] clause "+ "in :filters; got: " (pr-str (-> q :stages first :filters)))))+ (testing "venues join projects partition column in its inner stage"+ (is (contains? (get (inner-projection-by-join-alias q) venues-alias) partition-fid)+ (str "expected " partition-fid " in inner stage of venues join (" (pr-str venues-alias) ")")))))))))+;; Detail: Key (entity_id)=(6nmVTpCpKFRkZJigvqSVm) already exists.(deftest use-cached-field-values-test(testing "chain-filter should use cached FieldValues if applicable (#13832)"