Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Match remapped fields with ID or name based fields refs (#78246)

Riley Thompsona37f29e7374a
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
index 7cecdeb569e..e545a085d0d 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
@@ -2117,3 +2117,60 @@
(-> (lib/query mp (lib.metadata/table mp (mt/id :products)))
(lib/with-fields [(lib/ref (lib.metadata/field mp (mt/id :products :id)))])
(lib/limit 20)))))))))))))
+
+(deftest fk-remapping-with-sandboxing-and-specific-field-test
+ (testing "FK remapping should work for questions against sandboxed tables that select specific fields (#78187)"
+ (met/with-gtaps! {:gtaps {:people {:query (mt/native-query
+ {:query "SELECT * FROM PEOPLE WHERE STATE = {{state}}"
+ :template-tags {"state" {:display-name "State"
+ :id "1"
+ :name "state"
+ :type :text}}})
+ :remappings {"state" [:variable [:template-tag "state"]]}}
+ :orders {:query (mt/native-query
+ {:query (str "SELECT ORDERS.* FROM ORDERS "
+ "LEFT JOIN PEOPLE ON PEOPLE.ID = ORDERS.USER_ID "
+ "WHERE PEOPLE.STATE = {{state}}")
+ :template-tags {"state" {:display-name "State"
+ :id "2"
+ :name "state"
+ :type :text}}})
+ :remappings {"state" [:variable [:template-tag "state"]]}}}
+ :attributes {"state" "CA"}}
+ (data-perms/set-table-permission! &group (mt/id :products) :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! &group (mt/id) :perms/view-data :unrestricted)
+ (let [mp (lib.tu/remap-metadata-provider
+ (mt/metadata-provider)
+ (mt/id :orders :user_id) (mt/id :people :name)
+ (mt/id :orders :product_id) (mt/id :products :title))
+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ (lib/with-fields [(lib.metadata/field mp (mt/id :orders :user_id))
+ (lib.metadata/field mp (mt/id :orders :product_id))])
+ (lib/join (-> (lib/join-clause (lib.metadata/table mp (mt/id :people))
+ [(lib/= (lib.metadata/field mp (mt/id :orders :user_id))
+ (lib.metadata/field mp (mt/id :people :id)))])
+ (lib/with-join-fields [(lib.metadata/field mp (mt/id :people :state))])))
+ (lib/join (-> (lib/join-clause (lib.metadata/table mp (mt/id :products))
+ [(lib/= (lib.metadata/field mp (mt/id :orders :product_id))
+ (lib.metadata/field mp (mt/id :products :id)))])
+ (lib/with-join-fields [(lib.metadata/field mp (mt/id :products :category))])))
+ (lib/order-by (lib.metadata/field mp (mt/id :people :name)) :asc)
+ (lib/order-by (lib.metadata/field mp (mt/id :products :title)) :asc)
+ (lib/limit 3))
+ result (qp/process-query query)
+ cols (mt/cols result)
+ col-by-name (m/index-by :name cols)]
+ (is (= [[624 144 "CA" "Widget" "Abbie Parisian" "Aerodynamic Bronze Hat"]
+ [624 94 "CA" "Widget" "Abbie Parisian" "Awesome Bronze Plate"]
+ [624 101 "CA" "Gadget" "Abbie Parisian" "Durable Cotton Bench"]]
+ (mt/rows result)))
+ (is (= "NAME" (:remapped_to (col-by-name "USER_ID"))))
+ (is (= "USER_ID" (:remapped_from (col-by-name "NAME"))))
+ (is (= "TITLE" (:remapped_to (col-by-name "PRODUCT_ID"))))
+ (is (= "PRODUCT_ID" (:remapped_from (col-by-name "TITLE"))))
+ (testing "every column with :remapped_from must point at a column with a matching :remapped_to (the FE errors otherwise)"
+ (doseq [col cols
+ :when (:remapped_from col)
+ :let [source-col (col-by-name (:remapped_from col))]]
+ (is (= (:name col)
+ (:remapped_to source-col)))))))))
diff --git a/src/metabase/query_processor/middleware/add_remaps.clj b/src/metabase/query_processor/middleware/add_remaps.clj
index 3917ec8054e..1ae8a62ade0 100644
--- a/src/metabase/query_processor/middleware/add_remaps.clj
+++ b/src/metabase/query_processor/middleware/add_remaps.clj
@@ -159,18 +159,25 @@
:human-readable-field-name (-> dimension :human-readable-field-id unique-name))})))))
(lib.walk/apply-f-for-stage-at-path lib/returned-columns query path)))))
+(mu/defn- original-fields->new-fields
+ [remap-infos :- [:maybe [:sequential ::remap-info]]]
+ (into {}
+ (mapcat (fn [{:keys [original-field-clause new-field-clause dimension]}]
+ (let [original-with-id (assoc original-field-clause 2 (:field-id dimension))]
+ [[(simplify-ref-options original-field-clause) new-field-clause]
+ [(simplify-ref-options original-with-id) new-field-clause]])))
+ remap-infos))
+
(mu/defn- add-fk-remaps-rewrite-existing-fields-add-original-field-dimension-id :- ::lib.schema/fields
"Rewrite existing `:fields` in a query. Add `::original-field-dimension-id` to any Field clauses that are
remapped-from."
[infos :- [:maybe [:sequential ::remap-info]]
fields :- ::lib.schema/fields]
- (let [field->remapped-col (into {}
- (map (fn [{:keys [original-field-clause new-field-clause]}]
- [(simplify-ref-options original-field-clause) new-field-clause]))
- infos)]
+ (let [field->remapped-col (original-fields->new-fields infos)]
(mapv
(fn [field-ref]
- (if-let [[_tag {::keys [new-field-dimension-id], :as _opts} _id-or-name] (field->remapped-col (simplify-ref-options field-ref))]
+ (if-let [[_tag {::keys [new-field-dimension-id], :as _opts} _id-or-name]
+ (field->remapped-col (simplify-ref-options field-ref))]
(lib/update-options field-ref assoc ::original-field-dimension-id new-field-dimension-id)
field-ref))
fields)))
@@ -265,10 +272,7 @@
;; if they do, update `:fields`, `:order-by` and `:breakout` clauses accordingly and add to the query
(let [new-fields (add-fk-remaps-to-fields infos fields)
;; make a map of field-id-clause -> fk-clause from the tuples
- original->remapped (into {}
- (map (fn [{:keys [original-field-clause new-field-clause]}]
- [(simplify-ref-options original-field-clause) new-field-clause]))
- infos)
+ original->remapped (original-fields->new-fields infos)
;; PERF: More indexing on the same stuff! This really needs to be poured into a common context.
new-breakout (add-fk-remaps-rewrite-breakout original->remapped breakout)
new-order-by (add-fk-remaps-rewrite-order-by original->remapped order-by)
diff --git a/src/metabase/query_processor/middleware/fix_bad_field_id_refs.clj b/src/metabase/query_processor/middleware/fix_bad_field_id_refs.clj
index 75ea6dedd36..c4f07a75865 100644
--- a/src/metabase/query_processor/middleware/fix_bad_field_id_refs.clj
+++ b/src/metabase/query_processor/middleware/fix_bad_field_id_refs.clj
@@ -1,15 +1,16 @@
(ns metabase.query-processor.middleware.fix-bad-field-id-refs
"Middleware that adds `:join-alias` info to `:field` clauses where needed."
- (:refer-clojure :exclude [get-in])
+ (:refer-clojure :exclude [get-in select-keys])
(:require
[metabase.lib.core :as lib]
[metabase.lib.field.resolution :as lib.field.resolution]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema :as lib.schema]
[metabase.lib.walk :as lib.walk]
+ [metabase.query-processor.middleware.add-remaps :as qp.add-remaps]
[metabase.util.malli :as mu]
[metabase.util.match :as match]
- [metabase.util.performance :refer [get-in]]))
+ [metabase.util.performance :refer [get-in select-keys]]))
(mu/defn- fix-bad-field-id-refs-in-stage :- [:maybe ::lib.schema/stage]
[query :- ::lib.schema/query
@@ -29,9 +30,11 @@
(when-let [resolved (lib.walk/apply-f-for-stage-at-path
lib.field.resolution/resolve-field-ref
query path &match)]
- (cond-> (lib/ref resolved)
- (:lib/expression-name opts)
- (lib/update-options assoc :lib/expression-name (:lib/expression-name opts))))))
+ (lib/update-options (lib/ref resolved)
+ merge
+ (select-keys opts [:lib/expression-name
+ ::qp.add-remaps/original-field-dimension-id
+ ::qp.add-remaps/new-field-dimension-id])))))
&match)))
stage' (update-fields stage)]
(when-not (= stage' stage)
diff --git a/test/metabase/query_processor/middleware/add_remaps_test.clj b/test/metabase/query_processor/middleware/add_remaps_test.clj
index 58d221171f3..9b391ce0129 100644
--- a/test/metabase/query_processor/middleware/add_remaps_test.clj
+++ b/test/metabase/query_processor/middleware/add_remaps_test.clj
@@ -181,6 +181,21 @@
::qp.add-remaps/new-field-dimension-id pos-int?}]]}})
(lib/->legacy-MBQL query))))))
+(deftest ^:parallel add-remapped-columns-with-previous-stage-test
+ (let [query (-> (lib/query category-id-remap-metadata-provider (meta/table-metadata :venues))
+ (lib/append-stage)
+ (lib/with-fields [(meta/field-metadata :venues :category-id)]))
+ dimension-id (get-in (lib.metadata/field category-id-remap-metadata-provider (meta/id :venues :category-id))
+ [:lib/external-remap :id])]
+ ;; The `lib/append-stage` models the sandbox behaviour where an extra stage is added
+ ;; and the `remap-column-infos` become name based instead of id based
+ (is (=? {:stages [{}
+ {:fields [[:field {::qp.add-remaps/original-field-dimension-id dimension-id}
+ (meta/id :venues :category-id)]
+ [:field {::qp.add-remaps/new-field-dimension-id dimension-id}
+ (meta/id :categories :name)]]}]}
+ (qp.add-remaps/add-remapped-columns query)))))
+
;;; ---------------------------------------- remap-results (post-processing) -----------------------------------------
(defn- remap-results [query metadata rows]
diff --git a/test/metabase/query_processor/middleware/fix_bad_field_id_refs_test.clj b/test/metabase/query_processor/middleware/fix_bad_field_id_refs_test.clj
index 91e36dd6356..026f9d4a074 100644
--- a/test/metabase/query_processor/middleware/fix_bad_field_id_refs_test.clj
+++ b/test/metabase/query_processor/middleware/fix_bad_field_id_refs_test.clj
@@ -7,6 +7,7 @@
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.macros :as lib.tu.macros]
+ [metabase.query-processor.middleware.add-remaps :as qp.add-remaps]
[metabase.query-processor.middleware.fix-bad-field-id-refs :as fix-bad-field-id-refs]
[metabase.query-processor.test :as qp]
[metabase.test :as mt]
@@ -316,6 +317,31 @@
(is (every? string? (keys exprs))
(str "Assert failed: :expressions should always use string keys, got: " (pr-str exprs))))))))
+(deftest ^:parallel preserve-remapping-keys-when-fixing-field-id-ref-test
+ (testing "rewriting bad ID refs preserves the remapping dimension keys added by add-remaps (#78187)"
+ ;; a legacy card-source query with ID refs is the same shape sandboxing produces: the stage has no
+ ;; :source-table, so both the remapped-from and remapped-to refs hit the rewrite branch here
+ (let [mp (-> meta/metadata-provider
+ (lib.tu/metadata-provider-with-cards-for-queries
+ [(lib/query meta/metadata-provider (meta/table-metadata :venues))])
+ (lib.tu/remap-metadata-provider
+ (meta/field-metadata :venues :category-id)
+ (meta/field-metadata :categories :name)))
+ query (lib/query mp {:database (meta/id)
+ :type :query
+ :query {:source-table "card__1"
+ :fields [[:field (meta/id :venues :category-id) nil]]}})
+ dimension-id (get-in (lib.metadata/field mp (meta/id :venues :category-id))
+ [:lib/external-remap :id])]
+ (is (=? {:stages [{:fields [[:field {::qp.add-remaps/original-field-dimension-id dimension-id}
+ "CATEGORY_ID"]
+ [:field {:source-field (meta/id :venues :category-id)
+ ::qp.add-remaps/new-field-dimension-id dimension-id}
+ (meta/id :categories :name)]]}]}
+ (-> query
+ qp.add-remaps/add-remapped-columns
+ fix-bad-field-id-refs/fix-bad-field-id-refs))))))
+
(deftest ^:parallel resolve-join-conditions-test
(testing "Resolve fields in join :conditions"
(let [mp (lib.tu/mock-metadata-provider