The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Match remapped fields with ID or name based fields refs (#78246)
Riley Thompsona37f29e7374a
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex 7cecdeb569e..e545a085d0d 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -2117,3 +2117,60 @@(-> (lib/query mp (lib.metadata/table mp (mt/id :products)))(lib/with-fields [(lib/ref (lib.metadata/field mp (mt/id :products :id)))])(lib/limit 20)))))))))))))++(deftest fk-remapping-with-sandboxing-and-specific-field-test+ (testing "FK remapping should work for questions against sandboxed tables that select specific fields (#78187)"+ (met/with-gtaps! {:gtaps {:people {:query (mt/native-query+ {:query "SELECT * FROM PEOPLE WHERE STATE = {{state}}"+ :template-tags {"state" {:display-name "State"+ :id "1"+ :name "state"+ :type :text}}})+ :remappings {"state" [:variable [:template-tag "state"]]}}+ :orders {:query (mt/native-query+ {:query (str "SELECT ORDERS.* FROM ORDERS "+ "LEFT JOIN PEOPLE ON PEOPLE.ID = ORDERS.USER_ID "+ "WHERE PEOPLE.STATE = {{state}}")+ :template-tags {"state" {:display-name "State"+ :id "2"+ :name "state"+ :type :text}}})+ :remappings {"state" [:variable [:template-tag "state"]]}}}+ :attributes {"state" "CA"}}+ (data-perms/set-table-permission! &group (mt/id :products) :perms/create-queries :query-builder)+ (data-perms/set-database-permission! &group (mt/id) :perms/view-data :unrestricted)+ (let [mp (lib.tu/remap-metadata-provider+ (mt/metadata-provider)+ (mt/id :orders :user_id) (mt/id :people :name)+ (mt/id :orders :product_id) (mt/id :products :title))+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/with-fields [(lib.metadata/field mp (mt/id :orders :user_id))+ (lib.metadata/field mp (mt/id :orders :product_id))])+ (lib/join (-> (lib/join-clause (lib.metadata/table mp (mt/id :people))+ [(lib/= (lib.metadata/field mp (mt/id :orders :user_id))+ (lib.metadata/field mp (mt/id :people :id)))])+ (lib/with-join-fields [(lib.metadata/field mp (mt/id :people :state))])))+ (lib/join (-> (lib/join-clause (lib.metadata/table mp (mt/id :products))+ [(lib/= (lib.metadata/field mp (mt/id :orders :product_id))+ (lib.metadata/field mp (mt/id :products :id)))])+ (lib/with-join-fields [(lib.metadata/field mp (mt/id :products :category))])))+ (lib/order-by (lib.metadata/field mp (mt/id :people :name)) :asc)+ (lib/order-by (lib.metadata/field mp (mt/id :products :title)) :asc)+ (lib/limit 3))+ result (qp/process-query query)+ cols (mt/cols result)+ col-by-name (m/index-by :name cols)]+ (is (= [[624 144 "CA" "Widget" "Abbie Parisian" "Aerodynamic Bronze Hat"]+ [624 94 "CA" "Widget" "Abbie Parisian" "Awesome Bronze Plate"]+ [624 101 "CA" "Gadget" "Abbie Parisian" "Durable Cotton Bench"]]+ (mt/rows result)))+ (is (= "NAME" (:remapped_to (col-by-name "USER_ID"))))+ (is (= "USER_ID" (:remapped_from (col-by-name "NAME"))))+ (is (= "TITLE" (:remapped_to (col-by-name "PRODUCT_ID"))))+ (is (= "PRODUCT_ID" (:remapped_from (col-by-name "TITLE"))))+ (testing "every column with :remapped_from must point at a column with a matching :remapped_to (the FE errors otherwise)"+ (doseq [col cols+ :when (:remapped_from col)+ :let [source-col (col-by-name (:remapped_from col))]]+ (is (= (:name col)+ (:remapped_to source-col)))))))))diff --git a/src/metabase/query_processor/middleware/add_remaps.clj b/src/metabase/query_processor/middleware/add_remaps.cljindex 3917ec8054e..1ae8a62ade0 100644--- a/src/metabase/query_processor/middleware/add_remaps.clj+++ b/src/metabase/query_processor/middleware/add_remaps.clj@@ -159,18 +159,25 @@:human-readable-field-name (-> dimension :human-readable-field-id unique-name))})))))(lib.walk/apply-f-for-stage-at-path lib/returned-columns query path)))))+(mu/defn- original-fields->new-fields+ [remap-infos :- [:maybe [:sequential ::remap-info]]]+ (into {}+ (mapcat (fn [{:keys [original-field-clause new-field-clause dimension]}]+ (let [original-with-id (assoc original-field-clause 2 (:field-id dimension))]+ [[(simplify-ref-options original-field-clause) new-field-clause]+ [(simplify-ref-options original-with-id) new-field-clause]])))+ remap-infos))+(mu/defn- add-fk-remaps-rewrite-existing-fields-add-original-field-dimension-id :- ::lib.schema/fields"Rewrite existing `:fields` in a query. Add `::original-field-dimension-id` to any Field clauses that areremapped-from."[infos :- [:maybe [:sequential ::remap-info]]fields :- ::lib.schema/fields]- (let [field->remapped-col (into {}- (map (fn [{:keys [original-field-clause new-field-clause]}]- [(simplify-ref-options original-field-clause) new-field-clause]))- infos)]+ (let [field->remapped-col (original-fields->new-fields infos)](mapv(fn [field-ref]- (if-let [[_tag {::keys [new-field-dimension-id], :as _opts} _id-or-name] (field->remapped-col (simplify-ref-options field-ref))]+ (if-let [[_tag {::keys [new-field-dimension-id], :as _opts} _id-or-name]+ (field->remapped-col (simplify-ref-options field-ref))](lib/update-options field-ref assoc ::original-field-dimension-id new-field-dimension-id)field-ref))fields)))@@ -265,10 +272,7 @@;; if they do, update `:fields`, `:order-by` and `:breakout` clauses accordingly and add to the query(let [new-fields (add-fk-remaps-to-fields infos fields);; make a map of field-id-clause -> fk-clause from the tuples- original->remapped (into {}- (map (fn [{:keys [original-field-clause new-field-clause]}]- [(simplify-ref-options original-field-clause) new-field-clause]))- infos)+ original->remapped (original-fields->new-fields infos);; PERF: More indexing on the same stuff! This really needs to be poured into a common context.new-breakout (add-fk-remaps-rewrite-breakout original->remapped breakout)new-order-by (add-fk-remaps-rewrite-order-by original->remapped order-by)diff --git a/src/metabase/query_processor/middleware/fix_bad_field_id_refs.clj b/src/metabase/query_processor/middleware/fix_bad_field_id_refs.cljindex 75ea6dedd36..c4f07a75865 100644--- a/src/metabase/query_processor/middleware/fix_bad_field_id_refs.clj+++ b/src/metabase/query_processor/middleware/fix_bad_field_id_refs.clj@@ -1,15 +1,16 @@(ns metabase.query-processor.middleware.fix-bad-field-id-refs"Middleware that adds `:join-alias` info to `:field` clauses where needed."- (:refer-clojure :exclude [get-in])+ (:refer-clojure :exclude [get-in select-keys])(:require[metabase.lib.core :as lib][metabase.lib.field.resolution :as lib.field.resolution][metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema][metabase.lib.walk :as lib.walk]+ [metabase.query-processor.middleware.add-remaps :as qp.add-remaps][metabase.util.malli :as mu][metabase.util.match :as match]- [metabase.util.performance :refer [get-in]]))+ [metabase.util.performance :refer [get-in select-keys]]))(mu/defn- fix-bad-field-id-refs-in-stage :- [:maybe ::lib.schema/stage][query :- ::lib.schema/query@@ -29,9 +30,11 @@(when-let [resolved (lib.walk/apply-f-for-stage-at-pathlib.field.resolution/resolve-field-refquery path &match)]- (cond-> (lib/ref resolved)- (:lib/expression-name opts)- (lib/update-options assoc :lib/expression-name (:lib/expression-name opts))))))+ (lib/update-options (lib/ref resolved)+ merge+ (select-keys opts [:lib/expression-name+ ::qp.add-remaps/original-field-dimension-id+ ::qp.add-remaps/new-field-dimension-id])))))&match)))stage' (update-fields stage)](when-not (= stage' stage)diff --git a/test/metabase/query_processor/middleware/add_remaps_test.clj b/test/metabase/query_processor/middleware/add_remaps_test.cljindex 58d221171f3..9b391ce0129 100644--- a/test/metabase/query_processor/middleware/add_remaps_test.clj+++ b/test/metabase/query_processor/middleware/add_remaps_test.clj@@ -181,6 +181,21 @@::qp.add-remaps/new-field-dimension-id pos-int?}]]}})(lib/->legacy-MBQL query))))))+(deftest ^:parallel add-remapped-columns-with-previous-stage-test+ (let [query (-> (lib/query category-id-remap-metadata-provider (meta/table-metadata :venues))+ (lib/append-stage)+ (lib/with-fields [(meta/field-metadata :venues :category-id)]))+ dimension-id (get-in (lib.metadata/field category-id-remap-metadata-provider (meta/id :venues :category-id))+ [:lib/external-remap :id])]+ ;; The `lib/append-stage` models the sandbox behaviour where an extra stage is added+ ;; and the `remap-column-infos` become name based instead of id based+ (is (=? {:stages [{}+ {:fields [[:field {::qp.add-remaps/original-field-dimension-id dimension-id}+ (meta/id :venues :category-id)]+ [:field {::qp.add-remaps/new-field-dimension-id dimension-id}+ (meta/id :categories :name)]]}]}+ (qp.add-remaps/add-remapped-columns query)))))+;;; ---------------------------------------- remap-results (post-processing) -----------------------------------------(defn- remap-results [query metadata rows]diff --git a/test/metabase/query_processor/middleware/fix_bad_field_id_refs_test.clj b/test/metabase/query_processor/middleware/fix_bad_field_id_refs_test.cljindex 91e36dd6356..026f9d4a074 100644--- a/test/metabase/query_processor/middleware/fix_bad_field_id_refs_test.clj+++ b/test/metabase/query_processor/middleware/fix_bad_field_id_refs_test.clj@@ -7,6 +7,7 @@[metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros]+ [metabase.query-processor.middleware.add-remaps :as qp.add-remaps][metabase.query-processor.middleware.fix-bad-field-id-refs :as fix-bad-field-id-refs][metabase.query-processor.test :as qp][metabase.test :as mt]@@ -316,6 +317,31 @@(is (every? string? (keys exprs))(str "Assert failed: :expressions should always use string keys, got: " (pr-str exprs))))))))+(deftest ^:parallel preserve-remapping-keys-when-fixing-field-id-ref-test+ (testing "rewriting bad ID refs preserves the remapping dimension keys added by add-remaps (#78187)"+ ;; a legacy card-source query with ID refs is the same shape sandboxing produces: the stage has no+ ;; :source-table, so both the remapped-from and remapped-to refs hit the rewrite branch here+ (let [mp (-> meta/metadata-provider+ (lib.tu/metadata-provider-with-cards-for-queries+ [(lib/query meta/metadata-provider (meta/table-metadata :venues))])+ (lib.tu/remap-metadata-provider+ (meta/field-metadata :venues :category-id)+ (meta/field-metadata :categories :name)))+ query (lib/query mp {:database (meta/id)+ :type :query+ :query {:source-table "card__1"+ :fields [[:field (meta/id :venues :category-id) nil]]}})+ dimension-id (get-in (lib.metadata/field mp (meta/id :venues :category-id))+ [:lib/external-remap :id])]+ (is (=? {:stages [{:fields [[:field {::qp.add-remaps/original-field-dimension-id dimension-id}+ "CATEGORY_ID"]+ [:field {:source-field (meta/id :venues :category-id)+ ::qp.add-remaps/new-field-dimension-id dimension-id}+ (meta/id :categories :name)]]}]}+ (-> query+ qp.add-remaps/add-remapped-columns+ fix-bad-field-id-refs/fix-bad-field-id-refs))))))+(deftest ^:parallel resolve-join-conditions-test(testing "Resolve fields in join :conditions"(let [mp (lib.tu/mock-metadata-provider