The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Return nested fields from sandboxed queries (#75358)
metambenf02742f0a9d4
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.cljindex ac6b800d71b..49a22fa24d5 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj@@ -12,6 +12,7 @@^{:clj-kondo/ignore [:discouraged-namespace]}[metabase.legacy-mbql.schema :as mbql.s][metabase.lib.core :as lib]+ [metabase.lib.field.util :as lib.field.util][metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.result-metadata :as lib.metadata.result-metadata][metabase.lib.schema :as lib.schema]@@ -239,38 +240,50 @@We probably should have had this error from day 1 but now there are sandboxes in the wild that do this so I guess wejust have to work around them going forward."[metadata-providerable sandbox-query original-table-id]- (let [sandbox-cols (lib/returned-columns sandbox-query)- table-cols (lib.metadata/fields metadata-providerable original-table-id)- fixed-cols (remove (fn [{:keys [table-id], :as sandbox-col}]- (b/cond- (and table-id- (not= table-id original-table-id))- (do- (log/errorf (str "Sandboxes can only include columns from the original Table (%d),"- " query included %s from Table %d. This is unsupported and may not"- " work in the future.")- original-table-id- (pr-str (:name sandbox-col))- table-id)- true)-- :let [matching-table-col (m/find-first #(= (:name %)- (:name sandbox-col))- table-cols)]- (not matching-table-col)- (do- (log/errorf (str "Sandboxes can only include columns from the original Table,"- " but query included %s. This is unsupported and may not work in"- " the future.")- (pr-str (:name sandbox-col)))- true)-- :else- (do- ;; this will throw an exception if types don't match up- (sandbox/check-column-types-match sandbox-col matching-table-col)- false)))- sandbox-cols)]+ (let [sandbox-cols (lib/returned-columns sandbox-query)+ table-cols (lib.metadata/fields metadata-providerable original-table-id)+ ;; `dissoc nil` guards against the rare case of a table-col without `:id` shadowing the name lookup for+ ;; sandbox cols (e.g. native) that also have no `:id`.+ table-by-id (dissoc (m/index-by :id table-cols) nil)+ table-by-name (m/index-by :name table-cols)+ ;; For nested fields (e.g. Mongo objects), the sandbox col's `:name` comes from+ ;; `add-parent-column-metadata` as `parent.child.leaf`, while `table-cols` carry the raw leaf `:name`.+ ;; Index table-cols by their parent-qualified name so name-only matching (native sandboxes, no `:id`)+ ;; still finds nested fields. See #75305.+ table-by-flat-name (m/index-by #(lib.field.util/parent-qualified-name metadata-providerable %)+ table-cols)+ find-table-col (fn [{col-id :id, col-name :name}]+ (or (get table-by-id col-id)+ (get table-by-name col-name)+ (get table-by-flat-name col-name)))+ fixed-cols (remove (fn [{:keys [table-id], :as sandbox-col}]+ (b/cond+ (and table-id+ (not= table-id original-table-id))+ (do+ (log/errorf (str "Sandboxes can only include columns from the original Table (%d),"+ " query included %s from Table %d. This is unsupported and may not"+ " work in the future.")+ original-table-id+ (pr-str (:name sandbox-col))+ table-id)+ true)++ :let [matching-table-col (find-table-col sandbox-col)]+ (not matching-table-col)+ (do+ (log/errorf (str "Sandboxes can only include columns from the original Table,"+ " but query included %s. This is unsupported and may not work in"+ " the future.")+ (pr-str (:name sandbox-col)))+ true)++ :else+ (do+ ;; this will throw an exception if types don't match up+ (sandbox/check-column-types-match sandbox-col matching-table-col)+ false)))+ sandbox-cols)](if (= fixed-cols sandbox-cols)sandbox-query(dodiff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex e9f1355ef97..bde5aa6e1d2 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -1751,6 +1751,53 @@(is (= :completed (:status result)))(is (= 3 (count (mt/rows result))))))))))+(deftest ^:parallel project-only-columns-from-original-table-preserves-nested-fields-test+ (testing "Sandbox column projection must keep nested fields whose `:name` is path-joined after preprocessing (#75305)"+ ;; For nested fields (e.g. Mongo object children), `lib/returned-columns` on a preprocessed sandbox query produces+ ;; `:name` like `"parent.child.leaf"` while `lib.metadata/fields` returns the raw leaf `:name`. Matching only by+ ;; `:name` would drop nested columns and wrap the sandbox in an extra stage that omits them — making the columns+ ;; vanish for sandboxed users.+ ;;+ ;; To guard against a fix that simply disables the filter, the sandbox query below also joins in a column from+ ;; another table (`orders.total`) — that one *must* still be filtered out.+ (let [mp (lib.tu/mock-metadata-provider+ {:database {:id 1 :name "db" :dialect :h2 :engine :h2}+ :tables [{:id 100 :name "customers" :db-id 1}+ {:id 200 :name "orders" :db-id 1}]+ :fields [{:id 1001 :name "id" :base-type :type/Integer :table-id 100+ :semantic-type :type/PK}+ {:id 1002 :name "name" :base-type :type/Text :table-id 100}+ {:id 1100 :name "tier_and_details" :base-type :type/Dictionary :table-id 100}+ {:id 1200 :name "u1" :base-type :type/Dictionary :table-id 100+ :nfc-path ["tier_and_details"] :parent-id 1100}+ {:id 1201 :name "id" :base-type :type/Text :table-id 100+ :nfc-path ["tier_and_details" "u1"] :parent-id 1200}+ {:id 1202 :name "active" :base-type :type/Boolean :table-id 100+ :nfc-path ["tier_and_details" "u1"] :parent-id 1200}+ {:id 2002 :name "customer_id" :base-type :type/Integer :table-id 200+ :fk-target-field-id 1001 :semantic-type :type/FK}+ {:id 2003 :name "total" :base-type :type/Float :table-id 200}]})+ base (lib/query mp (lib.metadata/table mp 100))+ join (-> (lib/join-clause (lib.metadata/table mp 200)+ [(lib/= (lib.metadata/field mp 1001)+ (lib.metadata/field mp 2002))])+ (lib/with-join-fields [(lib.metadata/field mp 2003)]))+ ;; This append-stage / preprocess / drop-stage dance matches what `sandboxing/preprocess-query` does+ ;; internally: an extra trailing stage carries `:lib/stage-metadata` through preprocessing (legacy MBQL+ ;; round-tripping otherwise loses it), and we discard that stage afterward to get back the original shape+ ;; with metadata preserved.+ sandbox-query (-> (lib/join base join)+ lib/append-stage+ qp.preprocess/preprocess+ lib/drop-stage)]+ (testing "preprocessing produces path-joined `:name`s for nested fields (sanity check)"+ (is (contains? (set (map :name (lib/returned-columns sandbox-query)))+ "tier_and_details.u1.id")))+ (testing "nested customers fields are preserved; the joined orders.total column is still filtered out"+ (let [result (#'sandboxing/project-only-columns-from-original-table mp sandbox-query 100)]+ (is (= #{1001 1002 1100 1200 1201 1202}+ (set (map :id (lib/returned-columns result))))))))))+(deftest sandboxing-throws-on-ee-without-token(mt/test-drivers (e2e-test-drivers)(testing "Basic test around querying a table by a user with segmented only permissions and a GTAP question that is a native query"diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.cljindex 0335b008280..c751587c7b6 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj@@ -172,6 +172,13 @@:else[field-name]))+(defn- raw-path->components+ "Split a `parent.child.leaf`-style Mongo path string into a vector of path components. The Mongo driver+ treats `.` as the unambiguous nested-key separator everywhere (sync, projection, ordering); literal dots in+ document field names aren't supported."+ [^String path]+ (str/split path #"\."))+(mu/defn field->name"Return a single string name for column metadata `col` For nested fields, this creates a combined qualified name."([col]@@ -1529,12 +1536,22 @@ function(bin) {(vec (col->name-components (driver-api/field (driver-api/metadata-provider) field-id))))(defn- field-clauses->id->path- "Build a map of `field-id -> path-vector` for all `:field` clauses in `fields` that reference an integer ID."+ "Build a map of `field-id-or-name -> path-vector` for all `:field` clauses in `fields`. Integer IDs are+ resolved via the metadata provider; for string refs (e.g. from a wrapper stage), the path is derived from+ the opts `:source-alias` populated by `add-alias-info` (and path-prepended by [[HACK-update-aliases]] for+ nested fields), falling back to `id-or-name` when no source-alias is present. The path-joined string is+ split on the Mongo path delimiter."[fields](into {}- (keep (fn [[agg-type field-id & _]]- (when (and (= agg-type :field) (integer? field-id))- [field-id (field-id->path field-id)])))+ (keep (fn [[agg-type id-or-name opts]]+ (when (= agg-type :field)+ (cond+ (integer? id-or-name)+ [id-or-name (field-id->path id-or-name)]++ (string? id-or-name)+ [id-or-name (raw-path->components+ (get opts driver-api/qp.add.source-alias id-or-name))]))))fields))(defn- remove-parent-fields@@ -1550,9 +1567,9 @@ function(bin) {(when (> (count path) 1)(vec (butlast path)))))(vals id->path))]- (remove (fn [[_ field-id & _]]- (and (integer? field-id)- (contains? parent-paths (id->path field-id))))+ (remove (fn [[agg-type id-or-name & _]]+ (and (= agg-type :field)+ (contains? parent-paths (id->path id-or-name))))fields)))(defn- remove-child-fields@@ -1565,10 +1582,11 @@ function(bin) {[fields](let [id->path (field-clauses->id->path fields)all-paths (set (vals id->path))]- (remove (fn [[agg-type field-id]]- (when (and (= agg-type :field) (integer? field-id))- (let [path (id->path field-id)]- (and (> (count path) 1)+ (remove (fn [[agg-type id-or-name]]+ (when (= agg-type :field)+ (let [path (id->path id-or-name)]+ (and path+ (> (count path) 1)(contains? all-paths (vec (butlast path)))))))fields)))diff --git a/modules/drivers/mongo/test/metabase/driver/mongo_sandboxing_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo_sandboxing_test.cljnew file mode 100644index 00000000000..0c677f7e720--- /dev/null+++ b/modules/drivers/mongo/test/metabase/driver/mongo_sandboxing_test.clj@@ -0,0 +1,24 @@+(ns ^:mb/driver-tests metabase.driver.mongo-sandboxing-test+ "End-to-end sandboxing tests for the Mongo driver, focused on scenarios that involve nested object fields."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.test :as met]+ [metabase.driver.mongo-test :as mongo-test]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.test :as mt]))++(deftest sandbox-preserves-nested-object-columns-test+ (testing "MBQL sandbox on a Mongo table with nested object fields does not drop the nested columns (#75305)"+ (mt/test-driver :mongo+ (mt/dataset mongo-test/nested-bindata-coll+ (met/with-gtaps! {:gtaps {:nested-bindata {:remappings {:user-int ["variable" [:field (mt/id :nested-bindata :int) nil]]}}}+ :attributes {"user-int" 1}}+ (let [mp (mt/metadata-provider)+ query (lib/query mp (lib.metadata/table mp (mt/id :nested-bindata)))+ result (mt/user-http-request :rasta :post 202 "dataset" query)+ col-names (->> result :data :cols (map :name) set)]+ (is (=? {:status "completed" :row_count 1} result))+ (testing "nested-field columns are present in the sandboxed result"+ (is (contains? col-names "nested_mixed_uuid.nested_data"))+ (is (contains? col-names "nested_mixed_not_uuid.nested_data_2")))))))))diff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljcindex 9d13b0e38f5..55f12f83ba3 100644--- a/src/metabase/lib/field/resolution.cljc+++ b/src/metabase/lib/field/resolution.cljc@@ -46,20 +46,15 @@[metadata-providerable :- ::lib.schema.metadata/metadata-providerable{:keys [parent-id], :as metadata} :- ::lib.schema.metadata/column](if-some [parent-metadata (when parent-id (lib.metadata/field metadata-providerable parent-id))]- (let [{parent-name :name- parent-nfc-path :nfc-path+ (let [{parent-nfc-path :nfc-pathparent-display-name :display-name} (add-parent-column-metadata metadata-providerable parent-metadata)- new-name (str parent-name- \.- ((some-fn :lib/original-name :name) metadata))- new-display-name (str parent-display-name- ": "- ((some-fn :lib/original-display-name :display-name)- metadata))]+ new-display-name (str parent-display-name+ ": "+ ((some-fn :lib/original-display-name :display-name) metadata))](-> metadata- (assoc :name new-name- :nfc-path (conj (vec parent-nfc-path) (:name parent-metadata))- :display-name new-display-name+ (assoc :name (lib.field.util/parent-qualified-name metadata-providerable metadata)+ :nfc-path (conj (vec parent-nfc-path) (:name parent-metadata))+ :display-name new-display-name;; this is used by the `display-name-method` for `:metadata/column` in [[metabase.lib.field]]:lib/simple-display-name new-display-name)))metadata))diff --git a/src/metabase/lib/field/util.cljc b/src/metabase/lib/field/util.cljcindex 4707ba11645..193d772c36e 100644--- a/src/metabase/lib/field/util.cljc+++ b/src/metabase/lib/field/util.cljc@@ -3,6 +3,7 @@(:require[clojure.set :as set][metabase.lib.join.util :as lib.join.util]+ [metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.ref :as lib.ref][metabase.lib.schema :as lib.schema]@@ -11,6 +12,22 @@[metabase.lib.util.unique-name-generator :as lib.util.unique-name-generator][metabase.util.malli :as mu]))+(mu/defn parent-qualified-name :- :string+ "Returns the `parent.child.leaf` `:name` for a column, walking up `:parent-id` via `metadata-providerable`.+ For top-level fields, returns the column's own name unchanged.++ This is the canonical column-name shape that drivers must produce in query results for nested fields (Mongo+ flattens nested keys this way; sql-jdbc nested-JSON unfolding produces the same shape). Both sides — the lib+ metadata layer (see [[metabase.lib.field.resolution/add-parent-column-metadata]]) and the drivers — must+ agree on this name so the QP can match columns across stages. If you ever change the join convention, update+ both sites."+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ {:keys [parent-id], :as column} :- ::lib.schema.metadata/column]+ (let [own-name ((some-fn :lib/original-name :name) column)]+ (if-some [parent (when parent-id (lib.metadata/field metadata-providerable parent-id))]+ (str (parent-qualified-name metadata-providerable parent) \. own-name)+ own-name)))+(mu/defn inherited-column? :- :boolean"Is the `column` coming directly from a card, a native query, or a previous query stage?"[column :- [:mapdiff --git a/test/metabase/test/data/impl.clj b/test/metabase/test/data/impl.cljindex c081262d8ff..c83e72a298a 100644--- a/test/metabase/test/data/impl.clj+++ b/test/metabase/test/data/impl.clj@@ -252,26 +252,45 @@;;; +----------------------------------------------------------------------------------------------------------------+(defn- copy-table-fields! [old-table-id new-table-id]- (t2/insert! :model/Field- (for [field (t2/select :model/Field :table_id old-table-id, :active true, {:order-by [[:id :asc]]})]- (-> field- (dissoc :id :fk_target_field_id)- (assoc :table_id new-table-id))))- ;; now copy the FieldValues as well.- (let [old-field-id->name (t2/select-pk->fn :name :model/Field :table_id old-table-id :active true)- new-field-name->id (t2/select-fn->pk :name :model/Field :table_id new-table-id :active true)- old-field-values (when-let [field-ids (seq (keys old-field-id->name))]- (t2/select :model/FieldValues :field_id [:in (set field-ids)]))]- (t2/insert! :model/FieldValues- (for [{old-field-id :field_id, :as field-values} old-field-values- :let [field-name (get old-field-id->name old-field-id)]]- (-> field-values- (dissoc :id)- (assoc :field_id (get new-field-name->id field-name))- ;; Toucan after-select for FieldValues returns NULL human_readable_values as [] for FE-friendliness..- ;; preserve NULL in the app DB copy so we don't end up changing things that rely on checking whether its- ;; NULL like [[metabase.parameters.chain-filter/search-cached-field-values?]]- (update :human_readable_values not-empty))))))+ (let [old-fields (t2/select :model/Field :table_id old-table-id, :active true, {:order-by [[:id :asc]]})]+ (t2/insert! :model/Field+ (for [field old-fields]+ (-> field+ (dissoc :id :fk_target_field_id)+ (assoc :table_id new-table-id))))+ ;; Remap `:parent_id` on copied nested fields. The insert above leaves them pointing to the *old* parent+ ;; field ids; nested fields would otherwise reference fields in the original DB, which breaks anything that+ ;; walks the parent chain (e.g. resolve-fields → bulk-metadata-or-throw).+ ;; (name, nfc_path) uniquely identifies a field within a table, so we match across the old/new copies on that.+ (let [field-key (juxt :name :nfc_path)+ new-fields (t2/select :model/Field :table_id new-table-id, :active true)+ new-key->id (into {} (map (juxt field-key :id)) new-fields)+ old-id->new-id (into {}+ (keep (fn [{old-id :id, :as old-field}]+ (when-let [new-id (get new-key->id (field-key old-field))]+ [old-id new-id])))+ old-fields)]+ (doseq [{new-id :id, old-parent-id :parent_id} new-fields+ :when (some? old-parent-id)+ :let [new-parent-id (get old-id->new-id old-parent-id)]+ :when (and new-parent-id+ (not= new-parent-id old-parent-id))]+ (t2/update! :model/Field new-id {:parent_id new-parent-id})))+ ;; now copy the FieldValues as well.+ (let [old-field-id->name (t2/select-pk->fn :name :model/Field :table_id old-table-id :active true)+ new-field-name->id (t2/select-fn->pk :name :model/Field :table_id new-table-id :active true)+ old-field-values (when-let [field-ids (seq (keys old-field-id->name))]+ (t2/select :model/FieldValues :field_id [:in (set field-ids)]))]+ (t2/insert! :model/FieldValues+ (for [{old-field-id :field_id, :as field-values} old-field-values+ :let [field-name (get old-field-id->name old-field-id)]]+ (-> field-values+ (dissoc :id)+ (assoc :field_id (get new-field-name->id field-name))+ ;; Toucan after-select for FieldValues returns NULL human_readable_values as [] for FE-friendliness..+ ;; preserve NULL in the app DB copy so we don't end up changing things that rely on checking whether its+ ;; NULL like [[metabase.parameters.chain-filter/search-cached-field-values?]]+ (update :human_readable_values not-empty)))))))(defn- copy-db-tables! [old-db-id new-db-id](let [old-tables (t2/select :model/Table :db_id old-db-id, :active true, {:order-by [[:id :asc]]})