Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Return nested fields from sandboxed queries (#75358)

metambenf02742f0a9d4
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj
index ac6b800d71b..49a22fa24d5 100644
--- a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj
+++ b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj
@@ -12,6 +12,7 @@
^{:clj-kondo/ignore [:discouraged-namespace]}
[metabase.legacy-mbql.schema :as mbql.s]
[metabase.lib.core :as lib]
+ [metabase.lib.field.util :as lib.field.util]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.result-metadata :as lib.metadata.result-metadata]
[metabase.lib.schema :as lib.schema]
@@ -239,38 +240,50 @@
We probably should have had this error from day 1 but now there are sandboxes in the wild that do this so I guess we
just have to work around them going forward."
[metadata-providerable sandbox-query original-table-id]
- (let [sandbox-cols (lib/returned-columns sandbox-query)
- table-cols (lib.metadata/fields metadata-providerable original-table-id)
- fixed-cols (remove (fn [{:keys [table-id], :as sandbox-col}]
- (b/cond
- (and table-id
- (not= table-id original-table-id))
- (do
- (log/errorf (str "Sandboxes can only include columns from the original Table (%d),"
- " query included %s from Table %d. This is unsupported and may not"
- " work in the future.")
- original-table-id
- (pr-str (:name sandbox-col))
- table-id)
- true)
-
- :let [matching-table-col (m/find-first #(= (:name %)
- (:name sandbox-col))
- table-cols)]
- (not matching-table-col)
- (do
- (log/errorf (str "Sandboxes can only include columns from the original Table,"
- " but query included %s. This is unsupported and may not work in"
- " the future.")
- (pr-str (:name sandbox-col)))
- true)
-
- :else
- (do
- ;; this will throw an exception if types don't match up
- (sandbox/check-column-types-match sandbox-col matching-table-col)
- false)))
- sandbox-cols)]
+ (let [sandbox-cols (lib/returned-columns sandbox-query)
+ table-cols (lib.metadata/fields metadata-providerable original-table-id)
+ ;; `dissoc nil` guards against the rare case of a table-col without `:id` shadowing the name lookup for
+ ;; sandbox cols (e.g. native) that also have no `:id`.
+ table-by-id (dissoc (m/index-by :id table-cols) nil)
+ table-by-name (m/index-by :name table-cols)
+ ;; For nested fields (e.g. Mongo objects), the sandbox col's `:name` comes from
+ ;; `add-parent-column-metadata` as `parent.child.leaf`, while `table-cols` carry the raw leaf `:name`.
+ ;; Index table-cols by their parent-qualified name so name-only matching (native sandboxes, no `:id`)
+ ;; still finds nested fields. See #75305.
+ table-by-flat-name (m/index-by #(lib.field.util/parent-qualified-name metadata-providerable %)
+ table-cols)
+ find-table-col (fn [{col-id :id, col-name :name}]
+ (or (get table-by-id col-id)
+ (get table-by-name col-name)
+ (get table-by-flat-name col-name)))
+ fixed-cols (remove (fn [{:keys [table-id], :as sandbox-col}]
+ (b/cond
+ (and table-id
+ (not= table-id original-table-id))
+ (do
+ (log/errorf (str "Sandboxes can only include columns from the original Table (%d),"
+ " query included %s from Table %d. This is unsupported and may not"
+ " work in the future.")
+ original-table-id
+ (pr-str (:name sandbox-col))
+ table-id)
+ true)
+
+ :let [matching-table-col (find-table-col sandbox-col)]
+ (not matching-table-col)
+ (do
+ (log/errorf (str "Sandboxes can only include columns from the original Table,"
+ " but query included %s. This is unsupported and may not work in"
+ " the future.")
+ (pr-str (:name sandbox-col)))
+ true)
+
+ :else
+ (do
+ ;; this will throw an exception if types don't match up
+ (sandbox/check-column-types-match sandbox-col matching-table-col)
+ false)))
+ sandbox-cols)]
(if (= fixed-cols sandbox-cols)
sandbox-query
(do
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
index e9f1355ef97..bde5aa6e1d2 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
@@ -1751,6 +1751,53 @@
(is (= :completed (:status result)))
(is (= 3 (count (mt/rows result))))))))))
+(deftest ^:parallel project-only-columns-from-original-table-preserves-nested-fields-test
+ (testing "Sandbox column projection must keep nested fields whose `:name` is path-joined after preprocessing (#75305)"
+ ;; For nested fields (e.g. Mongo object children), `lib/returned-columns` on a preprocessed sandbox query produces
+ ;; `:name` like `"parent.child.leaf"` while `lib.metadata/fields` returns the raw leaf `:name`. Matching only by
+ ;; `:name` would drop nested columns and wrap the sandbox in an extra stage that omits them — making the columns
+ ;; vanish for sandboxed users.
+ ;;
+ ;; To guard against a fix that simply disables the filter, the sandbox query below also joins in a column from
+ ;; another table (`orders.total`) — that one *must* still be filtered out.
+ (let [mp (lib.tu/mock-metadata-provider
+ {:database {:id 1 :name "db" :dialect :h2 :engine :h2}
+ :tables [{:id 100 :name "customers" :db-id 1}
+ {:id 200 :name "orders" :db-id 1}]
+ :fields [{:id 1001 :name "id" :base-type :type/Integer :table-id 100
+ :semantic-type :type/PK}
+ {:id 1002 :name "name" :base-type :type/Text :table-id 100}
+ {:id 1100 :name "tier_and_details" :base-type :type/Dictionary :table-id 100}
+ {:id 1200 :name "u1" :base-type :type/Dictionary :table-id 100
+ :nfc-path ["tier_and_details"] :parent-id 1100}
+ {:id 1201 :name "id" :base-type :type/Text :table-id 100
+ :nfc-path ["tier_and_details" "u1"] :parent-id 1200}
+ {:id 1202 :name "active" :base-type :type/Boolean :table-id 100
+ :nfc-path ["tier_and_details" "u1"] :parent-id 1200}
+ {:id 2002 :name "customer_id" :base-type :type/Integer :table-id 200
+ :fk-target-field-id 1001 :semantic-type :type/FK}
+ {:id 2003 :name "total" :base-type :type/Float :table-id 200}]})
+ base (lib/query mp (lib.metadata/table mp 100))
+ join (-> (lib/join-clause (lib.metadata/table mp 200)
+ [(lib/= (lib.metadata/field mp 1001)
+ (lib.metadata/field mp 2002))])
+ (lib/with-join-fields [(lib.metadata/field mp 2003)]))
+ ;; This append-stage / preprocess / drop-stage dance matches what `sandboxing/preprocess-query` does
+ ;; internally: an extra trailing stage carries `:lib/stage-metadata` through preprocessing (legacy MBQL
+ ;; round-tripping otherwise loses it), and we discard that stage afterward to get back the original shape
+ ;; with metadata preserved.
+ sandbox-query (-> (lib/join base join)
+ lib/append-stage
+ qp.preprocess/preprocess
+ lib/drop-stage)]
+ (testing "preprocessing produces path-joined `:name`s for nested fields (sanity check)"
+ (is (contains? (set (map :name (lib/returned-columns sandbox-query)))
+ "tier_and_details.u1.id")))
+ (testing "nested customers fields are preserved; the joined orders.total column is still filtered out"
+ (let [result (#'sandboxing/project-only-columns-from-original-table mp sandbox-query 100)]
+ (is (= #{1001 1002 1100 1200 1201 1202}
+ (set (map :id (lib/returned-columns result))))))))))
+
(deftest sandboxing-throws-on-ee-without-token
(mt/test-drivers (e2e-test-drivers)
(testing "Basic test around querying a table by a user with segmented only permissions and a GTAP question that is a native query"
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
index 0335b008280..c751587c7b6 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
@@ -172,6 +172,13 @@
:else
[field-name]))
+(defn- raw-path->components
+ "Split a `parent.child.leaf`-style Mongo path string into a vector of path components. The Mongo driver
+ treats `.` as the unambiguous nested-key separator everywhere (sync, projection, ordering); literal dots in
+ document field names aren't supported."
+ [^String path]
+ (str/split path #"\."))
+
(mu/defn field->name
"Return a single string name for column metadata `col` For nested fields, this creates a combined qualified name."
([col]
@@ -1529,12 +1536,22 @@ function(bin) {
(vec (col->name-components (driver-api/field (driver-api/metadata-provider) field-id))))
(defn- field-clauses->id->path
- "Build a map of `field-id -> path-vector` for all `:field` clauses in `fields` that reference an integer ID."
+ "Build a map of `field-id-or-name -> path-vector` for all `:field` clauses in `fields`. Integer IDs are
+ resolved via the metadata provider; for string refs (e.g. from a wrapper stage), the path is derived from
+ the opts `:source-alias` populated by `add-alias-info` (and path-prepended by [[HACK-update-aliases]] for
+ nested fields), falling back to `id-or-name` when no source-alias is present. The path-joined string is
+ split on the Mongo path delimiter."
[fields]
(into {}
- (keep (fn [[agg-type field-id & _]]
- (when (and (= agg-type :field) (integer? field-id))
- [field-id (field-id->path field-id)])))
+ (keep (fn [[agg-type id-or-name opts]]
+ (when (= agg-type :field)
+ (cond
+ (integer? id-or-name)
+ [id-or-name (field-id->path id-or-name)]
+
+ (string? id-or-name)
+ [id-or-name (raw-path->components
+ (get opts driver-api/qp.add.source-alias id-or-name))]))))
fields))
(defn- remove-parent-fields
@@ -1550,9 +1567,9 @@ function(bin) {
(when (> (count path) 1)
(vec (butlast path)))))
(vals id->path))]
- (remove (fn [[_ field-id & _]]
- (and (integer? field-id)
- (contains? parent-paths (id->path field-id))))
+ (remove (fn [[agg-type id-or-name & _]]
+ (and (= agg-type :field)
+ (contains? parent-paths (id->path id-or-name))))
fields)))
(defn- remove-child-fields
@@ -1565,10 +1582,11 @@ function(bin) {
[fields]
(let [id->path (field-clauses->id->path fields)
all-paths (set (vals id->path))]
- (remove (fn [[agg-type field-id]]
- (when (and (= agg-type :field) (integer? field-id))
- (let [path (id->path field-id)]
- (and (> (count path) 1)
+ (remove (fn [[agg-type id-or-name]]
+ (when (= agg-type :field)
+ (let [path (id->path id-or-name)]
+ (and path
+ (> (count path) 1)
(contains? all-paths (vec (butlast path)))))))
fields)))
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo_sandboxing_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo_sandboxing_test.clj
new file mode 100644
index 00000000000..0c677f7e720
--- /dev/null
+++ b/modules/drivers/mongo/test/metabase/driver/mongo_sandboxing_test.clj
@@ -0,0 +1,24 @@
+(ns ^:mb/driver-tests metabase.driver.mongo-sandboxing-test
+ "End-to-end sandboxing tests for the Mongo driver, focused on scenarios that involve nested object fields."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.test :as met]
+ [metabase.driver.mongo-test :as mongo-test]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.test :as mt]))
+
+(deftest sandbox-preserves-nested-object-columns-test
+ (testing "MBQL sandbox on a Mongo table with nested object fields does not drop the nested columns (#75305)"
+ (mt/test-driver :mongo
+ (mt/dataset mongo-test/nested-bindata-coll
+ (met/with-gtaps! {:gtaps {:nested-bindata {:remappings {:user-int ["variable" [:field (mt/id :nested-bindata :int) nil]]}}}
+ :attributes {"user-int" 1}}
+ (let [mp (mt/metadata-provider)
+ query (lib/query mp (lib.metadata/table mp (mt/id :nested-bindata)))
+ result (mt/user-http-request :rasta :post 202 "dataset" query)
+ col-names (->> result :data :cols (map :name) set)]
+ (is (=? {:status "completed" :row_count 1} result))
+ (testing "nested-field columns are present in the sandboxed result"
+ (is (contains? col-names "nested_mixed_uuid.nested_data"))
+ (is (contains? col-names "nested_mixed_not_uuid.nested_data_2")))))))))
diff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljc
index 9d13b0e38f5..55f12f83ba3 100644
--- a/src/metabase/lib/field/resolution.cljc
+++ b/src/metabase/lib/field/resolution.cljc
@@ -46,20 +46,15 @@
[metadata-providerable :- ::lib.schema.metadata/metadata-providerable
{:keys [parent-id], :as metadata} :- ::lib.schema.metadata/column]
(if-some [parent-metadata (when parent-id (lib.metadata/field metadata-providerable parent-id))]
- (let [{parent-name :name
- parent-nfc-path :nfc-path
+ (let [{parent-nfc-path :nfc-path
parent-display-name :display-name} (add-parent-column-metadata metadata-providerable parent-metadata)
- new-name (str parent-name
- \.
- ((some-fn :lib/original-name :name) metadata))
- new-display-name (str parent-display-name
- ": "
- ((some-fn :lib/original-display-name :display-name)
- metadata))]
+ new-display-name (str parent-display-name
+ ": "
+ ((some-fn :lib/original-display-name :display-name) metadata))]
(-> metadata
- (assoc :name new-name
- :nfc-path (conj (vec parent-nfc-path) (:name parent-metadata))
- :display-name new-display-name
+ (assoc :name (lib.field.util/parent-qualified-name metadata-providerable metadata)
+ :nfc-path (conj (vec parent-nfc-path) (:name parent-metadata))
+ :display-name new-display-name
;; this is used by the `display-name-method` for `:metadata/column` in [[metabase.lib.field]]
:lib/simple-display-name new-display-name)))
metadata))
diff --git a/src/metabase/lib/field/util.cljc b/src/metabase/lib/field/util.cljc
index 4707ba11645..193d772c36e 100644
--- a/src/metabase/lib/field/util.cljc
+++ b/src/metabase/lib/field/util.cljc
@@ -3,6 +3,7 @@
(:require
[clojure.set :as set]
[metabase.lib.join.util :as lib.join.util]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
[metabase.lib.ref :as lib.ref]
[metabase.lib.schema :as lib.schema]
@@ -11,6 +12,22 @@
[metabase.lib.util.unique-name-generator :as lib.util.unique-name-generator]
[metabase.util.malli :as mu]))
+(mu/defn parent-qualified-name :- :string
+ "Returns the `parent.child.leaf` `:name` for a column, walking up `:parent-id` via `metadata-providerable`.
+ For top-level fields, returns the column's own name unchanged.
+
+ This is the canonical column-name shape that drivers must produce in query results for nested fields (Mongo
+ flattens nested keys this way; sql-jdbc nested-JSON unfolding produces the same shape). Both sides — the lib
+ metadata layer (see [[metabase.lib.field.resolution/add-parent-column-metadata]]) and the drivers — must
+ agree on this name so the QP can match columns across stages. If you ever change the join convention, update
+ both sites."
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ {:keys [parent-id], :as column} :- ::lib.schema.metadata/column]
+ (let [own-name ((some-fn :lib/original-name :name) column)]
+ (if-some [parent (when parent-id (lib.metadata/field metadata-providerable parent-id))]
+ (str (parent-qualified-name metadata-providerable parent) \. own-name)
+ own-name)))
+
(mu/defn inherited-column? :- :boolean
"Is the `column` coming directly from a card, a native query, or a previous query stage?"
[column :- [:map
diff --git a/test/metabase/test/data/impl.clj b/test/metabase/test/data/impl.clj
index c081262d8ff..c83e72a298a 100644
--- a/test/metabase/test/data/impl.clj
+++ b/test/metabase/test/data/impl.clj
@@ -252,26 +252,45 @@
;;; +----------------------------------------------------------------------------------------------------------------+
(defn- copy-table-fields! [old-table-id new-table-id]
- (t2/insert! :model/Field
- (for [field (t2/select :model/Field :table_id old-table-id, :active true, {:order-by [[:id :asc]]})]
- (-> field
- (dissoc :id :fk_target_field_id)
- (assoc :table_id new-table-id))))
- ;; now copy the FieldValues as well.
- (let [old-field-id->name (t2/select-pk->fn :name :model/Field :table_id old-table-id :active true)
- new-field-name->id (t2/select-fn->pk :name :model/Field :table_id new-table-id :active true)
- old-field-values (when-let [field-ids (seq (keys old-field-id->name))]
- (t2/select :model/FieldValues :field_id [:in (set field-ids)]))]
- (t2/insert! :model/FieldValues
- (for [{old-field-id :field_id, :as field-values} old-field-values
- :let [field-name (get old-field-id->name old-field-id)]]
- (-> field-values
- (dissoc :id)
- (assoc :field_id (get new-field-name->id field-name))
- ;; Toucan after-select for FieldValues returns NULL human_readable_values as [] for FE-friendliness..
- ;; preserve NULL in the app DB copy so we don't end up changing things that rely on checking whether its
- ;; NULL like [[metabase.parameters.chain-filter/search-cached-field-values?]]
- (update :human_readable_values not-empty))))))
+ (let [old-fields (t2/select :model/Field :table_id old-table-id, :active true, {:order-by [[:id :asc]]})]
+ (t2/insert! :model/Field
+ (for [field old-fields]
+ (-> field
+ (dissoc :id :fk_target_field_id)
+ (assoc :table_id new-table-id))))
+ ;; Remap `:parent_id` on copied nested fields. The insert above leaves them pointing to the *old* parent
+ ;; field ids; nested fields would otherwise reference fields in the original DB, which breaks anything that
+ ;; walks the parent chain (e.g. resolve-fields → bulk-metadata-or-throw).
+ ;; (name, nfc_path) uniquely identifies a field within a table, so we match across the old/new copies on that.
+ (let [field-key (juxt :name :nfc_path)
+ new-fields (t2/select :model/Field :table_id new-table-id, :active true)
+ new-key->id (into {} (map (juxt field-key :id)) new-fields)
+ old-id->new-id (into {}
+ (keep (fn [{old-id :id, :as old-field}]
+ (when-let [new-id (get new-key->id (field-key old-field))]
+ [old-id new-id])))
+ old-fields)]
+ (doseq [{new-id :id, old-parent-id :parent_id} new-fields
+ :when (some? old-parent-id)
+ :let [new-parent-id (get old-id->new-id old-parent-id)]
+ :when (and new-parent-id
+ (not= new-parent-id old-parent-id))]
+ (t2/update! :model/Field new-id {:parent_id new-parent-id})))
+ ;; now copy the FieldValues as well.
+ (let [old-field-id->name (t2/select-pk->fn :name :model/Field :table_id old-table-id :active true)
+ new-field-name->id (t2/select-fn->pk :name :model/Field :table_id new-table-id :active true)
+ old-field-values (when-let [field-ids (seq (keys old-field-id->name))]
+ (t2/select :model/FieldValues :field_id [:in (set field-ids)]))]
+ (t2/insert! :model/FieldValues
+ (for [{old-field-id :field_id, :as field-values} old-field-values
+ :let [field-name (get old-field-id->name old-field-id)]]
+ (-> field-values
+ (dissoc :id)
+ (assoc :field_id (get new-field-name->id field-name))
+ ;; Toucan after-select for FieldValues returns NULL human_readable_values as [] for FE-friendliness..
+ ;; preserve NULL in the app DB copy so we don't end up changing things that rely on checking whether its
+ ;; NULL like [[metabase.parameters.chain-filter/search-cached-field-values?]]
+ (update :human_readable_values not-empty)))))))
(defn- copy-db-tables! [old-db-id new-db-id]
(let [old-tables (t2/select :model/Table :db_id old-db-id, :active true, {:order-by [[:id :asc]]})