Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[QP] Correctly handle "struct" fields on Athena and similar drivers (#71917)

Braden Shepherdsoneb61194d47ab
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index 81a97c01716..5580de87561 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -890,6 +890,18 @@
[_driver [_ _nfc-path]]
nil)
+(defn- parent-id->ancestor-names
+ "Walk the parent chain for a field with `:parent-id` set and return the ancestor field names, \"oldest\" first.
+ For example, if field `c` has parent `b`, returns `[\"b\"]`.
+ If `c` has parent `b` which has parent `a`, returns `[\"a\" \"b\"]`."
+ [parent-id]
+ (loop [id parent-id
+ path ()]
+ (if-not id
+ (vec path)
+ (let [parent (driver-api/field (driver-api/metadata-provider) id)]
+ (recur (:parent-id parent) (cons (:name parent) path))))))
+
(defmethod ->honeysql [:sql :field]
[driver [_ id-or-name options :as field-clause]]
(try
@@ -902,6 +914,10 @@
;; https://linear.app/metabase-inc/issue/ENG-8766/[epic]-field-refs-overhaul
field-metadata (when (integer? id-or-name)
(driver-api/field (driver-api/metadata-provider) id-or-name))
+ ;; For fields with parent-id (struct-type nested fields), include parent field names
+ ;; in the identifier so e.g. `result.tag_name` is rendered instead of just `tag_name`.
+ parent-names (when-let [parent-id (:parent-id field-metadata)]
+ (parent-id->ancestor-names parent-id))
allow-casting? (and (or (:qp/native-sandbox-column.force-coercion-strategy options)
(and field-metadata
(or (pos-int? (driver-api/qp.add.source-table options))
@@ -909,7 +925,10 @@
(not (:qp/ignore-coercion options)))
;; preserve metadata attached to the original field clause, for example BigQuery temporal type information.
identifier (-> (apply h2x/identifier :field
- (concat source-table-aliases (->honeysql driver [::nfc-path source-nfc-path]) [source-alias]))
+ (concat source-table-aliases
+ (->honeysql driver [::nfc-path source-nfc-path])
+ parent-names
+ [source-alias]))
(with-meta (meta field-clause)))
identifier (->honeysql driver identifier)
;; If no field-metadata is available, but a coercion strategy must be applied, synthesize enough
diff --git a/test/metabase/driver/sql/parameters/substitute_test.clj b/test/metabase/driver/sql/parameters/substitute_test.clj
index 65046e9f6dd..1623fe4ebe5 100644
--- a/test/metabase/driver/sql/parameters/substitute_test.clj
+++ b/test/metabase/driver/sql/parameters/substitute_test.clj
@@ -7,15 +7,18 @@
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params]
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.parse :as params.parse]
[metabase.driver.sql.parameters.substitute :as sql.params.substitute]
+ [metabase.driver.sql.query-processor :as sql.qp]
[metabase.legacy-mbql.normalize :as mbql.normalize]
[metabase.lib.core :as lib]
[metabase.lib.test-metadata :as meta]
+ [metabase.lib.test-util :as lib.tu]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.middleware.parameters.native :as qp.native]
[metabase.query-processor.test :as qp]
[metabase.query-processor.test-util :as qp.test-util]
[metabase.test :as mt]
[metabase.test.data.interface :as tx]
+ [metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]))
(defn- optional [& args] (params/->Optional args))
@@ -26,6 +29,14 @@
(mt/with-metadata-provider meta/metadata-provider
(sql.params.substitute/substitute parsed param->value))))
+;; as with the MBQL parameters tests Redshift fail for unknown reasons; disable their tests for now
+;; TIMEZONE FIXME
+(defn- sql-parameters-engines []
+ (set (for [driver (mt/normal-drivers-with-feature :native-parameters)
+ :when (and (isa? driver/hierarchy driver :sql)
+ (not= driver :redshift))]
+ driver)))
+
(deftest ^:parallel substitute-test
(testing "normal substitution"
(is (= ["select * from foobars where bird_type = ?" ["Steller's Jay"]]
@@ -112,6 +123,53 @@
(is (= ["select * from orders" nil]
(substitute query {"created_at" (assoc (date-field-filter-value) :value params/no-value)}))))))))
+(deftest ^:parallel substitute-field-filter-for-nested-field-test
+ (testing "field filter for a nested field (with parent-id) should include parent field name in identifier (#47003)"
+ (mt/test-drivers (sql-parameters-engines)
+ (let [;; Use high IDs that won't collide with existing test metadata
+ parent-field-id 999901
+ nested-field-id 999902
+ ;; Create a metadata provider that has a parent struct field and a nested child field
+ metadata-provider
+ (lib.tu/merged-mock-metadata-provider
+ meta/metadata-provider
+ {:fields [{:id parent-field-id
+ :table-id (meta/id :venues)
+ :name "result"
+ :display-name "Result"
+ :base-type :type/Dictionary
+ :database-type "STRUCT"
+ :parent-id nil
+ :nfc-path nil}
+ {:id nested-field-id
+ :table-id (meta/id :venues)
+ :name "tag_name"
+ :display-name "Tag Name"
+ :base-type :type/Text
+ :database-type "CHARACTER VARYING"
+ :parent-id parent-field-id
+ :nfc-path nil}]})
+ query ["select * from venues where " (param "tag")]
+ field-metadata (assoc (meta/field-metadata :venues :name)
+ :id nested-field-id
+ :name "tag_name"
+ :display-name "Tag Name"
+ :base-type :type/Text
+ :database-type "CHARACTER VARYING"
+ :parent-id parent-field-id
+ :nfc-path nil)
+ field-filter (params/map->FieldFilter
+ {:field field-metadata
+ :value {:type :string/=
+ :value ["banana"]}})
+ [sql _args] (mt/with-metadata-provider metadata-provider
+ (sql.params.substitute/substitute query {"tag" field-filter}))]
+ (testing "The SQL identifier should include the parent field 'result' before 'tag_name'"
+ (let [[exp-identifier] (sql.qp/format-honeysql driver/*driver*
+ (h2x/identifier :field "result" "tag_name"))]
+ (is (str/includes? sql exp-identifier)
+ (str "Expected SQL to include parent field in identifier, got: " sql))))))))
+
(def ^:private substitute-field-filter-test-2-test-cases
(partition-all
2
@@ -902,14 +960,6 @@
`(let [sql# (:query (qp.compile/compile (mt/mbql-query ~table-name)))]
(second (re-find #"(?m)FROM\s+([^\s()]+)" sql#))))
-;; as with the MBQL parameters tests Redshift fail for unknown reasons; disable their tests for now
-;; TIMEZONE FIXME
-(defn- sql-parameters-engines []
- (set (for [driver (mt/normal-drivers-with-feature :native-parameters)
- :when (and (isa? driver/hierarchy driver :sql)
- (not= driver :redshift))]
- driver)))
-
(defn- process-native [& {:as query}]
(qp/process-query
(merge