Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Validate imported queries against this instance's MBQL schema (GHY-4241) (#79496)

Eric Normand6bd66a6d5eba
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index a89e54a4394..7e5027bd51e 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3724,7 +3724,10 @@
enterprise/serialization
{:team "Graphy"
- :api #{metabase-enterprise.serialization.api metabase-enterprise.serialization.cmd metabase-enterprise.serialization.core}
+ :api #{metabase-enterprise.serialization.api
+ metabase-enterprise.serialization.cmd
+ metabase-enterprise.serialization.core
+ metabase-enterprise.serialization.init}
:friends #{enterprise/memoize-monitor}
:uses #{analytics
api
@@ -3739,6 +3742,7 @@
premium-features
search
server
+ settings
setup
util}
:model-imports :bypass}
diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.clj
index 95b3a484fe7..eb2a57b7434 100644
--- a/enterprise/backend/src/metabase_enterprise/core/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj
@@ -26,6 +26,7 @@
[metabase-enterprise.scim.init]
[metabase-enterprise.security-center.init]
[metabase-enterprise.semantic-search.init]
+ [metabase-enterprise.serialization.init]
[metabase-enterprise.sso.init]
[metabase-enterprise.stale.init]
[metabase-enterprise.support-access-grants.init]
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/init.clj b/enterprise/backend/src/metabase_enterprise/serialization/init.clj
new file mode 100644
index 00000000000..a0fd6baaa0e
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/init.clj
@@ -0,0 +1,3 @@
+(ns metabase-enterprise.serialization.init
+ (:require
+ [metabase-enterprise.serialization.settings]))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/settings.clj b/enterprise/backend/src/metabase_enterprise/serialization/settings.clj
new file mode 100644
index 00000000000..37242b6ff6f
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/settings.clj
@@ -0,0 +1,15 @@
+(ns metabase-enterprise.serialization.settings
+ (:require
+ [metabase.settings.core :refer [defsetting]]))
+
+(defsetting serialization-skip-schema-validation
+ "Whether to import questions whose queries this Metabase's query schema rejects. Defaults to false."
+ :type :boolean
+ :default false
+ :visibility :internal
+ :setter :none
+ :audit :never
+ :export? false
+ :doc (str "On import, Metabase validates every question against the query format this version understands, and refuses "
+ "any it cannot read. Set this to true to skip that validation. "
+ "Skipping it will not necessarily make the import succeed. Import may still fail on a later step."))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
index 41c9dc98f2b..fcc9f84239f 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
@@ -4,6 +4,7 @@
(:require
[clojure.string :as str]
[diehard.core :as dh]
+ [metabase-enterprise.serialization.settings :as serialization.settings]
[metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
[metabase-enterprise.serialization.v2.models :as serdes.models]
[metabase.app-db.core :as mdb]
@@ -257,36 +258,37 @@
[ingestion & {:keys [continue-on-error reindex?]
:or {continue-on-error false
reindex? true}}]
- (u/prog1
- ;; Each entity is loaded in its own transaction (inside load-one!), so a deadlock or transient
- ;; failure on one entity doesn't abort the entire import. See #74412.
- ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared
- ;; dependencies guide the import, and make sure all containers are imported before contents, etc.
- (let [contents (serdes.ingest/ingest-list ingestion)
- ingest-errors (serdes.ingest/ingest-errors ingestion)
- ctx (cond-> (new-context ingestion)
- (seq ingest-errors) (update :errors into ingest-errors))]
- (when (and (seq ingest-errors) (not continue-on-error))
- (let [file-names (mapv #(or (:file (ex-data %)) (ex-message %)) ingest-errors)]
- (throw (ex-info (format "Failed to read %d file(s) during ingestion: %s"
- (count ingest-errors)
- (str/join ", " file-names))
- {:ingest-errors ingest-errors
- :files file-names}
- (first ingest-errors)))))
- (log/infof "Starting deserialization, total %s documents" (count contents))
- (reduce (fn [ctx item]
- (try
- (load-one! ctx item)
- (catch Exception e
- (when-not continue-on-error
- (throw e))
- ;; eschew big and scary stacktrace
- (log/warnf (u/strip-error e "Skipping deserialization error"))
- (update ctx :errors conj e))))
- ctx
- contents))
- (when reindex?
- ;; Reindex after all entities are loaded. Individual entity commits may have produced stale
- ;; search index entries; this ensures the index reflects the final state.
- (search/reindex!))))
+ (binding [serdes/*skip-schema-validation?* (serialization.settings/serialization-skip-schema-validation)]
+ (u/prog1
+ ;; Each entity is loaded in its own transaction (inside load-one!), so a deadlock or transient
+ ;; failure on one entity doesn't abort the entire import. See #74412.
+ ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared
+ ;; dependencies guide the import, and make sure all containers are imported before contents, etc.
+ (let [contents (serdes.ingest/ingest-list ingestion)
+ ingest-errors (serdes.ingest/ingest-errors ingestion)
+ ctx (cond-> (new-context ingestion)
+ (seq ingest-errors) (update :errors into ingest-errors))]
+ (when (and (seq ingest-errors) (not continue-on-error))
+ (let [file-names (mapv #(or (:file (ex-data %)) (ex-message %)) ingest-errors)]
+ (throw (ex-info (format "Failed to read %d file(s) during ingestion: %s"
+ (count ingest-errors)
+ (str/join ", " file-names))
+ {:ingest-errors ingest-errors
+ :files file-names}
+ (first ingest-errors)))))
+ (log/infof "Starting deserialization, total %s documents" (count contents))
+ (reduce (fn [ctx item]
+ (try
+ (load-one! ctx item)
+ (catch Exception e
+ (when-not continue-on-error
+ (throw e))
+ ;; eschew big and scary stacktrace
+ (log/warnf (u/strip-error e "Skipping deserialization error"))
+ (update ctx :errors conj e))))
+ ctx
+ contents))
+ (when reindex?
+ ;; Reindex after all entities are loaded. Individual entity commits may have produced stale
+ ;; search index entries; this ensures the index reflects the final state.
+ (search/reindex!)))))
diff --git a/enterprise/backend/test/metabase_enterprise/audit_app/instance_analytics_bundle_test.clj b/enterprise/backend/test/metabase_enterprise/audit_app/instance_analytics_bundle_test.clj
new file mode 100644
index 00000000000..268853a7ab8
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/audit_app/instance_analytics_bundle_test.clj
@@ -0,0 +1,50 @@
+(ns metabase-enterprise.audit-app.instance-analytics-bundle-test
+ (:require
+ [clojure.java.io :as io]
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.v2.ingest :as v2.ingest]
+ [metabase.models.serialization :as serdes]))
+
+(set! *warn-on-reflection* true)
+
+(defn- bundle-ingestion
+ "An ingestion over the shipped instance-analytics bundle."
+ []
+ (let [dir (io/file (io/resource "instance_analytics"))]
+ (v2.ingest/ingest-yaml (.getPath dir))))
+
+(defn- bundled-queries
+ "Every full query in `ingestion`, as `[entity-id query]` pairs."
+ [ingestion]
+ (vec (for [path (v2.ingest/ingest-list ingestion)
+ :let [query (:dataset_query (v2.ingest/ingest-one ingestion path))]
+ :when query]
+ [(:id (last path)) query])))
+
+(deftest ^:parallel bundled-queries-match-this-instances-mbql-schema-test
+ (testing (str "GHY-4241: every query in resources/instance_analytics must import cleanly into this version. "
+ "Instance analytics re-imports on every EE boot via `ensure-audit-db-installed!`, which rethrows "
+ "anything that isn't lock contention, and `start-normally` turns that into `(System/exit 1)`. So a "
+ "bundled query this version cannot represent is a boot crash, not a stale-content bug.")
+ ;; FK resolution is stubbed out: the bundle's portable refs point at the audit DB, which doesn't exist here, and
+ ;; what's under test is the query shape rather than the resolution.
+ (binding [serdes/*import-database-fk* (constantly 1)
+ serdes/*import-table-fk* (constantly 1)
+ serdes/*import-field-fk* (constantly 1)
+ serdes/*import-fk* (fn [& _] 1)]
+ (let [ingestion (bundle-ingestion)
+ queries (bundled-queries ingestion)
+ failures (for [[entity-id query] queries
+ :let [error (try
+ (serdes/import-mbql query)
+ nil
+ (catch Exception e (ex-message e)))]
+ :when error]
+ [entity-id error])]
+ (testing "the bundle was found and actually contains queries, so an empty pass can't be a false negative"
+ (is (pos? (count queries))))
+ (testing (str "every bundled file parses - `ingest-list` collects unreadable ones in an errors atom instead "
+ "of failing, so they would vanish from the queries above while still crashing boot")
+ (is (= [] (mapv #(or (:file (ex-data %)) (ex-message %))
+ (v2.ingest/ingest-errors ingestion)))))
+ (is (= [] (vec failures)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index a21b13bacef..648956064bf 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -61,6 +61,59 @@
;;; confound your tests with data from your dev appdb, remember to eagerly
;;; `(into [] (extract/extract ...))` in these tests.
+(defn- cause-chain-messages
+ "Messages of `e` and every exception beneath it, skipping any that have none."
+ [e]
+ ;; `keep`, not `map` - an exception with a nil message would NPE the callers' `re-find` and hide the real failure
+ (into [] (keep ex-message) (take-while some? (iterate ex-cause e))))
+
+(defn- load-failure-messages!
+ "Loads `ingestion`, expecting it to throw, and returns the thrown exception's cause-chain messages."
+ [ingestion]
+ (try
+ (serdes.load/load-metabase! ingestion)
+ ["load-metabase! unexpectedly succeeded"]
+ (catch Exception e
+ (cause-chain-messages e))))
+
+(deftest schema-validation-opt-out-reaches-import-test
+ (testing (str "GHY-4241: MB_SERIALIZATION_SKIP_SCHEMA_VALIDATION has to travel env var -> Setting -> the binding "
+ "in load-metabase! -> import-mbql. Nothing else covers that chain, so dropping the binding would "
+ "silently disable the opt-out.")
+ (let [extracted (atom nil)]
+ (mt/with-empty-h2-app-db!
+ (let [db (ts/create! :model/Database :name "my-db")
+ coll (ts/create! :model/Collection :name "Some collection")
+ card (ts/create! :model/Card
+ :name "Native with a variable"
+ :collection_id (:id coll)
+ :dataset_query {:database (:id db)
+ :type :native
+ :native {:template-tags {"id" {:id "e2d15f07-37b3-01fc-3944-2ff860a5eb46"
+ :name "id"
+ :display-name "ID"
+ :type :number}}
+ :query "SELECT 1 WHERE x = {{id}}"}})]
+ (reset! extracted {:db (serdes/extract-one "Database" {} db)
+ :coll (serdes/extract-one "Collection" {} coll)
+ :card (serdes/extract-one "Card" {} card)})))
+ ;; a tag type this version has no representation for - what an export from a newer Metabase that introduced
+ ;; one would look like
+ (let [{:keys [db coll card]} @extracted
+ bad-card (assoc-in card [:dataset_query :stages 0 :template-tags]
+ {"id" {:type :tag-type-from-the-future :name "id" :display-name "ID" :id "abc-123"}})
+ ingestion #(ingestion-in-memory [db coll bad-card])
+ ours? #(some (partial re-find #"does not match this Metabase's query schema") %)]
+ (testing "by default the schema check is what refuses the import"
+ (mt/with-empty-h2-app-db!
+ (is (ours? (load-failure-messages! (ingestion))))))
+ (testing "with the opt-out set the schema check is skipped, so the import fails downstream instead"
+ (mt/with-empty-h2-app-db!
+ (mt/with-temp-env-var-value! [mb-serialization-skip-schema-validation "true"]
+ (let [messages (load-failure-messages! (ingestion))]
+ (is (some (partial re-find #"Invalid input.*:template-tags") messages))
+ (is (not (ours? messages)))))))))))
+
(deftest load-basics-test
(testing "a simple, fresh collection is imported"
(let [serialized (atom nil)
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index fc8bdcb2a0f..a158d748a6f 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -64,6 +64,7 @@
;; legacy usages -- do not use in new code
^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]
[metabase.lib.core :as lib]
+ [metabase.lib.schema :as lib.schema]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.parameter :as lib.schema.parameter]
@@ -75,6 +76,7 @@
[metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
+ [metabase.util.malli.humanize :as mu.humanize]
[metabase.util.malli.registry :as mr]
[metabase.util.match :as match]
[toucan2.core :as t2]
@@ -1270,7 +1272,34 @@
(m :guard map?)
(import-mbql-map m)))
-(defn- normalize-imported [x]
+;; Unfortunately, settings depend on serdes, so we can't read settings directly in serdes (circular dep)
+(def ^:dynamic *skip-schema-validation?*
+ "When true, [[import-mbql]] stores a normalized query without checking it against this instance's query schema."
+ false)
+
+(defn- validate-imported-query!
+ "Throws when `query` is a full MBQL 5 query that this instance's query schema rejects. Anything else - bare refs,
+ the MBQL fragments inside visualization settings, legacy MBQL 4 queries - is left alone."
+ [query]
+ (when (and (= (:lib/type query) :mbql/query)
+ (not (mr/validate ::lib.schema/query query)))
+ (let [errors (mu.humanize/humanize (mr/explain ::lib.schema/query query))]
+ ;; the message names two causes because `mu/defn` is not instrumented in prod: an app DB can hold MBQL the QP
+ ;; tolerates but this schema rejects, so a refusal is not on its own evidence of a newer export
+ (throw (ex-info (str "Refusing to import a query that does not match this Metabase's query schema. It was "
+ "either exported by a newer Metabase whose query shape this version cannot represent, "
+ "or stored by an instance that never validated it. Pass continue_on_error to skip just "
+ "this entity, or set MB_SERIALIZATION_SKIP_SCHEMA_VALIDATION=true to skip this check "
+ "for the whole import.")
+ ;; no `:status`/`:status-code` here - `load-one!` rewraps everything thrown from this
+ ;; block in a fresh ex-info, so nothing we attach reaches the API's status handling
+ {:schema-errors errors}))))
+ query)
+
+(defn- normalize-imported
+ "Normalizes ingested MBQL/structure into this instance's representation, returning `x` unchanged if normalization
+ fails."
+ [x]
(when x
(try
(if (mbql-ref? x)
@@ -1278,6 +1307,7 @@
(lib/normalize x))
(catch Throwable e
(log/warnf "Error normalizing imported MBQL: %s" (ex-message e))
+ ;; many structures will fail normalization, but that is expected
x))))
(defn- import-mbql*
@@ -1312,12 +1342,16 @@
x))
(defn import-mbql
- "Given an MBQL expression as an EDN structure with portable IDs embedded, convert the IDs back to raw numeric IDs."
+ "Given an MBQL expression (or any structure that may contain portable references) as an EDN structure with portable
+ IDs embedded, convert the IDs back to raw numeric IDs.
+
+ Throws if an MBQL 5 expression doesn't match the schema."
[x]
- (-> x
- import-mbql*
- normalize-imported
- repair-card-template-tag-names))
+ (some-> x
+ import-mbql*
+ normalize-imported
+ (cond-> (not *skip-schema-validation?*) validate-imported-query!)
+ repair-card-template-tag-names))
(declare ^:private mbql-deps-map)
diff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.clj
index 39d1cd775cd..1c84595cb69 100644
--- a/test/metabase/models/serialization_test.clj
+++ b/test/metabase/models/serialization_test.clj
@@ -313,3 +313,40 @@
:values_source_type :card
:values_source_config {:card_id 1, :value_field [:field 53 nil]}
:position 0}])))))
+
+(def ^:private native-query-with-template-tag
+ "A native query with one variable, already past FK resolution (numeric `:database`), as [[serdes/import-mbql]]
+ sees it mid-import."
+ {:lib/type :mbql/query
+ :database 1
+ :stages [{:lib/type :mbql.stage/native
+ :native "SELECT * FROM PRODUCTS WHERE ID = {{id}}"
+ :template-tags {"id" {:type :number :name "id" :display-name "ID" :id "abc-123"}}}]})
+
+(def ^:private query-with-unknown-tag-type
+ "The same query with a template tag whose `:type` this version has no representation for - what an export from a
+ newer Metabase that introduced a new tag type would look like. It normalizes without complaint, so only validating
+ the result catches it."
+ (assoc-in native-query-with-template-tag [:stages 0 :template-tags]
+ {"id" {:type :tag-type-from-the-future :name "id" :display-name "ID" :id "abc-123"}}))
+
+(deftest ^:parallel import-mbql-validates-against-local-schema-test
+ (testing "GHY-4241: a query shape this version cannot represent is refused instead of stored"
+ (testing "a query matching this instance's schema imports"
+ (is (=? {:lib/type :mbql/query}
+ (serdes/import-mbql native-query-with-template-tag))))
+ (testing "a query carrying a shape with no representation here is refused"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"does not match this Metabase's query schema"
+ (serdes/import-mbql query-with-unknown-tag-type))))))
+
+(deftest ^:parallel import-mbql-schema-validation-opt-out-test
+ (testing "GHY-4241: binding *skip-schema-validation?* disables the schema check"
+ ;; Skipping does not make the import succeed - `repair-card-template-tag-names` rejects a tag type it does not
+ ;; know on its own. Asserting on that downstream failure keeps this from passing for the wrong reason.
+ (binding [serdes/*skip-schema-validation?* true]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Invalid input.*:template-tags"
+ (serdes/import-mbql query-with-unknown-tag-type))))))