The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Validate imported queries against this instance's MBQL schema (GHY-4241) (#79496)
Eric Normand6bd66a6d5eba
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex a89e54a4394..7e5027bd51e 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -3724,7 +3724,10 @@enterprise/serialization{:team "Graphy"- :api #{metabase-enterprise.serialization.api metabase-enterprise.serialization.cmd metabase-enterprise.serialization.core}+ :api #{metabase-enterprise.serialization.api+ metabase-enterprise.serialization.cmd+ metabase-enterprise.serialization.core+ metabase-enterprise.serialization.init}:friends #{enterprise/memoize-monitor}:uses #{analyticsapi@@ -3739,6 +3742,7 @@premium-featuressearchserver+ settingssetuputil}:model-imports :bypass}diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.cljindex 95b3a484fe7..eb2a57b7434 100644--- a/enterprise/backend/src/metabase_enterprise/core/init.clj+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj@@ -26,6 +26,7 @@[metabase-enterprise.scim.init][metabase-enterprise.security-center.init][metabase-enterprise.semantic-search.init]+ [metabase-enterprise.serialization.init][metabase-enterprise.sso.init][metabase-enterprise.stale.init][metabase-enterprise.support-access-grants.init]diff --git a/enterprise/backend/src/metabase_enterprise/serialization/init.clj b/enterprise/backend/src/metabase_enterprise/serialization/init.cljnew file mode 100644index 00000000000..a0fd6baaa0e--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/init.clj@@ -0,0 +1,3 @@+(ns metabase-enterprise.serialization.init+ (:require+ [metabase-enterprise.serialization.settings]))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/settings.clj b/enterprise/backend/src/metabase_enterprise/serialization/settings.cljnew file mode 100644index 00000000000..37242b6ff6f--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/settings.clj@@ -0,0 +1,15 @@+(ns metabase-enterprise.serialization.settings+ (:require+ [metabase.settings.core :refer [defsetting]]))++(defsetting serialization-skip-schema-validation+ "Whether to import questions whose queries this Metabase's query schema rejects. Defaults to false."+ :type :boolean+ :default false+ :visibility :internal+ :setter :none+ :audit :never+ :export? false+ :doc (str "On import, Metabase validates every question against the query format this version understands, and refuses "+ "any it cannot read. Set this to true to skip that validation. "+ "Skipping it will not necessarily make the import succeed. Import may still fail on a later step."))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.cljindex 41c9dc98f2b..fcc9f84239f 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj@@ -4,6 +4,7 @@(:require[clojure.string :as str][diehard.core :as dh]+ [metabase-enterprise.serialization.settings :as serialization.settings][metabase-enterprise.serialization.v2.ingest :as serdes.ingest][metabase-enterprise.serialization.v2.models :as serdes.models][metabase.app-db.core :as mdb]@@ -257,36 +258,37 @@[ingestion & {:keys [continue-on-error reindex?]:or {continue-on-error falsereindex? true}}]- (u/prog1- ;; Each entity is loaded in its own transaction (inside load-one!), so a deadlock or transient- ;; failure on one entity doesn't abort the entire import. See #74412.- ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared- ;; dependencies guide the import, and make sure all containers are imported before contents, etc.- (let [contents (serdes.ingest/ingest-list ingestion)- ingest-errors (serdes.ingest/ingest-errors ingestion)- ctx (cond-> (new-context ingestion)- (seq ingest-errors) (update :errors into ingest-errors))]- (when (and (seq ingest-errors) (not continue-on-error))- (let [file-names (mapv #(or (:file (ex-data %)) (ex-message %)) ingest-errors)]- (throw (ex-info (format "Failed to read %d file(s) during ingestion: %s"- (count ingest-errors)- (str/join ", " file-names))- {:ingest-errors ingest-errors- :files file-names}- (first ingest-errors)))))- (log/infof "Starting deserialization, total %s documents" (count contents))- (reduce (fn [ctx item]- (try- (load-one! ctx item)- (catch Exception e- (when-not continue-on-error- (throw e))- ;; eschew big and scary stacktrace- (log/warnf (u/strip-error e "Skipping deserialization error"))- (update ctx :errors conj e))))- ctx- contents))- (when reindex?- ;; Reindex after all entities are loaded. Individual entity commits may have produced stale- ;; search index entries; this ensures the index reflects the final state.- (search/reindex!))))+ (binding [serdes/*skip-schema-validation?* (serialization.settings/serialization-skip-schema-validation)]+ (u/prog1+ ;; Each entity is loaded in its own transaction (inside load-one!), so a deadlock or transient+ ;; failure on one entity doesn't abort the entire import. See #74412.+ ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared+ ;; dependencies guide the import, and make sure all containers are imported before contents, etc.+ (let [contents (serdes.ingest/ingest-list ingestion)+ ingest-errors (serdes.ingest/ingest-errors ingestion)+ ctx (cond-> (new-context ingestion)+ (seq ingest-errors) (update :errors into ingest-errors))]+ (when (and (seq ingest-errors) (not continue-on-error))+ (let [file-names (mapv #(or (:file (ex-data %)) (ex-message %)) ingest-errors)]+ (throw (ex-info (format "Failed to read %d file(s) during ingestion: %s"+ (count ingest-errors)+ (str/join ", " file-names))+ {:ingest-errors ingest-errors+ :files file-names}+ (first ingest-errors)))))+ (log/infof "Starting deserialization, total %s documents" (count contents))+ (reduce (fn [ctx item]+ (try+ (load-one! ctx item)+ (catch Exception e+ (when-not continue-on-error+ (throw e))+ ;; eschew big and scary stacktrace+ (log/warnf (u/strip-error e "Skipping deserialization error"))+ (update ctx :errors conj e))))+ ctx+ contents))+ (when reindex?+ ;; Reindex after all entities are loaded. Individual entity commits may have produced stale+ ;; search index entries; this ensures the index reflects the final state.+ (search/reindex!)))))diff --git a/enterprise/backend/test/metabase_enterprise/audit_app/instance_analytics_bundle_test.clj b/enterprise/backend/test/metabase_enterprise/audit_app/instance_analytics_bundle_test.cljnew file mode 100644index 00000000000..268853a7ab8--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/audit_app/instance_analytics_bundle_test.clj@@ -0,0 +1,50 @@+(ns metabase-enterprise.audit-app.instance-analytics-bundle-test+ (:require+ [clojure.java.io :as io]+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.v2.ingest :as v2.ingest]+ [metabase.models.serialization :as serdes]))++(set! *warn-on-reflection* true)++(defn- bundle-ingestion+ "An ingestion over the shipped instance-analytics bundle."+ []+ (let [dir (io/file (io/resource "instance_analytics"))]+ (v2.ingest/ingest-yaml (.getPath dir))))++(defn- bundled-queries+ "Every full query in `ingestion`, as `[entity-id query]` pairs."+ [ingestion]+ (vec (for [path (v2.ingest/ingest-list ingestion)+ :let [query (:dataset_query (v2.ingest/ingest-one ingestion path))]+ :when query]+ [(:id (last path)) query])))++(deftest ^:parallel bundled-queries-match-this-instances-mbql-schema-test+ (testing (str "GHY-4241: every query in resources/instance_analytics must import cleanly into this version. "+ "Instance analytics re-imports on every EE boot via `ensure-audit-db-installed!`, which rethrows "+ "anything that isn't lock contention, and `start-normally` turns that into `(System/exit 1)`. So a "+ "bundled query this version cannot represent is a boot crash, not a stale-content bug.")+ ;; FK resolution is stubbed out: the bundle's portable refs point at the audit DB, which doesn't exist here, and+ ;; what's under test is the query shape rather than the resolution.+ (binding [serdes/*import-database-fk* (constantly 1)+ serdes/*import-table-fk* (constantly 1)+ serdes/*import-field-fk* (constantly 1)+ serdes/*import-fk* (fn [& _] 1)]+ (let [ingestion (bundle-ingestion)+ queries (bundled-queries ingestion)+ failures (for [[entity-id query] queries+ :let [error (try+ (serdes/import-mbql query)+ nil+ (catch Exception e (ex-message e)))]+ :when error]+ [entity-id error])]+ (testing "the bundle was found and actually contains queries, so an empty pass can't be a false negative"+ (is (pos? (count queries))))+ (testing (str "every bundled file parses - `ingest-list` collects unreadable ones in an errors atom instead "+ "of failing, so they would vanish from the queries above while still crashing boot")+ (is (= [] (mapv #(or (:file (ex-data %)) (ex-message %))+ (v2.ingest/ingest-errors ingestion)))))+ (is (= [] (vec failures)))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex a21b13bacef..648956064bf 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -61,6 +61,59 @@;;; confound your tests with data from your dev appdb, remember to eagerly;;; `(into [] (extract/extract ...))` in these tests.+(defn- cause-chain-messages+ "Messages of `e` and every exception beneath it, skipping any that have none."+ [e]+ ;; `keep`, not `map` - an exception with a nil message would NPE the callers' `re-find` and hide the real failure+ (into [] (keep ex-message) (take-while some? (iterate ex-cause e))))++(defn- load-failure-messages!+ "Loads `ingestion`, expecting it to throw, and returns the thrown exception's cause-chain messages."+ [ingestion]+ (try+ (serdes.load/load-metabase! ingestion)+ ["load-metabase! unexpectedly succeeded"]+ (catch Exception e+ (cause-chain-messages e))))++(deftest schema-validation-opt-out-reaches-import-test+ (testing (str "GHY-4241: MB_SERIALIZATION_SKIP_SCHEMA_VALIDATION has to travel env var -> Setting -> the binding "+ "in load-metabase! -> import-mbql. Nothing else covers that chain, so dropping the binding would "+ "silently disable the opt-out.")+ (let [extracted (atom nil)]+ (mt/with-empty-h2-app-db!+ (let [db (ts/create! :model/Database :name "my-db")+ coll (ts/create! :model/Collection :name "Some collection")+ card (ts/create! :model/Card+ :name "Native with a variable"+ :collection_id (:id coll)+ :dataset_query {:database (:id db)+ :type :native+ :native {:template-tags {"id" {:id "e2d15f07-37b3-01fc-3944-2ff860a5eb46"+ :name "id"+ :display-name "ID"+ :type :number}}+ :query "SELECT 1 WHERE x = {{id}}"}})]+ (reset! extracted {:db (serdes/extract-one "Database" {} db)+ :coll (serdes/extract-one "Collection" {} coll)+ :card (serdes/extract-one "Card" {} card)})))+ ;; a tag type this version has no representation for - what an export from a newer Metabase that introduced+ ;; one would look like+ (let [{:keys [db coll card]} @extracted+ bad-card (assoc-in card [:dataset_query :stages 0 :template-tags]+ {"id" {:type :tag-type-from-the-future :name "id" :display-name "ID" :id "abc-123"}})+ ingestion #(ingestion-in-memory [db coll bad-card])+ ours? #(some (partial re-find #"does not match this Metabase's query schema") %)]+ (testing "by default the schema check is what refuses the import"+ (mt/with-empty-h2-app-db!+ (is (ours? (load-failure-messages! (ingestion))))))+ (testing "with the opt-out set the schema check is skipped, so the import fails downstream instead"+ (mt/with-empty-h2-app-db!+ (mt/with-temp-env-var-value! [mb-serialization-skip-schema-validation "true"]+ (let [messages (load-failure-messages! (ingestion))]+ (is (some (partial re-find #"Invalid input.*:template-tags") messages))+ (is (not (ours? messages)))))))))))+(deftest load-basics-test(testing "a simple, fresh collection is imported"(let [serialized (atom nil)diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex fc8bdcb2a0f..a158d748a6f 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -64,6 +64,7 @@;; legacy usages -- do not use in new code^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s][metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.parameter :as lib.schema.parameter]@@ -75,6 +76,7 @@[metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.humanize :as mu.humanize][metabase.util.malli.registry :as mr][metabase.util.match :as match][toucan2.core :as t2]@@ -1270,7 +1272,34 @@(m :guard map?)(import-mbql-map m)))-(defn- normalize-imported [x]+;; Unfortunately, settings depend on serdes, so we can't read settings directly in serdes (circular dep)+(def ^:dynamic *skip-schema-validation?*+ "When true, [[import-mbql]] stores a normalized query without checking it against this instance's query schema."+ false)++(defn- validate-imported-query!+ "Throws when `query` is a full MBQL 5 query that this instance's query schema rejects. Anything else - bare refs,+ the MBQL fragments inside visualization settings, legacy MBQL 4 queries - is left alone."+ [query]+ (when (and (= (:lib/type query) :mbql/query)+ (not (mr/validate ::lib.schema/query query)))+ (let [errors (mu.humanize/humanize (mr/explain ::lib.schema/query query))]+ ;; the message names two causes because `mu/defn` is not instrumented in prod: an app DB can hold MBQL the QP+ ;; tolerates but this schema rejects, so a refusal is not on its own evidence of a newer export+ (throw (ex-info (str "Refusing to import a query that does not match this Metabase's query schema. It was "+ "either exported by a newer Metabase whose query shape this version cannot represent, "+ "or stored by an instance that never validated it. Pass continue_on_error to skip just "+ "this entity, or set MB_SERIALIZATION_SKIP_SCHEMA_VALIDATION=true to skip this check "+ "for the whole import.")+ ;; no `:status`/`:status-code` here - `load-one!` rewraps everything thrown from this+ ;; block in a fresh ex-info, so nothing we attach reaches the API's status handling+ {:schema-errors errors}))))+ query)++(defn- normalize-imported+ "Normalizes ingested MBQL/structure into this instance's representation, returning `x` unchanged if normalization+ fails."+ [x](when x(try(if (mbql-ref? x)@@ -1278,6 +1307,7 @@(lib/normalize x))(catch Throwable e(log/warnf "Error normalizing imported MBQL: %s" (ex-message e))+ ;; many structures will fail normalization, but that is expectedx))))(defn- import-mbql*@@ -1312,12 +1342,16 @@x))(defn import-mbql- "Given an MBQL expression as an EDN structure with portable IDs embedded, convert the IDs back to raw numeric IDs."+ "Given an MBQL expression (or any structure that may contain portable references) as an EDN structure with portable+ IDs embedded, convert the IDs back to raw numeric IDs.++ Throws if an MBQL 5 expression doesn't match the schema."[x]- (-> x- import-mbql*- normalize-imported- repair-card-template-tag-names))+ (some-> x+ import-mbql*+ normalize-imported+ (cond-> (not *skip-schema-validation?*) validate-imported-query!)+ repair-card-template-tag-names))(declare ^:private mbql-deps-map)diff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.cljindex 39d1cd775cd..1c84595cb69 100644--- a/test/metabase/models/serialization_test.clj+++ b/test/metabase/models/serialization_test.clj@@ -313,3 +313,40 @@:values_source_type :card:values_source_config {:card_id 1, :value_field [:field 53 nil]}:position 0}])))))++(def ^:private native-query-with-template-tag+ "A native query with one variable, already past FK resolution (numeric `:database`), as [[serdes/import-mbql]]+ sees it mid-import."+ {:lib/type :mbql/query+ :database 1+ :stages [{:lib/type :mbql.stage/native+ :native "SELECT * FROM PRODUCTS WHERE ID = {{id}}"+ :template-tags {"id" {:type :number :name "id" :display-name "ID" :id "abc-123"}}}]})++(def ^:private query-with-unknown-tag-type+ "The same query with a template tag whose `:type` this version has no representation for - what an export from a+ newer Metabase that introduced a new tag type would look like. It normalizes without complaint, so only validating+ the result catches it."+ (assoc-in native-query-with-template-tag [:stages 0 :template-tags]+ {"id" {:type :tag-type-from-the-future :name "id" :display-name "ID" :id "abc-123"}}))++(deftest ^:parallel import-mbql-validates-against-local-schema-test+ (testing "GHY-4241: a query shape this version cannot represent is refused instead of stored"+ (testing "a query matching this instance's schema imports"+ (is (=? {:lib/type :mbql/query}+ (serdes/import-mbql native-query-with-template-tag))))+ (testing "a query carrying a shape with no representation here is refused"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"does not match this Metabase's query schema"+ (serdes/import-mbql query-with-unknown-tag-type))))))++(deftest ^:parallel import-mbql-schema-validation-opt-out-test+ (testing "GHY-4241: binding *skip-schema-validation?* disables the schema check"+ ;; Skipping does not make the import succeed - `repair-card-template-tag-names` rejects a tag type it does not+ ;; know on its own. Asserting on that downstream failure keeps this from passing for the wrong reason.+ (binding [serdes/*skip-schema-validation?* true]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Invalid input.*:template-tags"+ (serdes/import-mbql query-with-unknown-tag-type))))))