Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add a namespace to generate a JSON Schema for MBQL 5 queries. (#72381)

technomancy2871c86b3fda
diff --git a/.dir-locals.el b/.dir-locals.el
index 3df80a91c14..44dfd187e29 100644
--- a/.dir-locals.el
+++ b/.dir-locals.el
@@ -87,6 +87,8 @@
(cider-default-cljs-repl . shadow-select)
(cider-shadow-default-options . "app"))
+ (makefile-mode (indent-tabs-mode . t))
+
("bin"
(clojure-mode
(cider-clojure-cli-aliases . "dev:drivers:build:build-dev"))))
diff --git a/deps.edn b/deps.edn
index 4ed8fc5dd3a..3141882d85d 100644
--- a/deps.edn
+++ b/deps.edn
@@ -263,6 +263,7 @@
com.clojure-goes-fast/clj-memory-meter
{:mvn/version "0.4.0"} ; Enables easy memory measurement
com.gfredericks/test.chuck {:mvn/version "0.2.15"} ; generating strings from regexes (useful with malli)
+ com.github.erosb/json-sKema {:mvn/version "0.25.0"} ; validating MBQL JSON Schema
criterium/criterium {:mvn/version "0.4.6"} ; benchmarking library
djblue/portal {:mvn/version "0.62.0"} ; ui for inspecting values
integrant/integrant {:mvn/version "1.0.1"} ; component lifecycle management
diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljc
index f90389c6604..a363ad8a39a 100644
--- a/src/metabase/lib/schema.cljc
+++ b/src/metabase/lib/schema.cljc
@@ -558,11 +558,11 @@
::id/table [:cat ::id/database ::id/schema :string]
::id/field [:cat ::id/database ::id/schema :string [:+ :string]]
;; this spec has a :multi clause that assumes field IDs
- ;; must be integers. the 3 in the update call refers to
- ;; the :multi; if that gets moved, this'll need to change
- :mbql.clause/field (update (mr/schema :mbql.clause/field) 3
- conj [:dispatch-type/sequential
- ::ref/field.id])
+ ;; must be integers. the 3 in the assoc-in call refers to
+ ;; the :multi and the 2 refers to :dispatch-type/integer;
+ ;; if those get moved, this will need to change
+ :mbql.clause/field (assoc-in (mr/schema :mbql.clause/field)
+ [3 2 0] :dispatch-type/sequential)
;; similarly we need to get rid of the :lib/uuid key of
;; a map that's nested in position 2 of an :and schema:
::common/options (update (mr/schema ::common/options) 2
diff --git a/src/metabase/lib_be/json_schema.clj b/src/metabase/lib_be/json_schema.clj
new file mode 100644
index 00000000000..5541a06e4ed
--- /dev/null
+++ b/src/metabase/lib_be/json_schema.clj
@@ -0,0 +1,89 @@
+(ns metabase.lib-be.json-schema
+ "Create JSON schema for MBQL 5 queries based on Malli schema.
+
+ Malli already has built-in functionality for this. Why not use it directly?
+
+ We're working around a few bugs in Malli's own JSON schema compiler.
+ It sometimes emits branching nodes like :allOf, :anyOf, :oneOf with either
+ one or zero children, which doesn't make any sense. These should be
+ collapsed down. Empty conditions should be removed from these as well.
+
+ Our malli schema for what constitutes a valid JSON schema is a little stricter
+ about what the keys have to look like; for instance types must always be
+ keywords despite the fact that we're emitting them as JSON in the end so
+ it doesn't really matter."
+ (:require
+ [malli.json-schema :as mjs]
+ [medley.core :as m]
+ [metabase.lib.schema :as schema]
+ [metabase.util :as u]
+ [metabase.util.json :as json-util]
+ [metabase.util.performance :as mp]))
+
+(defn- collapse-branches [schema k]
+ ;; this happens when we use `[:and ... [:fn ...]]`, the `:fn` schema
+ ;; gets converted into an empty object in :allOf; for example ::literal/integer
+ ;; in Malli is [:or :int [:fn u.number/bigint?]] which Malli compiles to
+ ;; {:anyOf [{:type "integer"}]} but it should just become {:type "integer"}
+ (let [schema (update schema k (partial remove mp/empty?))]
+ (case (count (k schema))
+ 1 (merge (dissoc schema k) (first (k schema)))
+ 0 (dissoc schema k)
+ schema)))
+
+(defn- stringify [required]
+ (if (sequential? required)
+ (map u/qualified-name required)
+ required))
+
+(defn- update-schema-fields [node]
+ (if (map? node)
+ (-> node
+ (m/update-existing :required stringify)
+ (collapse-branches :allOf)
+ (collapse-branches :anyOf)
+ (collapse-branches :oneOf)
+ (m/update-existing :properties #(mp/update-keys % u/qualified-name)))
+ node))
+
+;; due to a bug in malli, :cat schemas get compiled to an empty schema. this
+;; wouldn't normally cause false negatives (just false positives) but there are
+;; cases when a :cat schema lands in a :oneOf schema where it can't be allowed
+;; to pass both branches; in that case a false positive becomes a false negative.
+;; we should fix this in malli, but for the time being, we work around it.
+(defn- remove-one-of-cat [definitions]
+ (assoc definitions
+ "metabase.lib.schema.expression.boolean" {}
+ "metabase.lib.schema.expression.expression" {}
+ "metabase.lib.schema.expression.orderable" {}
+ "metabase.lib.schema.mbql-clause.clause" {}))
+
+(defn make-schema
+ "Generate a schema from Malli and apply fixes."
+ []
+ ;; here we generate a json schema and then make some modifications to adjust
+ ;; it to "external MBQL". why not create an "external MBQL" malli schema? well
+ ;; the references to things like "metabase.lib.schema.id.field" are spread out
+ ;; all over the place so it would be hard to swap them all with our new
+ ;; "externalized" field-id schema. if we turn it into json-schema then there's
+ ;; only one place that those swaps need to happen.
+ (let [schema (-> (mjs/transform ::schema/external-query)
+ ;; many of the updates are done below in `walk` if they need
+ ;; to work at any nesting level, but definitions are top-level
+ ;; and can be adjusted immediately.
+ (update :definitions remove-one-of-cat)
+ ;; this one breaks because the first oneOf branch is supposed to be for
+ ;; UUIDs and the second branch falls back to any non-blank string, but a
+ ;; UUID-shaped string will match both branches, so just check the latter.
+ (update :definitions assoc
+ "metabase.lib.schema.template-tag.id"
+ {"$ref" "#/definitions/metabase.lib.schema.common.non-blank-string"}))]
+ (mp/postwalk update-schema-fields schema)))
+
+(defn write-schema
+ "Print JSON Schema for MBQL5 queries, or write to a file if given."
+ [{:keys [file]}]
+ (let [json (json-util/encode (make-schema) {:pretty true})]
+ (spit (or file "/dev/stdout") json)))
+
+(comment (write-schema {:file "schema.json"}))
diff --git a/test/metabase/lib_be/json_schema_test.clj b/test/metabase/lib_be/json_schema_test.clj
new file mode 100644
index 00000000000..f7427b1fa25
--- /dev/null
+++ b/test/metabase/lib_be/json_schema_test.clj
@@ -0,0 +1,58 @@
+(ns metabase.lib-be.json-schema-test
+ (:require
+ [clj-yaml.core :as yaml]
+ [clojure.java.io :as io]
+ [clojure.test :refer :all]
+ [clojure.walk :as walk]
+ [metabase.lib-be.json-schema :as js]
+ [metabase.lib.core :as lib]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.util.json :as json-util])
+ (:import (com.github.erosb.jsonsKema FormatValidationPolicy
+ JsonParser
+ SchemaLoader
+ Validator
+ ValidatorConfig)))
+
+(set! *warn-on-reflection* true)
+
+(defn check-node [node]
+ (when (map? node)
+ (when (contains? node "type")
+ (is (not= nil (node "type"))))
+ (when-let [one-of (:oneOf node)]
+ (is (< 1 (count one-of)))
+ (is (not (some empty? one-of)) (pr-str :empty-one-of one-of)))
+ (when-let [any-of (:anyOf node)]
+ (is (< 1 (count any-of)))
+ (is (not (some empty? any-of)) (pr-str :empty-any-of any-of)))
+ (when-let [all-of (:allOf node)]
+ (is (< 1 (count all-of)))
+ (is (not (some empty? all-of)) (pr-str :empty-all-of all-of))))
+ node)
+
+(def simple-query
+ (-> (lib/query meta/metadata-provider
+ (meta/table-metadata :orders))
+ (lib/aggregate (lib/count))
+ ;; lib generates "internal MBQL"; can't have integer :table/:database
+ (assoc-in [:stages 0 :source-table] ["hello" "schema" "tbl"])
+ (assoc :database "my-db")))
+
+(deftest fix-json-schema-test
+ (let [schema-map (js/make-schema)
+ schema (.load (SchemaLoader. (json-util/encode schema-map)))
+ validator (Validator/create schema (ValidatorConfig. FormatValidationPolicy/ALWAYS))]
+ (testing "does it pass the malli schema?"
+ (is (map? schema-map)))
+ (testing "it should remove malli quirks"
+ (walk/postwalk check-node schema))
+ (testing "simple query"
+ (is (nil? (.validate validator
+ (.parse (JsonParser. (json-util/encode simple-query)))))))
+ (testing "representation examples validate"
+ (let [examples-dir "serialization_baseline/collections/main/queries"]
+ (doseq [f (.listFiles (io/file (io/resource examples-dir)))]
+ (let [query (:dataset_query (yaml/parse-string (slurp f)))]
+ (is (nil? (.validate validator
+ (.parse (JsonParser. (json-util/encode query))))))))))))