The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add a namespace to generate a JSON Schema for MBQL 5 queries. (#72381)
technomancy2871c86b3fda
diff --git a/.dir-locals.el b/.dir-locals.elindex 3df80a91c14..44dfd187e29 100644--- a/.dir-locals.el+++ b/.dir-locals.el@@ -87,6 +87,8 @@(cider-default-cljs-repl . shadow-select)(cider-shadow-default-options . "app"))+ (makefile-mode (indent-tabs-mode . t))+("bin"(clojure-mode(cider-clojure-cli-aliases . "dev:drivers:build:build-dev"))))diff --git a/deps.edn b/deps.ednindex 4ed8fc5dd3a..3141882d85d 100644--- a/deps.edn+++ b/deps.edn@@ -263,6 +263,7 @@com.clojure-goes-fast/clj-memory-meter{:mvn/version "0.4.0"} ; Enables easy memory measurementcom.gfredericks/test.chuck {:mvn/version "0.2.15"} ; generating strings from regexes (useful with malli)+ com.github.erosb/json-sKema {:mvn/version "0.25.0"} ; validating MBQL JSON Schemacriterium/criterium {:mvn/version "0.4.6"} ; benchmarking librarydjblue/portal {:mvn/version "0.62.0"} ; ui for inspecting valuesintegrant/integrant {:mvn/version "1.0.1"} ; component lifecycle managementdiff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljcindex f90389c6604..a363ad8a39a 100644--- a/src/metabase/lib/schema.cljc+++ b/src/metabase/lib/schema.cljc@@ -558,11 +558,11 @@::id/table [:cat ::id/database ::id/schema :string]::id/field [:cat ::id/database ::id/schema :string [:+ :string]];; this spec has a :multi clause that assumes field IDs- ;; must be integers. the 3 in the update call refers to- ;; the :multi; if that gets moved, this'll need to change- :mbql.clause/field (update (mr/schema :mbql.clause/field) 3- conj [:dispatch-type/sequential- ::ref/field.id])+ ;; must be integers. the 3 in the assoc-in call refers to+ ;; the :multi and the 2 refers to :dispatch-type/integer;+ ;; if those get moved, this will need to change+ :mbql.clause/field (assoc-in (mr/schema :mbql.clause/field)+ [3 2 0] :dispatch-type/sequential);; similarly we need to get rid of the :lib/uuid key of;; a map that's nested in position 2 of an :and schema:::common/options (update (mr/schema ::common/options) 2diff --git a/src/metabase/lib_be/json_schema.clj b/src/metabase/lib_be/json_schema.cljnew file mode 100644index 00000000000..5541a06e4ed--- /dev/null+++ b/src/metabase/lib_be/json_schema.clj@@ -0,0 +1,89 @@+(ns metabase.lib-be.json-schema+ "Create JSON schema for MBQL 5 queries based on Malli schema.++ Malli already has built-in functionality for this. Why not use it directly?++ We're working around a few bugs in Malli's own JSON schema compiler.+ It sometimes emits branching nodes like :allOf, :anyOf, :oneOf with either+ one or zero children, which doesn't make any sense. These should be+ collapsed down. Empty conditions should be removed from these as well.++ Our malli schema for what constitutes a valid JSON schema is a little stricter+ about what the keys have to look like; for instance types must always be+ keywords despite the fact that we're emitting them as JSON in the end so+ it doesn't really matter."+ (:require+ [malli.json-schema :as mjs]+ [medley.core :as m]+ [metabase.lib.schema :as schema]+ [metabase.util :as u]+ [metabase.util.json :as json-util]+ [metabase.util.performance :as mp]))++(defn- collapse-branches [schema k]+ ;; this happens when we use `[:and ... [:fn ...]]`, the `:fn` schema+ ;; gets converted into an empty object in :allOf; for example ::literal/integer+ ;; in Malli is [:or :int [:fn u.number/bigint?]] which Malli compiles to+ ;; {:anyOf [{:type "integer"}]} but it should just become {:type "integer"}+ (let [schema (update schema k (partial remove mp/empty?))]+ (case (count (k schema))+ 1 (merge (dissoc schema k) (first (k schema)))+ 0 (dissoc schema k)+ schema)))++(defn- stringify [required]+ (if (sequential? required)+ (map u/qualified-name required)+ required))++(defn- update-schema-fields [node]+ (if (map? node)+ (-> node+ (m/update-existing :required stringify)+ (collapse-branches :allOf)+ (collapse-branches :anyOf)+ (collapse-branches :oneOf)+ (m/update-existing :properties #(mp/update-keys % u/qualified-name)))+ node))++;; due to a bug in malli, :cat schemas get compiled to an empty schema. this+;; wouldn't normally cause false negatives (just false positives) but there are+;; cases when a :cat schema lands in a :oneOf schema where it can't be allowed+;; to pass both branches; in that case a false positive becomes a false negative.+;; we should fix this in malli, but for the time being, we work around it.+(defn- remove-one-of-cat [definitions]+ (assoc definitions+ "metabase.lib.schema.expression.boolean" {}+ "metabase.lib.schema.expression.expression" {}+ "metabase.lib.schema.expression.orderable" {}+ "metabase.lib.schema.mbql-clause.clause" {}))++(defn make-schema+ "Generate a schema from Malli and apply fixes."+ []+ ;; here we generate a json schema and then make some modifications to adjust+ ;; it to "external MBQL". why not create an "external MBQL" malli schema? well+ ;; the references to things like "metabase.lib.schema.id.field" are spread out+ ;; all over the place so it would be hard to swap them all with our new+ ;; "externalized" field-id schema. if we turn it into json-schema then there's+ ;; only one place that those swaps need to happen.+ (let [schema (-> (mjs/transform ::schema/external-query)+ ;; many of the updates are done below in `walk` if they need+ ;; to work at any nesting level, but definitions are top-level+ ;; and can be adjusted immediately.+ (update :definitions remove-one-of-cat)+ ;; this one breaks because the first oneOf branch is supposed to be for+ ;; UUIDs and the second branch falls back to any non-blank string, but a+ ;; UUID-shaped string will match both branches, so just check the latter.+ (update :definitions assoc+ "metabase.lib.schema.template-tag.id"+ {"$ref" "#/definitions/metabase.lib.schema.common.non-blank-string"}))]+ (mp/postwalk update-schema-fields schema)))++(defn write-schema+ "Print JSON Schema for MBQL5 queries, or write to a file if given."+ [{:keys [file]}]+ (let [json (json-util/encode (make-schema) {:pretty true})]+ (spit (or file "/dev/stdout") json)))++(comment (write-schema {:file "schema.json"}))diff --git a/test/metabase/lib_be/json_schema_test.clj b/test/metabase/lib_be/json_schema_test.cljnew file mode 100644index 00000000000..f7427b1fa25--- /dev/null+++ b/test/metabase/lib_be/json_schema_test.clj@@ -0,0 +1,58 @@+(ns metabase.lib-be.json-schema-test+ (:require+ [clj-yaml.core :as yaml]+ [clojure.java.io :as io]+ [clojure.test :refer :all]+ [clojure.walk :as walk]+ [metabase.lib-be.json-schema :as js]+ [metabase.lib.core :as lib]+ [metabase.lib.test-metadata :as meta]+ [metabase.util.json :as json-util])+ (:import (com.github.erosb.jsonsKema FormatValidationPolicy+ JsonParser+ SchemaLoader+ Validator+ ValidatorConfig)))++(set! *warn-on-reflection* true)++(defn check-node [node]+ (when (map? node)+ (when (contains? node "type")+ (is (not= nil (node "type"))))+ (when-let [one-of (:oneOf node)]+ (is (< 1 (count one-of)))+ (is (not (some empty? one-of)) (pr-str :empty-one-of one-of)))+ (when-let [any-of (:anyOf node)]+ (is (< 1 (count any-of)))+ (is (not (some empty? any-of)) (pr-str :empty-any-of any-of)))+ (when-let [all-of (:allOf node)]+ (is (< 1 (count all-of)))+ (is (not (some empty? all-of)) (pr-str :empty-all-of all-of))))+ node)++(def simple-query+ (-> (lib/query meta/metadata-provider+ (meta/table-metadata :orders))+ (lib/aggregate (lib/count))+ ;; lib generates "internal MBQL"; can't have integer :table/:database+ (assoc-in [:stages 0 :source-table] ["hello" "schema" "tbl"])+ (assoc :database "my-db")))++(deftest fix-json-schema-test+ (let [schema-map (js/make-schema)+ schema (.load (SchemaLoader. (json-util/encode schema-map)))+ validator (Validator/create schema (ValidatorConfig. FormatValidationPolicy/ALWAYS))]+ (testing "does it pass the malli schema?"+ (is (map? schema-map)))+ (testing "it should remove malli quirks"+ (walk/postwalk check-node schema))+ (testing "simple query"+ (is (nil? (.validate validator+ (.parse (JsonParser. (json-util/encode simple-query)))))))+ (testing "representation examples validate"+ (let [examples-dir "serialization_baseline/collections/main/queries"]+ (doseq [f (.listFiles (io/file (io/resource examples-dir)))]+ (let [query (:dataset_query (yaml/parse-string (slurp f)))]+ (is (nil? (.validate validator+ (.parse (JsonParser. (json-util/encode query))))))))))))