The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Legacy MBQL normalization overhaul: 3x faster and bug fixes (#65018)
Cam Saul7d879a9a8607
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex 3b2f02bdbcd..39e28f834ba 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -553,6 +553,8 @@metabase.embedding.api.embed-test/with-temp-card hooks.common/let-one-with-optional-valuemetabase.embedding.api.embed-test/with-temp-dashcard hooks.common/let-one-with-optional-valuemetabase.legacy-mbql.schema.macros/defclause hooks.metabase.legacy-mbql.schemas.macros/defclause+ metabase.legacy-mbql.schema.macros/defclause* hooks.metabase.legacy-mbql.schemas.macros/defclause*+ metabase.legacy-mbql.schema.macros/one-of hooks.metabase.legacy-mbql.schemas.macros/one-ofmetabase.lib.schema.mbql-clause/define-catn-mbql-clause hooks.metabase.lib.schema.mbql-clause/define-mbql-clausemetabase.lib.schema.mbql-clause/define-mbql-clause hooks.metabase.lib.schema.mbql-clause/define-mbql-clausemetabase.lib.schema.mbql-clause/define-tuple-mbql-clause hooks.metabase.lib.schema.mbql-clause/define-mbql-clausediff --git a/.clj-kondo/src/hooks/metabase/legacy_mbql/schemas/macros.clj b/.clj-kondo/src/hooks/metabase/legacy_mbql/schemas/macros.cljindex 54c14dff8da..6434b322487 100644--- a/.clj-kondo/src/hooks/metabase/legacy_mbql/schemas/macros.clj+++ b/.clj-kondo/src/hooks/metabase/legacy_mbql/schemas/macros.clj@@ -33,18 +33,36 @@(defclause [ag:var var] field-or-expression FieldOrExpressionDef)=>(def ag:var [FieldOrExpressionDef])"- [{:keys [node]}]- (let [[_defclause clause-name-form & arg-specs] (:children node)]- {:node (-> (hooks/list-node- (list- (hooks/token-node 'def)- (unwrap-defclause-clause-name clause-name-form)- (hooks/string-node "Docstring.")- (collect-defclause-body-schemas arg-specs)))- (with-meta (meta node)))}))+ [{:keys [node], :as x}]+ (let [[_defclause clause-name & arg-specs] (:children node)]+ (assoc x :node (-> (hooks/list-node+ (list+ (hooks/token-node 'do)+ (-> (hooks/keyword-node (keyword "metabase.legacy-mbql.schema" (name (hooks/sexpr clause-name))))+ (with-meta (meta clause-name))+ (hooks/reg-keyword! 'metabase.legacy-mbql.schemas.macros/defclause))+ (collect-defclause-body-schemas arg-specs)))+ (with-meta (meta node))))))++(defn defclause*+ "e.g.++ (defclause [ag:var var] field-or-expression FieldOrExpressionDef)+ =>+ (def ag:var [FieldOrExpressionDef])"+ [{:keys [node], :as x}]+ (let [[_defclause clause-name schema] (:children node)]+ (assoc x :node (-> (hooks/list-node+ (list+ (hooks/token-node 'do)+ (-> (hooks/keyword-node (keyword "metabase.legacy-mbql.schema" (name (hooks/sexpr clause-name))))+ (with-meta (meta clause-name))+ (hooks/reg-keyword! 'metabase.legacy-mbql.schemas.macros/defclause*))+ schema))+ (with-meta (meta node))))))(comment- (defn- defclause* [form]+ (defn- expand-defclause [form](hooks/sexpr(:node(defclause@@ -55,15 +73,35 @@form)))}))))(defn- x []- (defclause* '(defclause [ag:var var]- field-or-expression FieldOrExpressionDef)))+ (expand-defclause '(defclause var+ field-or-expression FieldOrExpressionDef)));; =>- (def ag:var "Docstring." [FieldOrExpressionDef])+ (do :metabase.legacy-mbql.schema/var [FieldOrExpressionDef])(defn- y []- (defclause* '(defclause var- x (some-list-schema)- y (optional FieldOrExpressionDef)- z (rest (some-other-schema)))))+ (expand-defclause '(defclause var+ x (some-list-schema)+ y (optional FieldOrExpressionDef)+ z (rest (some-other-schema)))));; =>- (def var "Docstring." [(some-list-schema) FieldOrExpressionDef (some-other-schema)]))+ (do :metabase.legacy-mbql.schema/var [(some-list-schema) FieldOrExpressionDef (some-other-schema)]))++(defn one-of [{:keys [node], :as x}]+ (let [[_one-of & schema-nodes] (:children node)+ node' (-> (hooks/vector-node+ (mapv (fn [schema-node]+ (if (and (hooks/token-node? schema-node)+ (simple-symbol? (hooks/sexpr schema-node)))+ (-> (hooks/keyword-node (keyword "mteabase.legacy-mbql.schema" (name (hooks/sexpr schema-node))))+ (with-meta (meta schema-node)))+ schema-node))+ schema-nodes))+ (with-meta (meta node)))]+ (assoc x :node node')))++(comment+ (defn- expand-one-of [form]+ (hooks/sexpr (:node (one-of {:node (hooks/parse-string (with-out-str (clojure.pprint/pprint form)))}))))++ (defn- z []+ (expand-one-of '(one-of absolute-datetime relative-datetime time))))diff --git a/e2e/test/scenarios/permissions/sandboxing/sandboxing-via-api.cy.spec.js b/e2e/test/scenarios/permissions/sandboxing/sandboxing-via-api.cy.spec.jsindex fed86cb9c99..ba84501725a 100644--- a/e2e/test/scenarios/permissions/sandboxing/sandboxing-via-api.cy.spec.js+++ b/e2e/test/scenarios/permissions/sandboxing/sandboxing-via-api.cy.spec.js@@ -420,6 +420,7 @@ describe("admin > permissions > sandboxes (tested via the API)", () => {[[">", ["field", ORDERS.DISCOUNT, null], 0],["field", ORDERS.DISCOUNT],+ // no idea why this is here, a `case` subclause only has two args, this actually makes this invalid.null,],],diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.cljindex f709288c49e..a8acc129380 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj@@ -314,7 +314,7 @@[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}[:type [:= :query]][:query_id :string]- [:query mbql.s/Query]+ [:query ::mbql.s/Query][:result_columns [:sequential ::column]]]]][:map[:output :string]]])diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/api/gtap.clj b/enterprise/backend/src/metabase_enterprise/sandbox/api/gtap.cljindex ff4b664c37c..d3475d9d3a8 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/api/gtap.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/api/gtap.clj@@ -1,7 +1,8 @@(ns metabase-enterprise.sandbox.api.gtap"`/api/mt/gtap` endpoints, for CRUD operations and the like on GTAPs (Group Table Access Policies)."(:require- [metabase-enterprise.sandbox.models.sandbox :as gtap]+ [metabase-enterprise.sandbox.models.sandbox :as sandbox]+ [metabase-enterprise.sandbox.schema :as sandbox.schema][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.api.open-api :as open-api]@@ -33,12 +34,6 @@;; TODO - not sure what other endpoints we might need, e.g. for fetching the list above but for a given group or Table-(def ^:private AttributeRemappings- :any- ;; TODO -- fix me- #_(mu/with-api-error-message [:maybe [:map-of ms/NonBlankString ms/NonBlankString]]- "value must be a valid attribute remappings map (attribute name -> remapped name)"))-(api.macros/defendpoint :post "/""Create a new GTAP."[_route-params@@ -47,7 +42,7 @@[:table_id ms/PositiveInt][:card_id {:optional true} [:maybe ms/PositiveInt]][:group_id ms/PositiveInt]- [:attribute_remappings {:optional true} AttributeRemappings]]]+ [:attribute_remappings {:optional true} ::sandbox.schema/attribute-remappings]]](first (t2/insert-returning-instances!:model/Sandbox(select-keys body [:table_id :card_id :group_id :attribute_remappings]))))@@ -61,7 +56,7 @@_query-paramsbody :- [:map[:card_id {:optional true} [:maybe ms/PositiveInt]]- [:attribute_remappings {:optional true} AttributeRemappings]]]+ [:attribute_remappings {:optional true} ::sandbox.schema/attribute-remappings]]](api/check-404 (t2/select-one :model/Sandbox :id id));; Only update `card_id` and/or `attribute_remappings` if the values are present in the body of the request.;; This allows existing values to be "cleared" by being set to nil@@ -88,8 +83,8 @@(throw (ex-info (tru "Sandboxing with a saved question is not enabled for this database."){:status-code 400:message (tru "Sandboxing with a saved question is not enabled for this database.")})))))- (gtap/check-columns-match-table {:table_id table_id- :card_id card_id}))+ (sandbox/check-columns-match-table {:table_id table_id+ :card_id card_id}))(api.macros/defendpoint :delete "/:id""Delete a GTAP entry."diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj b/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.cljindex 1349568f0ee..9684b04398d 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj@@ -7,11 +7,10 @@system."(:require[medley.core :as m]+ [metabase-enterprise.sandbox.schema :as sandbox.schema][metabase.api.common :as api][metabase.audit-app.core :as audit][metabase.events.core :as events]- ;; legacy usage -- do not use in new code- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.models.interface :as mi]@@ -37,14 +36,8 @@(derive ::mi/read-policy.superuser)(derive ::mi/write-policy.superuser))-(defn- normalize-attribute-remapping-targets [attribute-remappings]- (m/map-vals- mbql.normalize/normalize- attribute-remappings))-(t2/deftransforms :model/Sandbox- {:attribute_remappings {:in (comp mi/json-in normalize-attribute-remapping-targets)- :out (comp normalize-attribute-remapping-targets mi/json-out-without-keywordization)}})+ {:attribute_remappings sandbox.schema/attribute-remappings-transform})(defn table-field-names->cols"Return a mapping of field names to corresponding cols for given table."diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/schema.clj b/enterprise/backend/src/metabase_enterprise/sandbox/schema.cljnew file mode 100644index 00000000000..1e2d20c6bc4--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/sandbox/schema.clj@@ -0,0 +1,33 @@+(ns metabase-enterprise.sandbox.schema+ (:require+ [metabase.lib.core :as lib]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.parameter :as lib.schema.parameter]+ [metabase.models.interface :as mi]+ [metabase.util :as u]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms]))++(mr/def ::attribute-remappings+ "value must be a valid attribute remappings map (attribute name -> remapped name)"+ [:maybe+ [:map-of+ [:schema+ {:decode/normalize (fn [x]+ (cond-> x+ (keyword? x) u/qualified-name))}+ ms/NonBlankString]+ [:orn+ [:field-name ms/NonBlankString]+ [:field-id ::lib.schema.id/field]+ [:parameter-target [:ref ::lib.schema.parameter/target]]]]])++(defn normalize-attribute-remappings+ "Normalize the Sandbox `attribute_remappings` map."+ [attribute-remappings]+ (lib/normalize ::attribute-remappings attribute-remappings))++(def attribute-remappings-transform+ "Toucan transform spec for the Sandbox `attribute_remappings` column."+ {:in (comp mi/json-in normalize-attribute-remappings)+ :out (comp normalize-attribute-remappings mi/json-out-without-keywordization)})diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/models/sandbox_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/models/sandbox_test.cljindex 0309a0e7b4e..0ca9f743927 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/models/sandbox_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/models/sandbox_test.clj@@ -8,27 +8,6 @@[metabase.util :as u][toucan2.core :as t2]))-(deftest normalize-attribute-remappings-test- (testing "make sure attribute-remappings come back from the DB normalized the way we'd expect"- (mt/with-temp [:model/Sandbox gtap {:table_id (mt/id :venues)- :group_id (u/the-id (perms-group/all-users))- :attribute_remappings {"venue_id"- {:type "category"- :target ["variable" ["field" (mt/id :venues :id) nil]]- :value 5}}}]- (is (= {"venue_id" {:type :category- :target [:variable [:field (mt/id :venues :id) nil]]- :value 5}}- (t2/select-one-fn :attribute_remappings :model/Sandbox :id (u/the-id gtap)))))-- (testing (str "apparently sometimes they are saved with just the target, but not type or value? Make sure these "- "get normalized correctly.")- (mt/with-temp [:model/Sandbox gtap {:table_id (mt/id :venues)- :group_id (u/the-id (perms-group/all-users))- :attribute_remappings {"user" ["variable" ["field" (mt/id :venues :id) nil]]}}]- (is (= {"user" [:variable [:field (mt/id :venues :id) nil]]}- (t2/select-one-fn :attribute_remappings :model/Sandbox :id (u/the-id gtap))))))))-(deftest disallow-changing-table-id-test(testing "You can't change the table_id of a sandbox after it has been created."(mt/with-temp [:model/Sandbox gtap {:table_id (mt/id :venues)diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/schema_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/schema_test.cljnew file mode 100644index 00000000000..dfa05ddde0a--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/sandbox/schema_test.clj@@ -0,0 +1,25 @@+(ns metabase-enterprise.sandbox.schema-test+ (:require+ [clojure.test :refer :all]+ [metabase.permissions.models.permissions-group :as perms-group]+ [metabase.test :as mt]+ [metabase.util :as u]+ [toucan2.core :as t2]))++(deftest ^:parallel normalize-attribute-remappings-test+ (testing "make sure attribute-remappings come back from the DB normalized the way we'd expect"+ (mt/with-temp [:model/Sandbox gtap {:table_id (mt/id :venues)+ :group_id (u/the-id (perms-group/all-users))+ :attribute_remappings {"venue_id" ["variable" ["field" (mt/id :venues :id) nil]]}}]+ (is (= {"venue_id" [:variable [:field (mt/id :venues :id) nil]]}+ (t2/select-one-fn :attribute_remappings :model/Sandbox :id (u/the-id gtap)))))))++(deftest ^:parallel normalize-attribute-remappings-test-2+ (testing "make sure attribute-remappings come back from the DB normalized the way we'd expect"+ (testing (str "apparently sometimes they are saved with just the target, but not type or value? Make sure these "+ "get normalized correctly.")+ (mt/with-temp [:model/Sandbox gtap {:table_id (mt/id :venues)+ :group_id (u/the-id (perms-group/all-users))+ :attribute_remappings {"user" ["variable" ["field" (mt/id :venues :id) nil]]}}]+ (is (= {"user" [:variable [:field (mt/id :venues :id) nil]]}+ (t2/select-one-fn :attribute_remappings :model/Sandbox :id (u/the-id gtap))))))))diff --git a/frontend/src/metabase-types/api/mocks/presets/sample_cards.ts b/frontend/src/metabase-types/api/mocks/presets/sample_cards.tsindex fb5c44c5d10..c3d775519ac 100644--- a/frontend/src/metabase-types/api/mocks/presets/sample_cards.ts+++ b/frontend/src/metabase-types/api/mocks/presets/sample_cards.ts@@ -41,7 +41,6 @@ export const createAdHocNativeCard = (database: SAMPLE_DB_ID,native: {query: "select * from orders",- "template-tags": {},},},...opts,@@ -57,7 +56,6 @@ export const createEmptyAdHocNativeCard = (database: SAMPLE_DB_ID,native: {query: "",- "template-tags": {},},},...opts,@@ -91,7 +89,6 @@ export const createSavedNativeCard = (database: SAMPLE_DB_ID,native: {query: "select * from orders",- "template-tags": {},},},...opts,diff --git a/src/metabase/driver/common/parameters/dates.clj b/src/metabase/driver/common/parameters/dates.cljindex 550d6197e6c..47c70fc0457 100644--- a/src/metabase/driver/common/parameters/dates.clj+++ b/src/metabase/driver/common/parameters/dates.clj@@ -207,11 +207,11 @@(def ^:private all-date-string-decoders(concat relative-date-string-decoders absolute-date-string-decoders))-(mu/defn date-string->filter :- mbql.s/Filter+(mu/defn date-string->filter :- ::mbql.s/Filter"Takes a string description of a *date* (not datetime) range such as 'lastmonth' or '2016-07-15~2016-08-6', oran absolute date *or datetime* string, and returns a corresponding MBQL filter clause for a given field reference."[date-string :- :string- field :- [:or ::lib.schema.id/field mbql.s/FieldOrExpressionRef]]+ field :- [:or ::lib.schema.id/field ::mbql.s/FieldOrExpressionRef]]#_{:clj-kondo/ignore [:deprecated-var]}(or (#'qp.parameters.dates/execute-decoders all-date-string-decoders :filter (mbql.u/wrap-field-id-if-needed field) date-string)(throw (ex-info (tru "Don''t know how to parse date string {0}" (pr-str date-string))diff --git a/src/metabase/driver/common/parameters/values.clj b/src/metabase/driver/common/parameters/values.cljindex 52bef73a30a..2f2e6b4edee 100644--- a/src/metabase/driver/common/parameters/values.clj+++ b/src/metabase/driver/common/parameters/values.clj@@ -97,7 +97,7 @@Targeting template tags by ID is preferable (as of version 44) but targeting by name is supported for backwardscompatibility."- [tag :- mbql.s/TemplateTag]+ [tag :- ::mbql.s/TemplateTag](let [target-type (case (:type tag)(:dimension :temporal-unit) :dimension:variable)]@@ -121,7 +121,7 @@(mu/defn- tag-params"Return params from the provided `params` list targeting the provided `tag`."- [tag :- mbql.s/TemplateTag+ [tag :- ::mbql.s/TemplateTagparams :- [:maybe [:sequential ::lib.schema.parameter/parameter]]](let [tag-target? (tag-target-pred tag)](seq (for [param params@@ -142,7 +142,7 @@(mu/defn- field-filter-value"Get parameter value(s) for a Field filter. Returns map if there is a normal single value, or a vector of maps formultiple values."- [tag :- mbql.s/TemplateTag+ [tag :- ::mbql.s/TemplateTagparams :- [:maybe [:sequential ::lib.schema.parameter/parameter]]](let [matching-params (tag-params tag params)tag-opts (:options tag)@@ -206,7 +206,7 @@params/no-value)))(mu/defmethod parse-tag :dimension :- [:maybe FieldFilter]- [{:keys [dimension alias], :as tag} :- mbql.s/TemplateTag+ [{:keys [dimension alias], :as tag} :- ::mbql.s/TemplateTagparams :- [:maybe [:sequential ::lib.schema.parameter/parameter]]](params/map->FieldFilter{:field (let [field-id (dimension->field-id dimension)]@@ -217,7 +217,7 @@:alias alias}))(mu/defmethod parse-tag :card :- ReferencedCardQuery- [{:keys [card-id], :as tag} :- mbql.s/TemplateTag _params]+ [{:keys [card-id], :as tag} :- ::mbql.s/TemplateTag _params](when-not card-id(throw (ex-info (tru "Invalid :card parameter: missing `:card-id`"){:tag tag, :type qp.error-type/invalid-parameter})))@@ -248,7 +248,7 @@e))))))(mu/defmethod parse-tag :snippet :- ReferencedQuerySnippet- [{:keys [snippet-name snippet-id], :as tag} :- mbql.s/TemplateTag+ [{:keys [snippet-name snippet-id], :as tag} :- ::mbql.s/TemplateTag_params](let [snippet-id (or snippet-id(throw (ex-info (tru "Unable to resolve Snippet: missing `:snippet-id`")@@ -301,7 +301,7 @@(mu/defn- param-value-for-raw-value-tag"Get the value that should be used for a raw value (i.e., non-Field filter) template tag from `params`."- [tag :- mbql.s/TemplateTag+ [tag :- ::mbql.s/TemplateTagparams :- [:maybe [:sequential ::lib.schema.parameter/parameter]]](let [matching-param (when-let [matching-params (not-empty (tag-params tag params))];; double-check and make sure we didn't end up with multiple mappings or something crazy like that.@@ -441,7 +441,7 @@(mu/defn- value-for-tag :- ParsedParamValue"Given a map `tag` (a value in the `:template-tags` dictionary) return the corresponding value from the `params`sequence. The `value` is something that can be compiled to SQL via `->replacement-snippet-info`."- [tag :- mbql.s/TemplateTag+ [tag :- ::mbql.s/TemplateTagparams :- [:maybe [:sequential ::lib.schema.parameter/parameter]]](try(parse-value-for-type (:type tag) (parse-tag tag params))diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.cljindex 20937bfc5c8..4bbd24b3eb8 100644--- a/src/metabase/driver_api/core.clj+++ b/src/metabase/driver_api/core.clj@@ -117,15 +117,11 @@limit/absolute-max-resultslimit/determine-query-max-rowslogger/level-enabled?- mbql.s/Join- mbql.s/MBQLQuerymbql.u/aggregation-at-indexmbql.u/assoc-field-optionsmbql.u/desugar-filter-clausembql.u/expression-with-namembql.u/field-options- mbql.u/is-clause?- mbql.u/mbql-clause?mbql.u/negate-filter-clausembql.u/normalize-tokenmbql.u/query->max-rows-limit@@ -139,6 +135,8 @@mdb/specmetabase.driver-api.impl/cachedmetabase.driver-api.impl/dispatch-by-clause-name-or-class+ metabase.driver-api.impl/is-clause?+ metabase.driver-api.impl/mbql-clause?metabase.driver-api.impl/nest-expressionsmi/instance-of?premium-features/is-hosted?@@ -266,7 +264,7 @@(def mbql.schema.value"mbql.s/value"- mbql.s/value)+ ::mbql.s/value)(def mbql.schema.field"mbql.s/field"@@ -311,3 +309,11 @@(def qp.compile.query-with-compiled-query"Schema for the output of [[compile]]: `:metabase.query-processor.compile/query-with-compiled-query`"::qp.compile/query-with-compiled-query)++(def MBQLQuery+ "Schema for a legacy MBQL inner query."+ ::mbql.s/MBQLQuery)++(def Join+ "Schema for a legacy MBQL join."+ ::mbql.s/Join)diff --git a/src/metabase/driver_api/impl.clj b/src/metabase/driver_api/impl.cljindex e3b2b92bf7f..04b7a1e0432 100644--- a/src/metabase/driver_api/impl.clj+++ b/src/metabase/driver_api/impl.clj@@ -1,11 +1,9 @@(ns metabase.driver-api.impl{:clj-kondo/config'{:linters- {:discouraged-namespace {metabase.legacy-mbql.util {:level :off}- metabase.query-processor.store {:level :off}}+ {:discouraged-namespace {metabase.query-processor.store {:level :off}}:deprecated-namespace {:exclude [metabase.query-processor.store]}}}}(:require- [metabase.legacy-mbql.util :as mbql.u][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.query-processor.store :as qp.store]@@ -38,12 +36,31 @@~ks(fn [] ~@body))))+(defn mbql-clause?+ "True if `x` is an MBQL clause (a sequence with a keyword as its first arg)."+ [x]+ (and (sequential? x)+ (not (map-entry? x))+ (keyword? (first x))))++(defn is-clause?+ "If `x` is an MBQL clause, and an instance of clauses defined by keyword(s) `k-or-ks`?++ (is-clause? :count [:count 10]) ; -> true+ (is-clause? #{:+ :- :* :/} [:+ 10 20]) ; -> true"+ [k-or-ks x]+ (and+ (mbql-clause? x)+ (if (coll? k-or-ks)+ ((set k-or-ks) (first x))+ (= k-or-ks (first x)))))+(defn dispatch-by-clause-name-or-class"Dispatch function perfect for use with multimethods that dispatch off elements of an MBQL query. If `x` is an MBQLclause, dispatches off the clause name; otherwise dispatches off `x`'s class."([x](letfn [(clause-type [x]- (when (mbql.u/mbql-clause? x)+ (when (mbql-clause? x)(first x)))(mbql5-lib-type [x](when (map? x)diff --git a/src/metabase/indexed_entities/models/model_index.clj b/src/metabase/indexed_entities/models/model_index.cljindex 5fac45d8041..077b3862696 100644--- a/src/metabase/indexed_entities/models/model_index.clj+++ b/src/metabase/indexed_entities/models/model_index.clj@@ -52,13 +52,13 @@"Filter function for valid tuples for indexing: an id and a value."[[id v]] (and id v))-(mu/defn- fix-expression-refs :- mbql.s/FieldOrExpressionRef+(mu/defn- fix-expression-refs :- ::mbql.s/FieldOrExpressionRef"Convert expression ref into a field ref.Expression refs (`[:expression \"full-name\"]`) are how the _query_ refers to a custom column. But nested queriesdon't, (and shouldn't) care that those are expressions. They are just another field. The field type is always`:type/Text` enforced by the endpoint to create model indexes."- [field-ref :- mbql.s/FieldOrExpressionRef+ [field-ref :- ::mbql.s/FieldOrExpressionRefbase-type :- ::lib.schema.common/base-type](case (first field-ref):field field-refdiff --git a/src/metabase/legacy_mbql/normalize.cljc b/src/metabase/legacy_mbql/normalize.cljcindex 3340c061619..d002495c297 100644--- a/src/metabase/legacy_mbql/normalize.cljc+++ b/src/metabase/legacy_mbql/normalize.cljc@@ -1,1158 +1,33 @@(ns metabase.legacy-mbql.normalize- "Logic for taking any sort of weird MBQL query and normalizing it into a standardized, canonical form. You can think- of this like taking any 'valid' MBQL query and rewriting it as-if it was written in perfect up-to-date MBQL in the- latest version. There are four main things done here, done as four separate steps:-- #### NORMALIZING TOKENS-- Converting all identifiers to lower-case, lisp-case keywords. e.g. `{\"SOURCE_TABLE\" 10}` becomes `{:source-table- 10}`.-- #### CANONICALIZING THE QUERY-- Rewriting deprecated MBQL 95/98 syntax and other things that are still supported for backwards-compatibility in- canonical modern MBQL syntax. For example `{:breakout [:count 10]}` becomes `{:breakout [[:count [:field 10 nil]]]}`.-- #### WHOLE-QUERY TRANSFORMATIONS-- Transformations and cleanup of the query structure as a whole to fix inconsistencies. Whereas the canonicalization- phase operates on a lower-level, transforming invidual clauses, this phase focuses on transformations that affect- multiple clauses, such as removing duplicate references to Fields if they are specified in both the `:breakout` and- `:fields` clauses.-- This is not the only place that does such transformations; several pieces of QP middleware perform similar- individual transformations, such as `reconcile-breakout-and-order-by-bucketing`.-- #### REMOVING EMPTY CLAUSES-- Removing empty clauses like `{:aggregation nil}` or `{:breakout []}`.-- Token normalization occurs first, followed by canonicalization, followed by removing empty clauses."- (:refer-clojure :exclude [mapv every? some select-keys #?(:clj doseq) #?(:clj for)])+ "Normalize MBQL 1-4 to MBQL 4."(:require- [clojure.set :as set]- [clojure.string :as str]- [medley.core :as m]- [metabase.legacy-mbql.predicates :as mbql.preds]+ #?@(:clj+ ([potemkin :as p]))[metabase.legacy-mbql.schema :as mbql.s]- [metabase.legacy-mbql.util :as mbql.u]+ [metabase.legacy-mbql.schema.helpers :as helpers][metabase.lib.normalize :as lib.normalize]- [metabase.lib.schema.expression.temporal :as lib.schema.expression.temporal]- [metabase.lib.util.match :as lib.util.match]- [metabase.util :as u]- [metabase.util.i18n :as i18n]- [metabase.util.log :as log]- [metabase.util.malli :as mu]- [metabase.util.performance :as perf :refer [mapv every? some select-keys #?(:clj doseq) #?(:clj for)]]- [metabase.util.time :as u.time]))--(defn- mbql-clause?- "True if `x` is an MBQL clause (a sequence with a token as its first arg). (This is different from the implementation- in `mbql.u` because it also supports un-normalized clauses. You shouldn't need to use this outside of this- namespace.)"- [x]- (and (sequential? x)- (not (map-entry? x))- ((some-fn keyword? string?) (first x))))--(defn- maybe-normalize-token- "Normalize token `x`, but only if it's a keyword or string."- [x]- (if ((some-fn keyword? string?) x)- (mbql.u/normalize-token x)- x))--(defn is-clause?- "If `x` an MBQL clause, and an instance of clauses defined by keyword(s) `k-or-ks`?-- (is-clause? :count [:count 10]) ; -> true- (is-clause? #{:+ :- :* :/} [:+ 10 20]) ; -> true-- (This is different from the implementation in `mbql.u` because it also supports un-normalized clauses. You shouldn't- need to use this outside of this namespace.)"- [k-or-ks x]- (and- (mbql-clause? x)- (let [clause-name (maybe-normalize-token (first x))]- (if (coll? k-or-ks)- ((set k-or-ks) clause-name)- (= k-or-ks clause-name)))))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | NORMALIZE TOKENS |-;;; +----------------------------------------------------------------------------------------------------------------+--(declare normalize-tokens path->special-token-normalization-fn)--(defmulti ^:private normalize-mbql-clause-tokens- {:arglists '([mbql-clause])}- (comp maybe-normalize-token first))--(defmethod normalize-mbql-clause-tokens :aggregation- ;; nil options should be removed from aggregation references (`[:aggregation 0]`).- [[_ aggregation-index option]]- (cond-> [:aggregation aggregation-index]- (some? option) (conj option)))--(defn- normalize-ref-opts [opts]- (let [opts (normalize-tokens opts nil)]- (cond-> opts- (:base-type opts) (update :base-type keyword)- (:effective-type opts) (update :effective-type keyword)- (:temporal-unit opts) (update :temporal-unit keyword)- (:inherited-temporal-unit opts) (update :inherited-temporal-unit keyword)- (:binning opts) (update :binning (fn [binning]- (cond-> binning- (:strategy binning) (update :strategy keyword)))))))--(defmethod normalize-mbql-clause-tokens :expression- ;; For expression references (`[:expression \"my_expression\"]`) keep the arg as is but make sure it is a string.- [[_ expression-name opts]]- (let [expression [:expression (if (keyword? expression-name)- (u/qualified-name expression-name)- expression-name)]- opts (->> opts- normalize-ref-opts- not-empty)]- (cond-> expression- opts (conj opts))))--(defmethod normalize-mbql-clause-tokens :binning-strategy- ;; For `:binning-strategy` clauses (which wrap other Field clauses) normalize the strategy-name and recursively- ;; normalize the Field it bins.- [[_ field strategy-name strategy-param]]- (if strategy-param- (conj (normalize-mbql-clause-tokens [:binning-strategy field strategy-name]) strategy-param)- [:binning-strategy (normalize-tokens field nil) (maybe-normalize-token strategy-name)]))--(defmethod normalize-mbql-clause-tokens :field- [[_ id-or-name opts]]- [:field- id-or-name- (normalize-ref-opts opts)])--(defmethod normalize-mbql-clause-tokens :field-literal- ;; Similarly, for Field literals, keep the arg as-is, but make sure it is a string."- [[_ field-name field-type]]- [:field-literal- (if (keyword? field-name)- (u/qualified-name field-name)- field-name)- (keyword field-type)])--(defmethod normalize-mbql-clause-tokens :datetime-field- ;; Datetime fields look like `[:datetime-field <field> <unit>]` or `[:datetime-field <field> :as <unit>]`- ;; normalize the unit, and `:as` (if present) tokens, and the Field."- [[_ field as-or-unit maybe-unit]]- (if maybe-unit- [:datetime-field (normalize-tokens field nil) :as (maybe-normalize-token maybe-unit)]- [:datetime-field (normalize-tokens field nil) (maybe-normalize-token as-or-unit)]))--(defmethod normalize-mbql-clause-tokens :time-interval- ;; `time-interval`'s `unit` should get normalized, and `amount` if it's not an integer."- [[_ field amount unit options]]- (if options- (conj (normalize-mbql-clause-tokens [:time-interval field amount unit])- (normalize-tokens options nil))- [:time-interval- (normalize-tokens field nil)- (if (integer? amount)- amount- (maybe-normalize-token amount))- (maybe-normalize-token unit)]))--(defmethod normalize-mbql-clause-tokens :relative-time-interval- [[_ col & [_value _bucket _offset-value _offset-bucket :as args]]]- (into [:relative-time-interval (normalize-tokens col nil)]- (map maybe-normalize-token)- args))--(defmethod normalize-mbql-clause-tokens :relative-datetime- ;; Normalize a `relative-datetime` clause. `relative-datetime` comes in two flavors:- ;;- ;; [:relative-datetime :current]- ;; [:relative-datetime -10 :day] ; amount & unit"- [[_ amount unit]]- (if unit- [:relative-datetime amount (maybe-normalize-token unit)]- [:relative-datetime :current]))--(defmethod normalize-mbql-clause-tokens :interval- [[_ amount unit]]- [:interval amount (maybe-normalize-token unit)])--(defmethod normalize-mbql-clause-tokens :datetime-add- [[_ field amount unit]]- [:datetime-add (normalize-tokens field nil) amount (maybe-normalize-token unit)])--(defmethod normalize-mbql-clause-tokens :datetime-subtract- [[_ field amount unit]]- [:datetime-subtract (normalize-tokens field nil) amount (maybe-normalize-token unit)])--(defmethod normalize-mbql-clause-tokens :datetime- [[_ field options]]- (if (empty? options)- [:datetime (normalize-tokens field nil)]- [:datetime (normalize-tokens field nil)- (let [options (normalize-tokens options nil)]- (cond-> options- (contains? options :mode)- (update :mode lib.schema.expression.temporal/normalize-datetime-mode)))]))--(defmethod normalize-mbql-clause-tokens :get-week- [[_ field mode]]- (if mode- [:get-week (normalize-tokens field nil) (maybe-normalize-token mode)]- [:get-week (normalize-tokens field nil)]))--(defmethod normalize-mbql-clause-tokens :get-day-of-week- [[_ field mode]]- (if mode- [:get-day-of-week (normalize-tokens field nil) (maybe-normalize-token mode)]- [:get-day-of-week (normalize-tokens field nil)]))--(defmethod normalize-mbql-clause-tokens :temporal-extract- [[_ field unit mode]]- (if mode- [:temporal-extract (normalize-tokens field nil) (maybe-normalize-token unit) (maybe-normalize-token mode)]- [:temporal-extract (normalize-tokens field nil) (maybe-normalize-token unit)]))--(defmethod normalize-mbql-clause-tokens :datetime-diff- [[_ x y unit]]- [:datetime-diff- (normalize-tokens x nil)- (normalize-tokens y nil)- (maybe-normalize-token unit)])--(defmethod normalize-mbql-clause-tokens :during- [[_ field value unit]]- [:during (normalize-tokens field nil) value (maybe-normalize-token unit)])--(defn- normalize-value-opts- [opts]- (when opts- (-> opts- (perf/update-keys (fn [k]- (keyword (u/->snake_case_en k))))- (m/update-existing :base_type keyword)- (m/update-existing :effective_type keyword)- (m/update-existing :semantic_type keyword)- (m/update-existing :unit keyword))))--(defmethod normalize-mbql-clause-tokens :value- ;; The args of a `value` clause shouldn't be normalized.- ;; See https://github.com/metabase/metabase/issues/23354 for details- [[_ value info]]- [:value value (normalize-value-opts info)])--(defmethod normalize-mbql-clause-tokens :offset- [[_tag opts expr n, :as clause]]- (when (= (count clause) 4)- (let [opts (lib.normalize/normalize :metabase.lib.schema.common/options (or opts {}))]- [:offset opts (normalize-tokens expr nil) n])))--(defmethod normalize-mbql-clause-tokens :default- ;; MBQL clauses by default are recursively normalized.- ;; This includes the clause name (e.g. `[\"COUNT\" ...]` becomes `[:count ...]`) and args.- [[clause-name & args]]- (into [(maybe-normalize-token clause-name)] (map #(normalize-tokens % nil)) args))--(defn- aggregation-subclause?- [x]- (or (when ((some-fn keyword? string?) x)- (#{:avg :count :cum-count :distinct :distinct-where :stddev :sum :min :max :+ :- :/ :*- :sum-where :count-where :share :var :median :percentile}- (maybe-normalize-token x)))- (when (mbql-clause? x)- (aggregation-subclause? (first x)))))--(defn- normalize-ag-clause-tokens- "For old-style aggregations like `{:aggregation :count}` make sure we normalize the ag type (`:count`). Other wacky- clauses like `{:aggregation [:count :count]}` need to be handled as well :("- [ag-clause]- (cond- ;; something like {:aggregations :count}- ((some-fn keyword? string?) ag-clause)- (maybe-normalize-token ag-clause)-- ;; named aggregation ([:named <ag> <name>])- (is-clause? :named ag-clause)- (let [[_ wrapped-ag & more] ag-clause]- (into [:named (normalize-ag-clause-tokens wrapped-ag)] more))-- ;; something wack like {:aggregations [:count [:sum 10]]} or {:aggregations [:count :count]}- (when (mbql-clause? ag-clause)- (aggregation-subclause? (second ag-clause)))- (mapv normalize-ag-clause-tokens ag-clause)-- :else- (normalize-tokens ag-clause nil)))--(defn- normalize-expressions-tokens- "For expressions, we don't want to normalize the name of the expression; keep that as is, and make it a string;- normalize the definitions as normal."- [expressions-clause]- (into {} (for [[expression-name definition] expressions-clause]- [(u/qualified-name expression-name)- (normalize-tokens definition nil)])))--(defn- normalize-order-by-tokens- "Normalize tokens in the order-by clause, which can have different syntax when using MBQL 95 or 98- rules (`[<field> :asc]` vs `[:asc <field>]`, for example)."- [clauses]- (vec (for [subclause clauses]- (if (mbql-clause? subclause)- ;; MBQL 98+ [direction field] style: normalize as normal- (normalize-mbql-clause-tokens subclause)- ;; otherwise it's MBQL 95 [field direction] style: flip the args and *then* normalize the clause. And then- ;; flip it back to put it back the way we found it.- (reverse (normalize-mbql-clause-tokens (reverse subclause)))))))--(defn- template-tag-definition-key->transform-fn- "Get the function that should be used to transform values for normalized key `k` in a template tag definition."- [k]- (get {:default identity- :type maybe-normalize-token- :widget-type maybe-normalize-token}- k- ;; if there's not a special transform function for the key in the map above, just wrap the key-value- ;; pair in a dummy map and let [[normalize-tokens]] take care of it. Then unwrap- (fn [v]- (-> (normalize-tokens {k v} nil)- (get k)))))--(defn- normalize-template-tag-definition- "For a template tag definition, normalize all the keys appropriately."- [tag-definition]- (let [tag-def (into- {}- (map (fn [[k v]]- (let [k (maybe-normalize-token k)- transform-fn (template-tag-definition-key->transform-fn k)]- [k (transform-fn v)])))- tag-definition)]- ;; `:widget-type` is a required key for Field Filter (dimension) template tags -- see- ;; [[metabase.legacy-mbql.schema/TemplateTag:FieldFilter]] -- but prior to v42 it wasn't usually included by the- ;; frontend. See #20643. If it's not present, just add in `:category` which will make things work they way they- ;; did in the past.- (cond-> tag-def- (and (= (:type tag-def) :dimension)- (not (:widget-type tag-def)))- (assoc :widget-type :category))))--(defn- normalize-template-tags- "Normalize native-query template tags. Like `expressions` we want to preserve the original name rather than normalize- it."- [template-tags]- (into- {}- (map (fn [[tag-name tag-definition]]- (let [tag-name (u/qualified-name tag-name)]- [tag-name- (-> (normalize-template-tag-definition tag-definition)- (assoc :name tag-name))])))- template-tags))--(defn normalize-query-parameter- "Normalize a parameter in the query `:parameters` list."- [{param-type :type, :keys [target id values_source_config], :as param}]- (cond-> param- id (update :id u/qualified-name)- ;; some things that get ran thru here, like dashcard param targets, do not have :type- param-type (update :type maybe-normalize-token)- target (update :target #(normalize-tokens % nil))- values_source_config (update-in [:values_source_config :label_field] #(normalize-tokens % nil))- values_source_config (update-in [:values_source_config :value_field] #(normalize-tokens % nil))))--(defn- normalize-source-query [source-query]- (let [{native? :native, :as source-query} (perf/update-keys source-query maybe-normalize-token)]- (if native?- (-> source-query- (set/rename-keys {:native :query})- (normalize-tokens (:native path->special-token-normalization-fn))- (set/rename-keys {:query :native}))- (normalize-tokens source-query (:query path->special-token-normalization-fn)))))--(defn- normalize-join [join]- ;; path in call to `normalize-tokens` is [:query] so it will normalize `:source-query` as appropriate- (let [{:keys [strategy fields], join-alias :alias, :as join}- (normalize-tokens join (:query path->special-token-normalization-fn))]- (cond-> join- strategy- (update :strategy maybe-normalize-token)-- ((some-fn keyword? string?) fields)- (update :fields maybe-normalize-token)-- join-alias- (update :alias u/qualified-name))))--(declare canonicalize-mbql-clauses)--(defn normalize-field-ref- "Normalize the field ref. Ensure it's well-formed mbql, not just json."- {:deprecated "0.57.0"}- [clause]- (-> clause normalize-tokens canonicalize-mbql-clauses))--(mu/defn- normalize-source-metadata- "Normalize source/results metadata for a single column.-- DEPRECATED: Use [[metabase.lib-be.core/instance->metadata]]- or [[metabase.lib.core/->normalized-stage-metadata]] (for Lib-style metadata) or [[metabase.lib.core/normalize]]- with `:metabase.query-processor.schema/result-metadata.column` (for QP-style metadata) going forward."- [metadata :- :map]- {:pre [(map? metadata)]}- (lib.normalize/normalize ::mbql.s/legacy-column-metadata metadata))--(mu/defn- normalize-native-query :- [:maybe :map]- "For native queries, normalize the top-level keys, and template tags, but nothing else."- [native-query :- [:maybe :map]]- (let [native-query (perf/update-keys native-query maybe-normalize-token)]- (cond-> native-query- (seq (:template-tags native-query)) (update :template-tags normalize-template-tags))))--(defn- normalize-actions-row [row]-- (cond-> row- (map? row) (perf/update-keys u/qualified-name)))--(def ^:private path->special-token-normalization-fn- "Map of special functions that should be used to perform token normalization for a given path. For example, the- `:expressions` key in an MBQL query should preserve the case of the expression names; this custom behavior is- defined below."- {:type maybe-normalize-token- ;; don't normalize native queries- :native normalize-native-query- :query {:aggregation normalize-ag-clause-tokens- :expressions normalize-expressions-tokens- :order-by normalize-order-by-tokens- :source-query normalize-source-query- :source-metadata {::sequence normalize-source-metadata}- :joins {::sequence normalize-join}}- ;; we smuggle metadata for Models and want to preserve their "database" form vs a normalized form so it matches- ;; the style in annotate.clj- :info {:metadata/model-metadata identity- ;; the original query that runs through qp.pivot should be ignored here entirely- :pivot/original-query (fn [_] nil)- :pivot/result-metadata identity- ;; don't try to normalize the keys in viz-settings passed in as part of `:info`.- :visualization-settings identity- :context maybe-normalize-token}- :parameters {::sequence normalize-query-parameter}- ;; TODO -- when does query ever have a top-level `:context` key??- :context #(some-> % maybe-normalize-token)- :source-metadata {::sequence normalize-source-metadata}- :viz-settings maybe-normalize-token ; TODO (Cam 8/8/25) -- we also have [[metabase.models.interface/normalize-visualization-settings]]- :create-row normalize-actions-row- :update-row normalize-actions-row- ;;- ;; HACK TODO (Cam 7/17/25) -- seems icky for the legacy MBQL schema to have to know about namespaced keys like- ;; this. I guess this can go away once we stop converting back and forth between MBQL 4 and 5 inside the QP- :metabase-enterprise.sandbox.query-processor.middleware.sandboxing/original-metadata- identity})--(defn normalize-tokens- "Recursively normalize tokens in `x`.-- Every time this function recurses (thru a map value) it adds a new (normalized) key to key path, e.g. `path` will be- `[:query :order-by]` when we're in the MBQL order-by clause. If we need to handle these top-level clauses in special- ways add a function to `path->special-token-normalization-fn` above.-- In some cases, dealing with the path isn't desirable, but we don't want to accidentally trigger normalization- functions (such as accidentally normalizing the `:type` key in something other than the top-level of the query), so- by convention please pass `nil` to avoid accidentally triggering special path functions.-- DEPRECATED: Use [[metabase.lib.core/normalize]] to normalize things going forward."- {:deprecated "0.57.0"}- ([x] (normalize-tokens x path->special-token-normalization-fn))- ([x special-fns]- (let [special-fn (when (and special-fns (fn? special-fns))- special-fns)]- (try- (cond- special-fn- (special-fn x)-- ;; Skip record types because this query is an `expanded` query, which is not going to play nice here. Hopefully we- ;; can remove expanded queries entirely soon.- (record? x)- x-- ;; maps should just get the keys normalized and then recursively call normalize-tokens on the values.- ;; Each recursive call peels away one level from special-fns map so we can handle top-level clauses in a special way if needed- (map? x)- (reduce-kv (fn [m k v]- (let [k' (maybe-normalize-token k)- v' (normalize-tokens v (get special-fns k'))]- (if (identical? k k')- (if (identical? v v')- m- (assoc m k v'))- (-> m (dissoc k) (assoc k' v')))))- x x)-- ;; MBQL clauses handled above because of special cases- (mbql-clause? x)- (normalize-mbql-clause-tokens x)-- ;; for non-mbql sequential collections (probably something like the subclauses of :order-by or something like- ;; that) recurse on all the args.- ;;- ;; To signify that we're recursing into a sequential collection, this peels away `::sequence` from special-fns- (sequential? x)- (let [special-fns (::sequence special-fns)]- (mapv #(normalize-tokens % special-fns) x))-- :else- x)- (catch #?(:clj Throwable :cljs js/Error) e- (throw (ex-info (i18n/tru "Error normalizing form: {0}" (ex-message e))- {:form x, :special-fn special-fn}- e)))))))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | CANONICALIZE |-;;; +----------------------------------------------------------------------------------------------------------------+--(defn- wrap-implicit-field-id- "Wrap raw integer Field IDs (i.e., those in an implicit 'field' position) in a `:field` clause if they're not- already. Done for MBQL 95 backwards-compatibility. e.g.:-- {:filter [:= 10 20]} ; -> {:filter [:= [:field 10 nil] 20]}"- [field]- (if (integer? field)- [:field field nil]- field))--(defmulti ^:private canonicalize-mbql-clause- {:arglists '([clause])}- (fn [clause]- (when (mbql-clause? clause)- (first clause))))--(defmethod canonicalize-mbql-clause :default- [clause]- clause)--(defn- canonicalize-implicit-field-id- "If `clause` is a raw integer ID wrap it in a `:field` clause. Either way, canonicalize the resulting clause."- [clause]- (canonicalize-mbql-clause (wrap-implicit-field-id clause)))--(defmethod canonicalize-mbql-clause :field- [[_ id-or-name opts, :as clause]]- (if-not ((some-fn map? nil?) opts)- ;; this is an MBQL 5+ clause, ignore it.- clause- (if (is-clause? :field id-or-name)- (let [[_ nested-id-or-name nested-opts] id-or-name]- (canonicalize-mbql-clause [:field nested-id-or-name (not-empty (merge nested-opts opts))]))- ;; remove empty stuff from the options map. The `remove-empty-clauses` step will further remove empty stuff- ;; afterwards- [:field id-or-name (not-empty opts)])))--(defmethod canonicalize-mbql-clause :aggregation- [[_tag index opts]]- (if (empty? opts)- [:aggregation index]- [:aggregation index opts]))--;;; legacy Field clauses--(defmethod canonicalize-mbql-clause :field-id- [[_ id]]- ;; if someone is dumb and does something like [:field-id [:field-literal ...]] be nice and fix it for them.- (if (mbql-clause? id)- (canonicalize-mbql-clause id)- [:field id nil]))--(defmethod canonicalize-mbql-clause :field-literal- [[_ field-name base-type]]- [:field field-name {:base-type base-type}])--(defmethod canonicalize-mbql-clause :fk->- [[_ field-1 field-2]]- (let [[_ source _] (canonicalize-implicit-field-id field-1)- [_ dest dest-opts] (canonicalize-implicit-field-id field-2)]- [:field dest (assoc dest-opts :source-field source)]))+ [metabase.util.malli :as mu]))-(defmethod canonicalize-mbql-clause :joined-field- [[_ join-alias field]]- (-> (canonicalize-implicit-field-id field)- (mbql.u/assoc-field-options :join-alias join-alias)))+;; this is only for the sake of a few random namespaces that use this still+#?(:clj+ (p/import-vars [helpers is-clause?]))-(defmethod canonicalize-mbql-clause :datetime-field- [clause]- (case (count clause)- 3- (let [[_ field unit] clause]- (-> (canonicalize-implicit-field-id field)- #_{:clj-kondo/ignore [:deprecated-var]}- (mbql.u/with-temporal-unit unit)))-- 4- (let [[_ field _ unit] clause]- (canonicalize-mbql-clause [:datetime-field field unit]))))--(defmethod canonicalize-mbql-clause :binning-strategy- [[_ field strategy param binning-options]]- (let [[_ id-or-name opts] (canonicalize-implicit-field-id field)]- [:field- id-or-name- (assoc opts :binning (merge {:strategy strategy}- (when param- {strategy param})- binning-options))]))--;;; filter clauses--;; For `and`/`or`/`not` compound filters, recurse on the arg(s), then simplify the whole thing.-(defn- canonicalize-compound-filter-clause [[filter-name & args]]- #_{:clj-kondo/ignore [:deprecated-var]}- (mbql.u/simplify-compound-filter- (into [filter-name]- ;; we need to canonicalize any other mbql clauses that might show up in args here because- ;; simplify-compund-filter validates its output :(- (map canonicalize-mbql-clause args))))--(doseq [clause-name [:and :or :not]]- (defmethod canonicalize-mbql-clause clause-name- [clause]- (canonicalize-compound-filter-clause clause)))--(defmethod canonicalize-mbql-clause :inside- [[_ field-1 field-2 & coordinates]]- (into [:inside- (canonicalize-implicit-field-id field-1)- (canonicalize-implicit-field-id field-2)]- coordinates))--(defmethod canonicalize-mbql-clause :time-interval- [[_ field & args]]- ;; if you specify a `:temporal-unit` for the Field inside a `:time-interval`, remove it. The unit in- ;; `:time-interval` takes precedence.- (let [field (cond-> (canonicalize-implicit-field-id field)- (mbql.u/is-clause? :field field) (mbql.u/update-field-options dissoc :temporal-unit))]- (into [:time-interval field] args)))--;; all the other filter types have an implict field ID for the first arg-;; (e.g. [:= 10 20] gets canonicalized to [:= [:field-id 10] 20]-(defn- canonicalize-simple-filter-clause- [[filter-name first-arg & other-args]]- ;; Support legacy expressions like [:> 1 25] where 1 is a field id.- (into [filter-name (canonicalize-implicit-field-id first-arg)]- (map canonicalize-mbql-clause other-args)))--(doseq [clause-name [:= :!= :< :<= :> :>=- :in :not-in- :is-empty :not-empty :is-null :not-null- :between]]- (defmethod canonicalize-mbql-clause clause-name- [clause]- (canonicalize-simple-filter-clause clause)))--;; These clauses have pMBQL-style options in index 1, when they have multiple arguments.-(doseq [tag [:starts-with :ends-with :contains :does-not-contain]]- (defmethod canonicalize-mbql-clause tag- [[_tag opts & args :as clause]]- (if (> (count args) 2)- (into [tag (or opts {})] (map canonicalize-mbql-clause args))- (canonicalize-simple-filter-clause clause))))--;;; aggregations/expression subclauses--;; Remove `:rows` type aggregation (long-since deprecated; simpliy means no aggregation) if present-(defmethod canonicalize-mbql-clause :rows- [_]- nil)--;; TODO -- if options is empty, should we just unwrap the clause?-(defmethod canonicalize-mbql-clause :aggregation-options- [[_ wrapped-aggregation-clause options]]- [:aggregation-options (canonicalize-mbql-clause wrapped-aggregation-clause) options])--;; for legacy `:named` aggregations convert them to a new-style `:aggregation-options` clause.-;;-;; 99.99% of clauses should have no options, however if they do and `:use-as-display-name?` is false (default is-;; true) then generate options to change `:name` rather than `:display-name`-(defmethod canonicalize-mbql-clause :named- [[_ wrapped-ag expr-name & more]]- (canonicalize-mbql-clause- [:aggregation-options- (canonicalize-mbql-clause wrapped-ag)- (let [[{:keys [use-as-display-name?]}] more]- (if (false? use-as-display-name?)- {:name expr-name}- {:display-name expr-name}))]))--(defn- canonicalize-count-clause [[clause-name field]]- (if field- [clause-name (canonicalize-implicit-field-id field)]- [clause-name]))--(doseq [clause-name [:count :cum-count]]- (defmethod canonicalize-mbql-clause clause-name- [clause]- (canonicalize-count-clause clause)))--(defn- canonicalize-simple-aggregation-with-field- [[clause-name field]]- [clause-name (canonicalize-implicit-field-id field)])--(doseq [clause-name [:avg :cum-sum :distinct :stddev :sum :min :max :median :var]]- (defmethod canonicalize-mbql-clause clause-name- [clause]- (canonicalize-simple-aggregation-with-field clause)))--(defmethod canonicalize-mbql-clause :percentile- [[_ field percentile]]- [:percentile (canonicalize-implicit-field-id field) percentile])--(defn- canonicalize-filtered-aggregation-clause- [[clause-name filter-subclause]]- [clause-name (canonicalize-mbql-clause filter-subclause)])--(doseq [clause-name [:share :count-where]]- (defmethod canonicalize-mbql-clause clause-name- [clause]- (canonicalize-filtered-aggregation-clause clause)))--(defmethod canonicalize-mbql-clause :sum-where- [[_ field filter-subclause]]- [:sum-where (canonicalize-mbql-clause field) (canonicalize-mbql-clause filter-subclause)])--(doseq [tag [:case :if]]- (defmethod canonicalize-mbql-clause tag- [[_ clauses options]]- (if options- (conj (canonicalize-mbql-clause [tag clauses])- (normalize-tokens options nil))- [tag (vec (for [[pred expr] clauses]- [(canonicalize-mbql-clause pred) (canonicalize-mbql-clause expr)]))])))--(defmethod canonicalize-mbql-clause :substring- [[_ arg start & more]]- (into [:substring- (canonicalize-mbql-clause arg)- ;; 0 indexes were allowed in the past but we are now enforcing this rule in MBQL.- ;; This allows stored queries with literal 0 in the index to work.- (if (= 0 start) 1 (canonicalize-mbql-clause start))]- (map canonicalize-mbql-clause more)))--(defmethod canonicalize-mbql-clause :offset- [[_tag opts expr n, :as clause]]- {:pre [(= (count clause) 4)]}- [:offset (or opts {}) (canonicalize-mbql-clause expr) n])--;;; top-level key canonicalization--(defn- canonicalize-mbql-clauses- "Walk an `mbql-query` an canonicalize non-top-level clauses like `:fk->`."- [form]- (cond- ;; Special handling for records so that they are not converted into plain maps.- ;; Only the values are canonicalized.- (record? form)- (reduce-kv (fn [r k x] (assoc r k (canonicalize-mbql-clauses x))) form form)-- ;; Only the values are canonicalized.- (map? form)- (update-vals form canonicalize-mbql-clauses)-- (mbql-clause? form)- (let [top-canonical- (try- (canonicalize-mbql-clause form)- (catch #?(:clj Throwable :cljs js/Error) e- (log/error "Invalid clause:" form)- (throw (ex-info (i18n/tru "Invalid MBQL clause: {0}" (ex-message e))- {:clause form}- e))))]- ;; Canonical clauses are assumed to be sequential things conj'd at the end.- ;; In fact, they should better be vectors.- (if (seq top-canonical)- (into (conj (empty top-canonical) (first top-canonical))- (map canonicalize-mbql-clauses)- (rest top-canonical))- top-canonical))-- ;; ISeq instances (e.g., list and lazy sequences) are converted to vectors.- (seq? form)- (mapv canonicalize-mbql-clauses form)-- ;; Other collections (e.g., vectors, sets, and queues) are assumed to be conj'd at the end- ;; and we keep their types.- (coll? form)- (into (empty form) (map canonicalize-mbql-clauses) form)-- :else- form))--(defn- wrap-single-aggregations- "Convert old MBQL 95 single-aggregations like `{:aggregation :count}` or `{:aggregation [:count]}` to MBQL 98+- multiple-aggregation syntax (e.g. `{:aggregation [[:count]]}`)."- [aggregations]- (lib.util.match/replace aggregations- seq? (recur (vec &match))-- ;; something like {:aggregations :count} -- MBQL 95 single aggregation- keyword?- [[&match]]-- ;; special-case: MBQL 98 multiple aggregations using unwrapped :count or :rows- ;; e.g. {:aggregations [:count [:sum 10]]} or {:aggregations [:count :count]}- [(_ :guard (every-pred keyword? (complement #{:named :+ :- :* :/})))- (_ :guard aggregation-subclause?)- & _]- (into [] (mapcat wrap-single-aggregations) aggregations)-- ;; something like {:aggregations [:sum 10]} -- MBQL 95 single aggregation- [(_ :guard keyword?) & _]- [&match]-- _- &match))--(defn- canonicalize-aggregations- "Canonicalize subclauses (see above) and make sure `:aggregation` is a sequence of clauses instead of a single- clause."- [aggregations]- (->> (wrap-single-aggregations aggregations)- (keep canonicalize-mbql-clauses)- vec))--(defn- canonicalize-breakouts [breakouts]- (if (mbql-clause? breakouts)- (recur [breakouts])- (not-empty (mapv wrap-implicit-field-id breakouts))))--(defn- canonicalize-order-by- "Make sure order by clauses like `[:asc 10]` get `:field-id` added where appropriate, e.g. `[:asc [:field-id 10]]`"- [clauses]- (lib.util.match/replace clauses- seq? (recur (vec &match))-- ;; MBQL 95 reversed [<field> <direction>] clause- [field :asc] (recur [:asc field])- [field :desc] (recur [:desc field])- [field :ascending] (recur [:asc field])- [field :descending] (recur [:desc field])-- ;; MBQL 95 names but MBQL 98+ reversed syntax- [:ascending field] (recur [:asc field])- [:descending field] (recur [:desc field])-- [:asc field] [:asc (canonicalize-implicit-field-id field)]- [:desc field] [:desc (canonicalize-implicit-field-id field)]-- ;; this case should be the first one hit when we come in with a vector of clauses e.g. [[:asc 1] [:desc 2]]- [& clauses] (vec (distinct (map canonicalize-order-by clauses)))))--(declare canonicalize-inner-mbql-query)--(defn- canonicalize-template-tag [{:keys [dimension], :as tag}]- (cond-> tag- dimension (update :dimension canonicalize-mbql-clause)))--(defn- canonicalize-template-tags [tags]- (into {} (for [[tag-name tag] tags]- [tag-name (canonicalize-template-tag tag)])))--(defn- canonicalize-native-query [{:keys [template-tags], :as native-query}]- (cond-> native-query- template-tags (update :template-tags canonicalize-template-tags)))--(defn- canonicalize-source-query [{native? :native, :as source-query}]- (cond-> source-query- (not native?) canonicalize-inner-mbql-query- native? canonicalize-native-query))--(defn- non-empty? [x]- (if (coll? x)- (seq x)- (some? x)))--(defn- canonicalize-top-level-mbql-clauses- "Perform specific steps to canonicalize the various top-level clauses in an MBQL query."- [mbql-query]- (cond-> mbql-query- (non-empty? (:aggregation mbql-query)) (update :aggregation canonicalize-aggregations)- (non-empty? (:breakout mbql-query)) (update :breakout canonicalize-breakouts)- (non-empty? (:fields mbql-query)) (update :fields (partial mapv wrap-implicit-field-id))- (non-empty? (:order-by mbql-query)) (update :order-by canonicalize-order-by)- (non-empty? (:source-query mbql-query)) (update :source-query canonicalize-source-query)))--(def ^:private ^{:arglists '([query])} canonicalize-inner-mbql-query- (comp canonicalize-mbql-clauses canonicalize-top-level-mbql-clauses))--(defn- move-source-metadata-to-mbql-query- "In Metabase 0.33.0 `:source-metadata` about resolved queries is added to the 'inner' MBQL query rather than to the- top-level; if we encounter the old style, move it to the appropriate location."- [{:keys [source-metadata], :as query}]- (-> query- (dissoc :source-metadata)- (assoc-in [:query :source-metadata] source-metadata)))--(defn- canonicalize-mbql-clauses-excluding-native- [{:keys [native] :as outer-query}]- (if native- (-> outer-query (dissoc :native) canonicalize-mbql-clauses (assoc :native native))- (canonicalize-mbql-clauses outer-query)))--(defn- canonicalize- "Canonicalize a query [MBQL query], rewriting the query as if you perfectly followed the recommended style guides for- writing MBQL. Does things like removes unneeded and empty clauses, converts older MBQL '95 syntax to MBQL '98, etc."- [{:keys [query parameters source-metadata native], :as outer-query}]- (try- (cond-> outer-query- source-metadata move-source-metadata-to-mbql-query- query (update :query canonicalize-inner-mbql-query)- parameters (update :parameters (partial mapv canonicalize-mbql-clauses))- native (update :native canonicalize-native-query)- true canonicalize-mbql-clauses-excluding-native)- (catch #?(:clj Throwable :cljs js/Error) e- (throw (ex-info (i18n/tru "Error canonicalizing query: {0}" (ex-message e))- {:query query}- e)))))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | WHOLE-QUERY TRANSFORMATIONS |-;;; +----------------------------------------------------------------------------------------------------------------+--(defn- remove-breakout-fields-from-fields- "Remove any Fields specified in both `:breakout` and `:fields` from `:fields`; it is implied that any breakout Field- will be returned, specifying it in both would imply it is to be returned twice, which tends to cause confusion for- the QP and drivers. (This is done to work around historic bugs with the way queries were generated on the frontend;- I'm not sure this behavior makes sense, but removing it would break existing queries.)-- We will remove either exact matches:-- {:breakout [[:field-id 10]], :fields [[:field-id 10]]} ; -> {:breakout [[:field-id 10]]}-- or unbucketed matches:-- {:breakout [[:datetime-field [:field-id 10] :month]], :fields [[:field-id 10]]}- ;; -> {:breakout [[:field-id 10]]}"- [{{:keys [breakout fields]} :query, :as query}]- (if-not (and (seq breakout) (seq fields))- query- ;; get a set of all Field clauses (of any type) in the breakout. For temporal-bucketed fields, we'll include both- ;; the bucketed `[:datetime-field <field> ...]` clause and the `<field>` clause it wraps- (let [breakout-fields (into #{} cat (lib.util.match/match breakout- [:field id-or-name opts]- [&match- [:field id-or-name (dissoc opts :temporal-unit)]]))]- ;; now remove all the Fields in `:fields` that match the ones in the set- (update-in query [:query :fields] (comp vec (partial remove breakout-fields))))))--(defn- perform-whole-query-transformations- "Perform transformations that operate on the query as a whole, making sure the structure as a whole is logical and- consistent."- [query]- (try- (remove-breakout-fields-from-fields query)- (catch #?(:clj Throwable :cljs js/Error) e- (throw (ex-info (i18n/tru "Error performing whole query transformations")- {:query query}- e)))))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | LEGACY FILTER CLAUSES |-;;; +----------------------------------------------------------------------------------------------------------------+--(defn- temporal-unit- [field]- (-> field mbql.u/field-options :temporal-unit))--(defn- temporal-unit-is? [units]- #(units (temporal-unit %)))--(defn- remove-temporal-unit- [field]- (mbql.u/update-field-options field dissoc :temporal-unit :original-temporal-unit))--(mu/defn ^:private replace-relative-date-filters :- mbql.s/Filter- "Replaces broken relative date filter clauses with `:relative-time-interval` calls.-- Previously we generated a complex expression for relative date filters with an offset on the FE. It turned out that- the expression was wrong by 1 offset unit, e.g. if the offset was by months, it was wrong by 1 month. To fix the issue- we introduced a new `:relative-time-interval` function that served several purposes. It captured the user intent- clearly while hiding the implementation details; it also fixed the underlying expression. Here we match the old- expression and convert it to a `:relative-time-interval` call, honoring the original user intent. See #46211 and- #46438 for details."- [filter-clause :- mbql.s/Filter]- (lib.util.match/replace filter-clause- [:between- [:+- field- [:interval (offset-value :guard integer?) (offset-unit :guard keyword?)]]- [:relative-datetime- (start-value :guard integer?)- (start-unit :guard keyword?)]- [:relative-datetime- (end-value :guard integer?)- (end-unit :guard keyword?)]]- (let [offset-value (- offset-value)]- (if (and (= start-unit end-unit)- (or (and (pos? offset-value) (zero? start-value) (pos? end-value))- (and (neg? offset-value) (neg? start-value) (zero? end-value))))- [:relative-time-interval- field- (if (neg? offset-value) start-value end-value)- start-unit- offset-value- offset-unit]- &match))))--(mu/defn ^:private replace-exclude-date-filters :- mbql.s/Filter- "Replaces legacy exclude date filter clauses that rely on temporal bucketing with `:temporal-extract` function calls."- [filter-clause :- mbql.s/Filter]- (lib.util.match/replace filter-clause- [:!=- (field :guard (every-pred mbql.preds/Field? (temporal-unit-is? #{:hour-of-day})))- & (args :guard #(every? number? %))]- (into [:!= [:get-hour (remove-temporal-unit field)]] args)-- [:!=- (field :guard (every-pred mbql.preds/Field? (temporal-unit-is? #{:day-of-week :month-of-year :quarter-of-year})))- & (args :guard #(every? u.time/timestamp-coercible? %))]- (let [args (mapv u.time/coerce-to-timestamp args)]- (if (every? u.time/valid? args)- (let [unit (temporal-unit field)- field (remove-temporal-unit field)- extract-expr (case unit- :day-of-week [:get-day-of-week field :iso]- :month-of-year [:get-month field]- :quarter-of-year [:get-quarter field])- extract-unit (if (= unit :day-of-week) :day-of-week-iso unit)]- (into [:!= extract-expr]- (map #(u.time/extract % extract-unit))- args))- &match))))--(defn- replace-legacy-filters- "Replaces legacy filter clauses with modern alternatives."- [query]- (lib.util.match/replace query- (filter-clause :guard mbql.preds/Filter?)- (try- (-> filter-clause- replace-relative-date-filters- replace-exclude-date-filters)- (catch #?(:clj Throwable :cljs :default) e- (throw (ex-info (i18n/tru "Error replacing legacy filters")- {:filter-clause filter-clause, :query query}- e))))))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | REMOVING EMPTY CLAUSES |-;;; +----------------------------------------------------------------------------------------------------------------+--(declare remove-empty-clauses path->special-remove-empty-clauses-fn)--(defn- remove-empty-clauses-in-map [m special-fns]- (when (map? m)- (not-empty (reduce-kv (fn [m k v]- (let [v' (remove-empty-clauses v (get special-fns k))]- (cond (nil? v') (dissoc m k)- (identical? v v') m- :else (assoc m k v'))))- m m))))--(defn- remove-empty-clauses-in-sequence* [xs special-fns]- (let [special-fns (::sequence special-fns)- xs (#?(:clj perf/mapv :default mapv) #(remove-empty-clauses % special-fns) xs)]- (when (some some? xs)- xs)))--(defmulti ^:private remove-empty-clauses-in-mbql-clause- {:arglists '([clause special-fns])}- (fn [[tag] _special-fns]- tag))--(defmethod remove-empty-clauses-in-mbql-clause :default- [clause special-fns]- (remove-empty-clauses-in-sequence* clause special-fns))--(defmethod remove-empty-clauses-in-mbql-clause :offset- [[_tag opts expr n] special-fns]- [:offset opts (remove-empty-clauses expr (:offset special-fns)) n])--(defn- remove-empty-clauses-in-sequence [x special-fns]- (if (mbql-clause? x)- (remove-empty-clauses-in-mbql-clause x special-fns)- (remove-empty-clauses-in-sequence* x special-fns)))--(defn- remove-empty-clauses-in-join [join]- (remove-empty-clauses join (:query path->special-remove-empty-clauses-fn)))--(defn- remove-empty-clauses-in-source-query [{native? :native, :as source-query}]- (if native?- source-query- (remove-empty-clauses source-query (:query path->special-remove-empty-clauses-fn))))--(defn- remove-empty-clauses-in-parameter [parameter]- (when (map? parameter)- (merge- ;; don't remove `value: nil` from a parameter, the FE code (`haveParametersChanged`) is extremely dumb and will- ;; consider the parameter to have changed and thus the query to be 'dirty' if we do this.- (select-keys parameter [:value])- (remove-empty-clauses-in-map parameter (-> path->special-remove-empty-clauses-fn :parameters ::sequence)))))--(def ^:private path->special-remove-empty-clauses-fn- {:native identity- :query {:source-query remove-empty-clauses-in-source-query- :joins {::sequence remove-empty-clauses-in-join}}- :parameters {::sequence remove-empty-clauses-in-parameter}- :viz-settings identity ; TODO (Cam 8/8/25) -- we also have [[metabase.models.interface/normalize-visualization-settings]]- :create-row identity- :update-row identity})--(defn- remove-empty-clauses- "Remove any empty or `nil` clauses in a query."- ([query]- (remove-empty-clauses query path->special-remove-empty-clauses-fn))-- ([x special-fns]- (try- (let [special-fn (when (and special-fns (fn? special-fns))- special-fns)]- (cond- special-fn (special-fn x)- (record? x) x- (map? x) (remove-empty-clauses-in-map x special-fns)- (sequential? x) (remove-empty-clauses-in-sequence x special-fns)- :else x))- (catch #?(:clj Throwable :cljs js/Error) e- (throw (ex-info (let [msg (ex-message e)]- (if (str/starts-with? msg "Error removing empty clauses")- msg- (str "Error removing empty clauses from form: " msg)))- {:form x}- e))))))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | PUTTING IT ALL TOGETHER |-;;; +----------------------------------------------------------------------------------------------------------------++(defn- infer-schema [x]+ (or (when (sequential? x)+ (when-let [tag (helpers/effective-clause-tag x)]+ (keyword "metabase.legacy-mbql.schema" (name tag))))+ ::mbql.s/Query))(defn normalize"Normalize the tokens in a Metabase query (i.e., make them all `lisp-case` keywords), rewrite deprecated clauses as- up-to-date MBQL 2000, and remove empty clauses."- [query]- (when (:lib/type query)- (throw (ex-info "Legacy MBQL normalization code cannot normalize MBQL >= 5" {:query query})))- (try- (-> query- normalize-tokens- canonicalize- perform-whole-query-transformations- replace-legacy-filters- remove-empty-clauses)- (catch #?(:clj Throwable :cljs js/Error) e- (throw (ex-info (i18n/tru "Error normalizing query: {0}" (ex-message e))- {:query query}- e)))))+ MBQL 4, and remove empty clauses."+ ([x]+ (normalize (infer-schema x) x))++ ([schema x]+ (when (:lib/type x)+ (throw (ex-info "Legacy MBQL normalization code cannot normalize MBQL >= 5" {:query x})))+ (lib.normalize/normalize schema x)))(mu/defn normalize-or-throw :- ::mbql.s/Query"Like [[normalize]], but checks the result against the Malli schema for a legacy query, which will cause it to throw@@ -1160,17 +35,10 @@[query :- :map](normalize query))-(mu/defn normalize-fragment- "Normalize just a specific fragment of a query, such as just the inner MBQL part or just a filter clause. `path` is- where this fragment would normally live in a full query.-- (normalize-fragment [:query :filter] [\"=\" 100 200])- ;;-> [:= [:field-id 100] 200]+(defn normalize-field-ref+ "Normalize the field ref. Ensure it's well-formed mbql, not just json.- DEPRECATED -- this is notoriously unreliable, convert to MBQL 5 and use [[metabase.lib.normalize]] instead."+ DEPRECATED: use Lib + MBQL 5 in new code going forward."{:deprecated "0.57.0"}- [path :- [:maybe [:sequential :keyword]]- x]- (if (empty? path)- (normalize x)- (get (normalize-fragment (butlast path) {(last path) x}) (last path))))+ [clause]+ (normalize ::mbql.s/field clause))diff --git a/src/metabase/legacy_mbql/predicates.cljc b/src/metabase/legacy_mbql/predicates.cljcindex ef3c4f1d2ba..f1b7e2a0641 100644--- a/src/metabase/legacy_mbql/predicates.cljc+++ b/src/metabase/legacy_mbql/predicates.cljc@@ -8,7 +8,7 @@(defn Field?"Is this a valid Field clause?"[x]- ((mr/validator ::mbql.s/field-or-expression-ref) x))+ ((mr/validator ::mbql.s/FieldOrExpressionRef) x))(defn Filter?"Is this a valid `:filter` clause?"diff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljcindex 54a674750b3..8bff6f3ebc8 100644--- a/src/metabase/legacy_mbql/schema.cljc+++ b/src/metabase/legacy_mbql/schema.cljc@@ -1,14 +1,22 @@(ns metabase.legacy-mbql.schema- "Schema for validating a *normalized* MBQL query. This is also the definitive grammar for MBQL, wow!"- (:refer-clojure :exclude [count distinct min max + - / * and or not not-empty = < > <= >= time case concat replace- abs float every? select-keys #?(:clj doseq) some])+ "Schema for validating a *normalized* MBQL 4 query.++ For 'historical reasons' all the MBQL clauses in this namespace have a schema like+ `::tag`, e.g. `metabase.legacy-mbql.schema/starts-with` for the `:starts-with` clause.+ Other schemas use `UpperCamelCase` names (as an artifact of when this was written using Schema 100 years ago) --+ changing this convention is more work than it's work, so follow it here if you need to touch this namespace.++ GENERALLY you should not need to touch this namespace, since MBQL 4 should be considered frozen in time in 57+ with+ the move to MBQL 5 in the app DB and over the wire -- any new MBQL clauses should only get added to the MBQL 5+ schema in [[metabase.lib.schema]] going forward. We don't want to have to add things to two places for the rest of+ our lives; it's ok if MBQL 4 doesn't support some new features."+ (:refer-clojure :exclude [every? select-keys #?(:clj doseq) some mapv update-keys])(:require- [clojure.core :as core][clojure.set :as set][clojure.string :as str][malli.core :as mc][metabase.legacy-mbql.schema.helpers :as helpers :refer [is-clause?]]- [metabase.legacy-mbql.schema.macros :refer [defclause one-of]]+ [metabase.legacy-mbql.schema.macros :refer [defclause defclause* one-of]][metabase.lib.schema.actions :as lib.schema.actions][metabase.lib.schema.binning :as lib.schema.binning][metabase.lib.schema.common :as lib.schema.common]@@ -27,9 +35,12 @@[metabase.lib.schema.settings :as lib.schema.settings][metabase.lib.schema.template-tag :as lib.schema.template-tag][metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]+ [metabase.lib.util.match :as lib.util.match]+ [metabase.util :as u][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- [metabase.util.performance :as perf :refer [every? select-keys #?(:clj doseq) some]]))+ [metabase.util.performance :as perf :refer [every? select-keys #?(:clj doseq) some mapv update-keys]]+ [metabase.util.time :as u.time]));; A NOTE ABOUT METADATA:;;@@ -44,7 +55,7 @@;; information that simplify driver implementations. When writing MBQL queries yourself you should pretend these;; clauses don't exist.;;-;; * Clauses marked `^{:requires-features #{feature+}}` require a certain set of features to be used. At some date in+;; * Clauses marked ` require a certain set of features to be used. At some date in;; the future we will likely add middleware that uses this metadata to automatically validate that a driver has the;; features needed to run the query in question.@@ -121,10 +132,6 @@"Valid unit for *datetime* bucketing."(into [:enum {:error/message "datetime bucketing unit"}] datetime-bucketing-units))-(def ^:private TimezoneId- "Valid timezone id."- [:ref ::lib.schema.expression.temporal/timezone-id])-(mr/def ::TemporalExtractUnit"Valid units to extract from a temporal."[:enum@@ -144,18 +151,30 @@(mr/def ::DatetimeDiffUnit"Valid units for a datetime-diff clause."- [:enum {:error/message "datetime-diff unit"} :second :minute :hour :day :week :month :quarter :year])+ [:enum+ {:error/message "datetime-diff unit", :decode/normalize helpers/normalize-keyword}+ :second :minute :hour :day :week :month :quarter :year])(mr/def ::ExtractWeekMode"Valid modes to extract weeks."- [:enum {:error/message "temporal-extract week extraction mode"} :iso :us :instance])+ [:enum+ {:error/message "temporal-extract week extraction mode", :decode/normalize helpers/normalize-keyword}+ :iso :us :instance])(mr/def ::RelativeDatetimeUnit- [:enum {:error/message "relative-datetime unit"} :default :minute :hour :day :week :month :quarter :year])+ [:enum+ {:error/message "relative-datetime unit", :decode/normalize helpers/normalize-keyword}+ :default :minute :hour :day :week :month :quarter :year]);; TODO - `unit` is not allowed if `n` is `current`(defclause relative-datetime- n [:or [:= :current] :int]+ n [:or+ {:decode/normalize (fn [x]+ (if (number? x)+ x+ :current))}+ [:= :current]+ :int]unit (optional [:ref ::RelativeDatetimeUnit]))(defclause interval@@ -172,13 +191,13 @@;;;; becomes:;; [:= [:field 10 {:temporal-unit :day}] [:absolute-datetime #inst "2018-10-02" :day]]-(mr/def ::absolute-datetime+(defclause* absolute-datetime[:multi {:error/message "valid :absolute-datetime clause":dispatch (fn [x](cond- (core/not (is-clause? :absolute-datetime x)) :invalid- (mr/validate ::lib.schema.literal/date (second x)) :date- :else :datetime))}+ (not (is-clause? :absolute-datetime x)) :invalid+ (mr/validate ::lib.schema.literal/date (second x)) :date+ :else :datetime))}[:invalid [:fn{:error/message "not an :absolute-datetime clause"}(constantly false)]]@@ -191,35 +210,31 @@"datetime" ::lib.schema.literal/datetime"unit" ::DateTimeUnit)]])-(def ^:internal absolute-datetime- "Schema for an `:absolute-datetime` clause."- (with-meta [:ref ::absolute-datetime] {:clause-name :absolute-datetime}))-;; almost exactly the same as `absolute-datetime`, but generated in some sitations where the literal in question was;; clearly a time (e.g. "08:00:00.000") and/or the Field derived from `:type/Time` and/or the unit was a;; time-bucketing unit-(defclause ^:internal time+(defclause timetime ::lib.schema.literal/time- unit ::TimeUnit)+ unit [:ref ::TimeUnit])(mr/def ::DateOrDatetimeLiteral"Schema for a valid date or datetime literal."[:or{:error/message "date or datetime literal"}- relative-datetime- absolute-datetime+ [:ref ::relative-datetime]+ [:ref ::absolute-datetime];; literal datetime strings and Java types will get transformed to [[absolute-datetime]] clauses automatically by;; middleware so drivers don't need to deal with these directly. You only need to worry about handling;; `absolute-datetime` clauses.- ::lib.schema.literal/datetime- ::lib.schema.literal/date])+ [:ref ::lib.schema.literal/datetime]+ [:ref ::lib.schema.literal/date]])(mr/def ::TimeLiteral"Schema for valid time literals."[:or{:error/message "time literal"}- time- ::lib.schema.literal/time])+ [:ref ::time]+ [:ref ::lib.schema.literal/time]])(mr/def ::TemporalLiteral"Schema for valid temporal literals."@@ -233,12 +248,13 @@`relative-datetime` form."(one-of absolute-datetime relative-datetime time))-;;; -------------------------------------------------- Other Values ---------------------------------------------------(mr/def ::ValueTypeInfo[:map- {:description (str "Type info about a value in a `:value` clause. Added automatically by `wrap-value-literals`"- " middleware to values in filter clauses based on the Field in the clause.")}+ {:decode/normalize (fn [m]+ (when (map? m)+ (update-keys m (comp keyword u/->snake_case_en))))+ :description (str "Type info about a value in a `:value` clause. Added automatically by `wrap-value-literals`"+ " middleware to values in filter clauses based on the Field in the clause.")}[:database_type {:optional true} [:maybe ::lib.schema.common/non-blank-string]][:base_type {:optional true} [:maybe ::lib.schema.common/base-type]][:semantic_type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]]@@ -259,14 +275,12 @@(defmethod options-style-method :value [_tag] ::options-style.last-always.snake_case)-;;; ----------------------------------------------------- Fields ------------------------------------------------------;; Expression *references* refer to a something in the `:expressions` clause, e.g. something like;;;; [:+ [:field 1 nil] [:field 2 nil]];;;; As of 0.42.0 `:expression` references can have an optional options map-(mr/def ::expression.options+(mr/def ::ExpressionRefOptions"Options for a legacy `:expression` ref in MBQL 4 are the same as in MBQL 5, except that `:lib/uuid` is optional andit cannot be empty."[:and@@ -283,29 +297,32 @@:decode/normalize perf/not-empty}seq]])-(mr/def ::expression+(mr/def ::ExpressionName+ [:ref+ {:decode/normalize #(cond-> % (keyword? %) u/qualified-name)}+ ::lib.schema.common/non-blank-string])++(defclause* expression[:and(helpers/clause:expression- "expression-name" ::lib.schema.common/non-blank-string- "options" [:optional ::expression.options])+ "expression-name" [:ref ::ExpressionName]+ "options" [:optional ::ExpressionRefOptions])[:fn- {:error/message "Expression should not have empty opts"- :decode/normalize (fn [[tag expression-name opts]]- (if (empty? opts)- [tag expression-name]- [tag expression-name opts]))}+ {:error/message ":expression should not have empty opts"+ :decode/normalize (fn [x]+ (when (helpers/mbql-clause? x)+ (if (and (= (count x) 3)+ (empty? (last x)))+ (pop (vec x)) ; remove nil/empty options maps.+ x)))}(fn [[_tag _expression-name opts :as expression-ref]]- (core/or (core/= (core/count expression-ref) 2)- (seq opts)))]])--(def expression- "Schema for an MBQL 4 :expression reference."- ^{:clause-name :expression} [:ref ::expression])+ (or (= (count expression-ref) 2)+ (seq opts)))]])(defmethod options-style-method :expression [_tag] ::options-style.last-unless-empty)-(mr/def ::FieldOptions+(mr/def ::FieldRefOptions"Options for an MBQL 4 `:field` ref are the same as MBQL 5, except that `:lib/uuid` is not required and it cannot beempty."[:maybe@@ -334,24 +351,71 @@(defmethod options-style-method :field [_tag] ::options-style.last-always)-(mr/def ::field+(defn- normalize-field [x]+ (case (cond+ (pos-int? x) ::raw-int+ (helpers/mbql-clause? x) (helpers/actual-clause-tag x))+ ::raw-int [:field x nil]+ :field-id (let [[_tag id] x]+ ;; sometimes the old FE code was dumb and passed in `:field-literal` wrapped inside` `:field-id`+ (if (sequential? id)+ (normalize-field id)+ [:field id nil]))+ :field-literal (let [[_tag field-name base-type] x]+ [:field (u/qualified-name field-name) {:base-type (keyword base-type)}])+ :datetime-field (let [[_tag nested-ref temporal-unit] (if (= (count x) 4)+ ;; handle MBQL 2 (?) `:datetime-field` which had the `:as`+ ;; keyword for "readability" e.g.+ ;;+ ;; [:datetime-field 20 :as :day]+ (let [[tag nested-ref _as temporal-unit] x]+ [tag nested-ref temporal-unit])+ x)+ [_tag id-or-name opts] (normalize-field nested-ref)]+ [:field id-or-name (assoc opts :temporal-unit (helpers/normalize-keyword temporal-unit))])+ :binning-strategy (let [[_tag nested-ref binning-strategy arg bin-opts] x+ [_tag id-or-name opts] (normalize-field nested-ref)+ strategy (helpers/normalize-keyword binning-strategy)]+ [:field id-or-name (assoc opts :binning (cond-> {:strategy strategy}+ arg (assoc strategy arg)+ bin-opts (merge bin-opts)))])+ :joined-field (let [[_tag join-alias nested-ref] x+ [_tag id-or-name opts] (normalize-field nested-ref)]+ [:field id-or-name (assoc opts :join-alias (u/qualified-name join-alias))])+ :fk-> (let [[_tag source-field dest-field] x+ [_tag source-field-id _opts] (normalize-field source-field)+ [_tag dest-field-id _opts] (normalize-field dest-field)]+ [:field dest-field-id {:source-field source-field-id}])+ :field (let [[_tag id-or-name opts] x]+ ;; if someone accidentally nests `:field` clauses fix it for them+ (if (sequential? id-or-name)+ (let [[_tag id-or-name recursive-opts] (normalize-field id-or-name)]+ [:field id-or-name (not-empty (merge recursive-opts opts))])+ [:field id-or-name (not-empty opts)]))+ x))++(defclause* ^{:added "0.39.0"} field[:and+ {:decode/normalize #'normalize-field}(helpers/clause:field"id-or-name" [:or ::lib.schema.id/field :string]- "options" [:maybe [:ref ::FieldOptions]])+ "options" [:maybe [:ref ::FieldRefOptions]])::require-base-type-for-field-name])-(def ^{:added "0.39.0"} field- "Schema for a `:field` clause."- ^{:clause-name :field} [:ref ::field])+(defn- normalize-raw-positive-int-to-field-ref+ "Treats raw positive integers as Field IDs for backwards compatibility with MBQL 2, e.g.-(mr/def ::field-or-expression-ref- (one-of expression field))+ [:= 10 20] => [:= [:field 10 nil] 20]"+ [x]+ (if (pos-int? x)+ [:field x nil]+ x))-(def FieldOrExpressionRef- "Schema for either a `:field` clause (reference to a Field) or an `:expression` clause (reference to an expression)."- [:ref ::field-or-expression-ref])+(mr/def ::FieldOrExpressionRef+ [:schema+ {:decode/normalize #'normalize-raw-positive-int-to-field-ref}+ (one-of expression field)]);; aggregate field reference refers to an aggregation, e.g.;;@@ -369,22 +433,29 @@;; wouldn't it;;;; As of 0.42.0 `:aggregation` references can have an optional options map.-(defclause aggregation- aggregation-clause-index :int- options (optional :map))+(defclause* aggregation+ [:and+ (helpers/clause+ :aggregation+ "aggregation-clause-index" :int+ "options" [:optional :map])+ [:fn+ {:error/message ":aggregation should not have empty opts"+ :decode/normalize (fn [x]+ (when (helpers/mbql-clause? x)+ (if (and (= (count x) 3)+ (empty? (last x)))+ (pop (vec x)) ; remove nil/empty options maps.+ x)))}+ (fn [[_tag _aggregation-index opts :as expression-ref]]+ (or (= (count expression-ref) 2)+ (seq opts)))]])(defmethod options-style-method :aggregation [_tag] ::options-style.last-unless-empty)(mr/def ::Reference- [:schema- ;; this is provided for the convenience of Lib which uses this schema for `:field-ref` in results metadata- {:decode/normalize (fn [x]- ((#?(:clj requiring-resolve :cljs resolve) 'metabase.legacy-mbql.normalize/normalize-field-ref) x))}- (one-of aggregation expression field)])--(def Reference"Schema for any type of valid Field clause, or for an indexed reference to an aggregation clause."- [:ref ::Reference])+ (one-of aggregation expression field))(defclause ^{:added "0.50.0"} offsetopts [:ref ::lib.schema.common/options]@@ -393,8 +464,6 @@(defmethod options-style-method :offset [_tag] ::options-style.mbql5)-;;; -------------------------------------------------- Expressions ----------------------------------------------------;; Expressions are "calculated column" definitions, defined once and then used elsewhere in the MBQL query.(def string-functions@@ -402,10 +471,6 @@#{:substring :trim :rtrim :ltrim :upper :lower :replace :concat :regex-match-first :coalesce :case :if:host :domain :subdomain :path :month-name :quarter-name :day-name :text :split-part})-(def ^:private StringExpression- "Schema for the definition of an string expression."- [:ref ::StringExpression])-(mr/def ::StringExpressionArg[:multi{:dispatch (fn [x]@@ -415,15 +480,12 @@(is-clause? :value x) :value:else :else))}[:string :string]- [:string-expression StringExpression]- [:value value]- [:else FieldOrExpressionRef]])--(def ^:private StringExpressionArg- [:ref ::StringExpressionArg])+ [:string-expression [:ref ::StringExpression]]+ [:value [:ref ::value]]+ [:else [:ref ::FieldOrExpressionRef]]])(def numeric-functions- "Functions that return numeric values. Should match [[NumericExpression]]."+ "Functions that return numeric values. Should match `::NumericExpression`."#{:+ :- :/ :* :coalesce :length :round :ceil :floor :abs :power :sqrt :log :exp :case :if :datetime-diff :integer :float;; extraction functions (get some component of a given temporal value/column):temporal-extract@@ -431,7 +493,7 @@:get-year :get-quarter :get-month :get-week :get-day :get-day-of-week :get-hour :get-minute :get-second})(def boolean-functions- "Functions that return boolean values. Should match [[BooleanExpression]]."+ "Functions that return boolean values. Should match `::BooleanExpression`."#{:and :or :not :< :<= :> :>= := :!= :in :not-in :between :starts-with :ends-with :contains:does-not-contain :inside :is-empty :not-empty :is-null :not-null :relative-time-interval :time-interval :during})@@ -440,25 +502,9 @@:distinct-where :metric :aggregation-options :count :offset})(def ^:private datetime-functions- "Functions that return Date or DateTime values. Should match [[DatetimeExpression]]."+ "Functions that return Date or DateTime values. Should match `::DatetimeExpression`."#{:+ :datetime-add :datetime-subtract :convert-timezone :now :date :datetime :today})-(def ^:private NumericExpression- "Schema for the definition of a numeric expression. All numeric expressions evaluate to numeric values."- [:ref ::NumericExpression])--(def ^:private BooleanExpression- "Schema for the definition of an arithmetic expression."- [:ref ::BooleanExpression])--(def DatetimeExpression- "Schema for the definition of a date function expression."- [:ref ::DatetimeExpression])--(def Aggregation- "Schema for anything that is a valid `:aggregation` clause."- [:ref ::Aggregation])-(mr/def ::NumericExpressionArg[:multi{:error/message "numeric expression argument"@@ -470,12 +516,10 @@(is-clause? :value x) :value:else :field))}[:number number?]- [:numeric-expression NumericExpression]- [:aggregation Aggregation]- [:value value]- [:field Reference]])-(def ^:private NumericExpressionArg- [:ref ::NumericExpressionArg])+ [:numeric-expression [:ref ::NumericExpression]]+ [:aggregation [:ref ::Aggregation]]+ [:value [:ref ::value]]+ [:field [:ref ::Reference]]])(mr/def ::DateTimeExpressionArg[:multi@@ -486,13 +530,12 @@(is-clause? :value x) :value(is-clause? datetime-functions x) :datetime-expression:else :else))}- [:aggregation Aggregation]- [:value value]- [:datetime-expression DatetimeExpression]- [:else [:or [:ref ::DateOrDatetimeLiteral] FieldOrExpressionRef]]])--(def ^:private DateTimeExpressionArg- [:ref ::DateTimeExpressionArg])+ [:aggregation [:ref ::Aggregation]]+ [:value [:ref ::value]]+ [:datetime-expression [:ref ::DatetimeExpression]]+ [:else [:or+ [:ref ::DateOrDatetimeLiteral]+ [:ref ::FieldOrExpressionRef]]]])(mr/def ::ExpressionArg[:multi@@ -511,27 +554,21 @@:else :else))}[:number number?][:boolean :boolean]- [:boolean-expression BooleanExpression]- [:numeric-expression NumericExpression]- [:datetime-expression DatetimeExpression]- [:aggregation Aggregation]+ [:boolean-expression [:ref ::BooleanExpression]]+ [:numeric-expression [:ref ::NumericExpression]]+ [:datetime-expression [:ref ::DatetimeExpression]]+ [:aggregation [:ref ::Aggregation]][:string :string]- [:string-expression StringExpression]- [:value value]- [:else FieldOrExpressionRef]])--(def ^:private ExpressionArg- [:ref ::ExpressionArg])+ [:string-expression [:ref ::StringExpression]]+ [:value [:ref ::value]]+ [:else [:ref ::FieldOrExpressionRef]]])(mr/def ::Addable[:or{:error/message "numeric expression arg or interval"}- DateTimeExpressionArg- interval- NumericExpressionArg])--(def ^:private Addable- [:ref ::Addable])+ [:ref ::NumericExpressionArg]+ [:ref ::DateTimeExpressionArg]+ (one-of interval segment)])(mr/def ::IntGreaterThanZeroOrNumericExpression[:multi@@ -540,221 +577,289 @@(if (number? x):number:else))}- [:number pos-int?]- [:else NumericExpression]])--(def ^:private IntGreaterThanZeroOrNumericExpression- [:ref ::IntGreaterThanZeroOrNumericExpression])--(defclause ^{:requires-features #{:expressions}} coalesce- a ExpressionArg, b ExpressionArg, more (rest ExpressionArg))--(defclause ^{:requires-features #{:expressions}} substring- s StringExpressionArg, start IntGreaterThanZeroOrNumericExpression, length (optional NumericExpressionArg))--(defclause ^{:requires-features #{:expressions :split-part}} split-part- text StringExpressionArg, delimiter [:string {:min 1}], position IntGreaterThanZeroOrNumericExpression)--(defclause ^{:requires-features #{:expressions}} length- s StringExpressionArg)--(defclause ^{:requires-features #{:expressions}} trim- s StringExpressionArg)--(defclause ^{:requires-features #{:expressions}} rtrim- s StringExpressionArg)--(defclause ^{:requires-features #{:expressions}} ltrim- s StringExpressionArg)--(defclause ^{:requires-features #{:expressions}} upper- s StringExpressionArg)--(defclause ^{:requires-features #{:expressions}} lower- s StringExpressionArg)--(defclause ^{:requires-features #{:expressions}} replace- s StringExpressionArg, match :string, replacement :string)--(defclause ^{:requires-features #{:expressions :expressions/text}} text+ [:number [:schema+ {:decode/normalize (fn [n]+ (let [n (long n)]+ (if (pos-int? n)+ n+ 1)))}+ pos-int?]]+ [:else [:ref ::NumericExpression]]])++(defclause coalesce+ a [:ref ::ExpressionArg]+ b [:ref ::ExpressionArg]+ more (rest [:ref ::ExpressionArg]))++(defclause substring+ s [:ref ::StringExpressionArg]+ start [:ref ::IntGreaterThanZeroOrNumericExpression]+ length (optional [:ref ::NumericExpressionArg]))++(defclause split-part+ text [:ref ::StringExpressionArg]+ delimiter [:string {:min 1}]+ position [:ref ::IntGreaterThanZeroOrNumericExpression])++(defclause length+ s [:ref ::StringExpressionArg])++(defclause trim+ s [:ref ::StringExpressionArg])++(defclause rtrim+ s [:ref ::StringExpressionArg])++(defclause ltrim+ s [:ref ::StringExpressionArg])++(defclause upper+ s [:ref ::StringExpressionArg])++(defclause lower+ s [:ref ::StringExpressionArg])++(defclause replace+ s [:ref ::StringExpressionArg]+ match :string+ replacement :string)++(defclause textx :any);; Relax the arg types to ExpressionArg for concat since many DBs allow to concatenate non-string types. This also;; aligns with the corresponding MLv2 schema and with the reference docs we publish.-(defclause ^{:requires-features #{:expressions}} concat- a ExpressionArg, b ExpressionArg, more (rest ExpressionArg))+(defclause concat+ a [:ref ::ExpressionArg]+ b [:ref ::ExpressionArg]+ more (rest [:ref ::ExpressionArg]))-(defclause ^{:requires-features #{:expressions :regex}} regex-match-first- s StringExpressionArg, pattern :string)+(defclause regex-match-first+ s [:ref ::StringExpressionArg]+ pattern :string)-(defclause ^{:requires-features #{:expressions :regex}} host- s StringExpressionArg)+(defclause host+ s [:ref ::StringExpressionArg])-(defclause ^{:requires-features #{:expressions :regex}} domain- s StringExpressionArg)+(defclause domain+ s [:ref ::StringExpressionArg])-(defclause ^{:requires-features #{:expressions :regex}} subdomain- s StringExpressionArg)+(defclause subdomain+ s [:ref ::StringExpressionArg])-(defclause ^{:requires-features #{:expressions :regex}} path- s StringExpressionArg)+(defclause path+ s [:ref ::StringExpressionArg])-(defclause ^{:requires-features #{:expressions}} month-name- n NumericExpressionArg)+(defclause month-name+ n [:ref ::NumericExpressionArg])-(defclause ^{:requires-features #{:expressions}} quarter-name- n NumericExpressionArg)+(defclause quarter-name+ n [:ref ::NumericExpressionArg])-(defclause ^{:requires-features #{:expressions}} day-name- n NumericExpressionArg)+(defclause day-name+ n [:ref ::NumericExpressionArg])-(defclause ^{:requires-features #{:expressions}} +- x Addable, y Addable, more (rest Addable))+(defclause ++ x [:ref ::Addable]+ y [:ref ::Addable]+ more (rest [:ref ::Addable]))-(defclause ^{:requires-features #{:expressions}} -- x NumericExpressionArg, y Addable, more (rest Addable))+(defclause -+ x [:ref ::NumericExpressionArg]+ y [:ref ::Addable]+ more (rest [:ref ::Addable]))-(defclause ^{:requires-features #{:expressions}} /, x NumericExpressionArg, y NumericExpressionArg, more (rest NumericExpressionArg))+(defclause /+ x [:ref ::NumericExpressionArg]+ y [:ref ::NumericExpressionArg]+ more (rest [:ref ::NumericExpressionArg]))-(defclause ^{:requires-features #{:expressions}} *, x NumericExpressionArg, y NumericExpressionArg, more (rest NumericExpressionArg))+(defclause *+ x [:ref ::NumericExpressionArg]+ y [:ref ::NumericExpressionArg]+ more (rest [:ref ::NumericExpressionArg]))-(defclause ^{:requires-features #{:expressions}} floor- x NumericExpressionArg)+(defclause floor+ x [:ref ::NumericExpressionArg])-(defclause ^{:requires-features #{:expressions}} ceil- x NumericExpressionArg)+(defclause ceil+ x [:ref ::NumericExpressionArg])-(defclause ^{:requires-features #{:expressions}} round- x NumericExpressionArg)+(defclause round+ x [:ref ::NumericExpressionArg])-(defclause ^{:requires-features #{:expressions}} abs- x NumericExpressionArg)+(defclause abs+ x [:ref ::NumericExpressionArg])-(defclause ^{:requires-features #{:advanced-math-expressions}} power- x NumericExpressionArg, y NumericExpressionArg)+(defclause power+ x [:ref ::NumericExpressionArg]+ y [:ref ::NumericExpressionArg])-(defclause ^{:requires-features #{:advanced-math-expressions}} sqrt- x NumericExpressionArg)+(defclause sqrt+ x [:ref ::NumericExpressionArg])-(defclause ^{:requires-features #{:advanced-math-expressions}} exp- x NumericExpressionArg)+(defclause exp+ x [:ref ::NumericExpressionArg])-(defclause ^{:requires-features #{:advanced-math-expressions}} log- x NumericExpressionArg)+(defclause log+ x [:ref ::NumericExpressionArg])-(defclause ^{:requires-features #{:expressions :expressions/integer}} integer- x [:or NumericExpressionArg StringExpressionArg])+(defclause integer+ x [:or [:ref ::NumericExpressionArg] [:ref ::StringExpressionArg]])-(defclause ^{:requires-features #{:expressions :expressions/float}} float- x StringExpressionArg)+(defclause float+ x [:ref ::StringExpressionArg]);; The result is positive if x <= y, and negative otherwise.;;;; Days, weeks, months, and years are only counted if they are whole to the "day".;; For example, `datetimeDiff("2022-01-30", "2022-02-28", "month")` returns 0 months.;;-;; If the values are datetimes, the time doesn't matter for these units.-;; For example, `datetimeDiff("2022-01-01T09:00:00", "2022-01-02T08:00:00", "day")` returns 1 day even though it is less than 24 hours.+;; If the values are datetimes, the time doesn't matter for these units. For example,+;; `datetimeDiff("2022-01-01T09:00:00", "2022-01-02T08:00:00", "day")` returns 1 day even though it is less than 24+;; hours.;;;; Hours, minutes, and seconds are only counted if they are whole.;; For example, datetimeDiff("2022-01-01T01:00:30", "2022-01-01T02:00:29", "hour") returns 0 hours.-(defclause ^{:requires-features #{:datetime-diff}} datetime-diff- datetime-x DateTimeExpressionArg- datetime-y DateTimeExpressionArg+(defclause datetime-diff+ datetime-x [:ref ::DateTimeExpressionArg]+ datetime-y [:ref ::DateTimeExpressionArg]unit [:ref ::DatetimeDiffUnit])-(defclause ^{:requires-features #{:temporal-extract}} temporal-extract- datetime DateTimeExpressionArg+(defclause temporal-extract+ datetime [:ref ::DateTimeExpressionArg]unit [:ref ::TemporalExtractUnit]mode (optional [:ref ::ExtractWeekMode])) ;; only for get-week and get-day-of-week-;; SUGAR CLAUSE: get-year, get-month... clauses are all sugars clause that will be rewritten as [:temporal-extract column :year]-(defclause ^{:requires-features #{:temporal-extract}} ^:sugar get-year- date DateTimeExpressionArg)+;; SUGAR CLAUSE: get-year, get-month... clauses are all sugars clause that will be rewritten as+;;+;; [:temporal-extract column :year]+(defclause get-year+ date [:ref ::DateTimeExpressionArg])-(defclause ^{:requires-features #{:temporal-extract}} ^:sugar get-quarter- date DateTimeExpressionArg)+(defclause get-quarter+ date [:ref ::DateTimeExpressionArg])-(defclause ^{:requires-features #{:temporal-extract}} ^:sugar get-month- date DateTimeExpressionArg)+(defclause get-month+ date [:ref ::DateTimeExpressionArg])-(defclause ^{:requires-features #{:temporal-extract}} ^:sugar get-week- date DateTimeExpressionArg+(defclause get-week+ date [:ref ::DateTimeExpressionArg]mode (optional [:ref ::ExtractWeekMode]))-(defclause ^{:requires-features #{:temporal-extract}} ^:sugar get-day- date DateTimeExpressionArg)+(defclause get-day+ date [:ref ::DateTimeExpressionArg])-(defclause ^{:requires-features #{:temporal-extract}} ^:sugar get-day-of-week- date DateTimeExpressionArg+(defclause get-day-of-week+ date [:ref ::DateTimeExpressionArg]mode (optional [:ref ::ExtractWeekMode]))-(defclause ^{:requires-features #{:temporal-extract}} ^:sugar get-hour- datetime DateTimeExpressionArg)+(defclause get-hour+ datetime [:ref ::DateTimeExpressionArg])-(defclause ^{:requires-features #{:temporal-extract}} ^:sugar get-minute- datetime DateTimeExpressionArg)+(defclause get-minute+ datetime [:ref ::DateTimeExpressionArg])-(defclause ^{:requires-features #{:temporal-extract}} ^:sugar get-second- datetime DateTimeExpressionArg)+(defclause get-second+ datetime [:ref ::DateTimeExpressionArg])-(defclause ^{:requires-features #{:convert-timezone}} convert-timezone- datetime DateTimeExpressionArg- to TimezoneId- from (optional TimezoneId))+(defclause convert-timezone+ datetime [:ref ::DateTimeExpressionArg]+ to [:ref ::lib.schema.expression.temporal/timezone-id]+ from (optional [:ref ::lib.schema.expression.temporal/timezone-id]))-(def ^:private ArithmeticDateTimeUnit- [:enum {:error/message "datetime arithmetic unit"} :millisecond :second :minute :hour :day :week :month :quarter :year])+(mr/def ::ArithmeticDateTimeUnit+ [:enum {:error/message "datetime arithmetic unit", :decode/normalize keyword}+ :millisecond :second :minute :hour :day :week :month :quarter :year])-(defclause ^{:requires-features #{:date-arithmetics}} datetime-add- datetime DateTimeExpressionArg- amount NumericExpressionArg- unit ArithmeticDateTimeUnit)+(defclause datetime-add+ datetime [:ref ::DateTimeExpressionArg]+ amount [:ref ::NumericExpressionArg]+ unit [:ref ::ArithmeticDateTimeUnit])-(defclause ^{:requires-features #{:now}} now)+(defclause now)-(defclause ^{:requires-features #{:date-arithmetics}} datetime-subtract- datetime DateTimeExpressionArg- amount NumericExpressionArg- unit ArithmeticDateTimeUnit)+(defclause datetime-subtract+ datetime [:ref ::DateTimeExpressionArg]+ amount [:ref ::NumericExpressionArg]+ unit [:ref ::ArithmeticDateTimeUnit])-(defclause ^{:requires-features #{:expressions :expressions/date}} date- string [:or StringExpressionArg DateTimeExpressionArg])+(defclause date+ string [:or+ [:ref ::StringExpressionArg]+ [:ref ::DateTimeExpressionArg]])-(defclause ^{:requires-features #{:expressions :expressions/today}} today)+(defclause today)-(def ^:private LiteralDatetimeModeString- (into [:enum {:error/message "datetime mode string"}]+(mr/def ::DatetimeOptionsMode+ (into [:enum {:error/message "datetime mode string", :decode/normalize lib.schema.expression.temporal/normalize-datetime-mode}]lib.schema.expression.temporal/datetime-modes))-(defclause ^{:requires-features #{:expressions :expressions/datetime}} datetime- value :any ;; normally a string, number, or bytes- options (optional [:map [:mode {:optional true} LiteralDatetimeModeString]]))+(mr/def ::DatetimeOptions+ [:map {:decode/normalize lib.schema.common/normalize-map}+ [:mode {:optional true} [:ref ::DatetimeOptionsMode]]])++(defclause datetime+ value [:ref ::ExpressionArg]+ options (optional [:ref ::DatetimeOptions]))(defmethod options-style-method :datetime [_tag] ::options-style.last-unless-empty)(mr/def ::DatetimeExpression+ "Schema for the definition of a date function expression."(one-of + datetime-add datetime-subtract convert-timezone now date datetime today))-;;; ----------------------------------------------------- Filter -------------------------------------------------------(def Filter- "Schema for a valid MBQL `:filter` clause."- [:ref ::Filter])--(defclause and- first-clause Filter- second-clause Filter- other-clauses (rest Filter))--(defclause or- first-clause Filter- second-clause Filter- other-clauses (rest Filter))--(defclause not, clause Filter)+(defn- compound-filter-schema [tag]+ [:multi+ {:decode/normalize (fn [x]+ (when (sequential? x)+ (let [[_tag & subclauses] x]+ (if (= (count subclauses) 1)+ (first subclauses)+ x))))+ :dispatch helpers/effective-clause-tag}+ [tag [:and+ (helpers/clause+ tag+ "first-clause" [:ref ::Filter]+ "second-clause" [:ref ::Filter]+ "other-clauses" [:rest [:ref ::Filter]])+ ;; flatten nested compound filters of the same type, e.g. `[:and x [:and y z]]` => `[:and x y z]`+ [:schema+ {:decode/normalize (fn [[tag & subclauses]]+ (into [tag]+ (mapcat (fn [subclause]+ (if (= (helpers/actual-clause-tag subclause) tag)+ (rest subclause)+ [subclause])))+ subclauses))}+ :any]]]+ [::mc/default [:ref ::Filter]]])++(defclause* and+ (compound-filter-schema :and))++(defclause* or+ (compound-filter-schema :or))++(defn- normalize-not+ "`not` inside of a `not` should get elimated entirely."+ [[_tag subclause-1 :as clause]]+ (or (when (= (helpers/actual-clause-tag subclause-1) :not)+ (let [[_tag subclause-2] subclause-1]+ subclause-2))+ clause))++(defclause* not+ [:and+ (helpers/clause+ :not+ "clause" [:ref :metabase.legacy-mbql.schema/Filter])+ [:schema+ {:decode/normalize #'normalize-not}+ :any]])-(def ^:private FieldOrExpressionRefOrRelativeDatetime+(mr/def ::FieldOrExpressionRefOrRelativeDatetime[:multi{:error/message ":field or :expression reference or :relative-datetime":error/fn (constantly ":field or :expression reference or :relative-datetime")@@ -762,10 +867,11 @@(if (is-clause? :relative-datetime x):relative-datetime:else))}- [:relative-datetime relative-datetime]- [:else FieldOrExpressionRef]])+ [:relative-datetime [:ref ::relative-datetime]]+ [:else [:ref ::FieldOrExpressionRef]]])(mr/def ::EqualityComparable+ "Schema for things that make sense in a `=` or `!=` filter, i.e. things that can be compared for equality."[:maybe{:error/message "equality comparable"}[:or@@ -773,32 +879,25 @@number?:string[:ref ::TemporalLiteral]- FieldOrExpressionRefOrRelativeDatetime- ExpressionArg- value]])--(def ^:private EqualityComparable- "Schema for things that make sense in a `=` or `!=` filter, i.e. things that can be compared for equality."- [:ref ::EqualityComparable])+ [:ref ::FieldOrExpressionRefOrRelativeDatetime]+ [:ref ::ExpressionArg]+ [:ref ::value]]])(mr/def ::OrderComparable+ "Schema for things that make sense in a filter like `>` or `<`, i.e. things that can be sorted."[:multi{:error/message "order comparable":dispatch (fn [x](if (is-clause? :value x):value:else))}- [:value value]+ [:value [:ref ::value]][:else [:ornumber?:string[:ref ::TemporalLiteral]- ExpressionArg- FieldOrExpressionRefOrRelativeDatetime]]])--(def ^:private OrderComparable- "Schema for things that make sense in a filter like `>` or `<`, i.e. things that can be sorted."- [:ref ::OrderComparable])+ [:ref ::ExpressionArg]+ [:ref ::FieldOrExpressionRefOrRelativeDatetime]]]]);; For all of the non-compound Filter clauses below the first arg is an implicit Field ID@@ -813,90 +912,172 @@;;;; [:!= [:field 1 nil] 2 3] --[DESUGAR]--> [:and [:!= [:field 1 nil] 2] [:!= [:field 1 nil] 3]]-(defclause =, field EqualityComparable, value-or-field EqualityComparable, more-values-or-fields (rest EqualityComparable))-(defclause !=, field EqualityComparable, value-or-field EqualityComparable, more-values-or-fields (rest EqualityComparable))+(mr/def ::EqualityFilterFieldArg+ "Schema for the first arg to `=`, `!=`, and friends."+ [:schema+ {:decode/normalize #'normalize-raw-positive-int-to-field-ref}+ [:ref ::EqualityComparable]])++(defclause =+ field [:ref ::EqualityFilterFieldArg]+ value-or-field [:ref ::EqualityComparable]+ more-values-or-fields (rest [:ref ::EqualityComparable]))++(defn- replace-exclude-date-filters+ "Replaces legacy exclude date filter clauses that rely on temporal bucketing with `:temporal-extract` function calls."+ [filter-clause]+ (lib.util.match/replace filter-clause+ [:!=+ [:field id-or-name (opts :guard #(= (:temporal-unit %) :hour-of-day))]+ & (args :guard #(every? number? %))]+ (into [:!= [:get-hour [:field id-or-name (not-empty (dissoc opts :temporal-unit))]]] args)++ [:!=+ [:field id-or-name (opts :guard #(#{:day-of-week :month-of-year :quarter-of-year} (:temporal-unit %)))]+ & (args :guard #(every? u.time/timestamp-coercible? %))]+ (let [args (mapv u.time/coerce-to-timestamp args)]+ (if (every? u.time/valid? args)+ (let [unit (:temporal-unit opts)+ field [:field id-or-name (not-empty (dissoc opts :temporal-unit))]+ extract-expr (case unit+ :day-of-week [:get-day-of-week field :iso]+ :month-of-year [:get-month field]+ :quarter-of-year [:get-quarter field])+ extract-unit (if (= unit :day-of-week) :day-of-week-iso unit)]+ (into [:!= extract-expr]+ (map #(u.time/extract % extract-unit))+ args))+ &match))))++(defclause* !=+ [:and+ (helpers/clause+ :!=+ "field" [:ref ::EqualityFilterFieldArg]+ "value-or-field" [:ref ::EqualityComparable]+ "more-values-or-fields" [:rest [:ref ::EqualityComparable]])+ [:schema+ {:decode/normalize #'replace-exclude-date-filters}+ :any]]);; aliases for `:=` and `:!=`-(defclause ^:sugar in, field EqualityComparable, value-or-field EqualityComparable, more-values-or-fields (rest EqualityComparable))-(defclause ^:sugar not-in, field EqualityComparable, value-or-field EqualityComparable, more-values-or-fields (rest EqualityComparable))+(defclause in+ field [:ref ::EqualityFilterFieldArg]+ value-or-field [:ref ::EqualityComparable]+ more-values-or-fields (rest [:ref ::EqualityComparable]))++(defclause not-in+ field [:ref ::EqualityFilterFieldArg]+ value-or-field [:ref ::EqualityComparable]+ more-values-or-fields (rest [:ref ::EqualityComparable]))-(defclause <, field OrderComparable, value-or-field OrderComparable)-(defclause >, field OrderComparable, value-or-field OrderComparable)-(defclause <=, field OrderComparable, value-or-field OrderComparable)-(defclause >=, field OrderComparable, value-or-field OrderComparable)+(mr/def ::OrderedFilterFieldArg+ [:schema+ {:decode/normalize #'normalize-raw-positive-int-to-field-ref}+ [:ref ::OrderComparable]])++(defclause <, field [:ref ::OrderedFilterFieldArg], value-or-field [:ref ::OrderComparable])+(defclause >, field [:ref ::OrderedFilterFieldArg], value-or-field [:ref ::OrderComparable])+(defclause <=, field [:ref ::OrderedFilterFieldArg], value-or-field [:ref ::OrderComparable])+(defclause >=, field [:ref ::OrderedFilterFieldArg], value-or-field [:ref ::OrderComparable])++(defn- replace-relative-date-filters+ "Replaces broken relative date filter clauses with `:relative-time-interval` calls.++ Previously we generated a complex expression for relative date filters with an offset on the FE. It turned out that+ the expression was wrong by 1 offset unit, e.g. if the offset was by months, it was wrong by 1 month. To fix the issue+ we introduced a new `:relative-time-interval` function that served several purposes. It captured the user intent+ clearly while hiding the implementation details; it also fixed the underlying expression. Here we match the old+ expression and convert it to a `:relative-time-interval` call, honoring the original user intent. See #46211 and+ #46438 for details."+ [clause]+ (lib.util.match/replace clause+ [:between+ [:++ field+ [:interval (offset-value :guard integer?) (offset-unit :guard keyword?)]]+ [:relative-datetime+ (start-value :guard integer?)+ (start-unit :guard keyword?)]+ [:relative-datetime+ (end-value :guard integer?)+ (end-unit :guard keyword?)]]+ (let [offset-value (- offset-value)]+ (if (and (= start-unit end-unit)+ (or (and (pos? offset-value) (zero? start-value) (pos? end-value))+ (and (neg? offset-value) (neg? start-value) (zero? end-value))))+ [:relative-time-interval+ field+ (if (neg? offset-value) start-value end-value)+ start-unit+ offset-value+ offset-unit]+ &match))));; :between is INCLUSIVE just like SQL !!!-(defclause between field OrderComparable, min OrderComparable, max OrderComparable)+(defclause* between+ [:and+ (helpers/clause+ :between+ "field" [:ref ::OrderedFilterFieldArg]+ "min" [:ref ::OrderComparable]+ "max" [:ref ::OrderComparable])+ [:schema+ {:decode/normalize #'replace-relative-date-filters}+ :any]]);; SUGAR CLAUSE: This is automatically written as a pair of `:between` clauses by the `:desugar` middleware.-(defclause ^:sugar inside- lat-field OrderComparable- lon-field OrderComparable- lat-max OrderComparable- lon-min OrderComparable- lat-min OrderComparable- lon-max OrderComparable)+(defclause inside+ lat-field [:ref ::OrderedFilterFieldArg]+ lon-field [:ref ::OrderedFilterFieldArg]+ lat-max [:ref ::OrderComparable]+ lon-min [:ref ::OrderComparable]+ lat-min [:ref ::OrderComparable]+ lon-max [:ref ::OrderComparable]);; SUGAR CLAUSES: These are rewritten as `[:= <field> nil]` and `[:not= <field> nil]` respectively-(defclause ^:sugar is-null, field FieldOrExpressionRef)-(defclause ^:sugar not-null, field FieldOrExpressionRef)+(defclause is-null, field [:ref ::FieldOrExpressionRef])+(defclause not-null, field [:ref ::FieldOrExpressionRef])(mr/def ::Emptyable- [:or StringExpressionArg FieldOrExpressionRef])--(def Emptyable"Schema for a valid is-empty or not-empty argument."- [:ref ::Emptyable])+ [:or+ [:ref ::StringExpressionArg]+ [:ref ::FieldOrExpressionRef]]);; These are rewritten as `[:or [:= <field> nil] [:= <field> ""]]` and;; `[:and [:not= <field> nil] [:not= <field> ""]]`-(defclause ^:sugar is-empty field Emptyable)-(defclause ^:sugar not-empty field Emptyable)+(defclause is-empty field [:ref ::Emptyable])+(defclause not-empty field [:ref ::Emptyable])-(def ^:private StringFilterOptions+(mr/def ::StringFilterOptions[:map+ {:decode/normalize lib.schema.common/normalize-map};; default true[:case-sensitive {:optional true} :boolean]])(doseq [clause-keyword [::starts-with ::ends-with ::contains ::does-not-contain]](defmethod options-style-method (keyword (name clause-keyword)) [_tag] ::options-style.𝕨𝕚𝕝𝕕)- (mr/def clause-keyword+ (helpers/defclause clause-keyword[:or;; Binary form(helpers/clause (keyword (name clause-keyword))- "field" StringExpressionArg- "string-or-field" StringExpressionArg- "options" [:optional StringFilterOptions])+ "field" [:ref ::StringExpressionArg]+ "string-or-field" [:ref ::StringExpressionArg]+ "options" [:optional [:ref ::StringFilterOptions]]);; Multi-arg form(helpers/clause (keyword (name clause-keyword))- "options" [:maybe StringFilterOptions]- "field" StringExpressionArg- "string-or-field" StringExpressionArg- "second-string-or-field" StringExpressionArg- "more-strings-or-fields" [:rest StringExpressionArg])]))--(def starts-with- "Schema for a valid :starts-with clause."- (with-meta [:ref ::starts-with] {:clause-name :starts-with}))--(def ends-with- "Schema for a valid :ends-with clause."- (with-meta [:ref ::ends-with] {:clause-name :ends-with}))--(def contains- "Schema for a valid :contains clause."- (with-meta [:ref ::contains] {:clause-name :contains}))--;; SUGAR: this is rewritten as [:not [:contains ...]]-(def ^:sugar does-not-contain- "Schema for a valid :does-not-contain clause."- (with-meta [:ref ::does-not-contain] {:clause-name :does-not-contain}))--(def ^:private TimeIntervalOptions- ;; Should we include partial results for the current day/month/etc? Defaults to `false`; set this to `true` to- ;; include them.+ "options" [:maybe [:ref ::StringFilterOptions]]+ "field" [:ref ::StringExpressionArg]+ "string-or-field" [:ref ::StringExpressionArg]+ "second-string-or-field" [:ref ::StringExpressionArg]+ "more-strings-or-fields" [:rest [:ref ::StringExpressionArg]])]))++(mr/def ::TimeIntervalOptions[:map- ;; default false+ {:decode/normalize lib.schema.common/normalize-map}+ ;; Should we include partial results for the current day/month/etc? Defaults to `false`; set this to `true` to+ ;; include them.[:include-current {:optional true} :boolean]]);; Filter subclause. Syntactic sugar for specifying a specific time interval.@@ -911,23 +1092,37 @@;; [:time-interval [:field 100 nil] :current :month {:include-current true}];;;; SUGAR: This is automatically rewritten as a filter clause with a relative-datetime value-(defclause ^:sugar time-interval- field FieldOrExpressionRef- n [:or- :int- [:enum :current :last :next]]- unit [:ref ::RelativeDatetimeUnit]- options (optional TimeIntervalOptions))+(defn- normalize-time-interval+ "If you specify a `:temporal-unit` for the Field inside a `:time-interval`, remove it. The unit in `:time-interval`+ takes precedence."+ [[_tag field :as clause]]+ (or (when (= (helpers/actual-clause-tag field) :field)+ (let [[_tag id-or-name field-opts] field]+ (when (:temporal-unit field-opts)+ (assoc clause 1 [:field id-or-name (not-empty (dissoc field-opts :temporal-unit))]))))+ clause))++(defclause* time-interval+ [:and+ (helpers/clause+ :time-interval+ "field" [:ref ::FieldOrExpressionRef]+ "n" [:or :int [:enum #:decode{:normalize #(cond-> % (string? %) keyword)} :current :last :next]]+ "unit" [:ref ::RelativeDatetimeUnit]+ "options" [:optional [:ref ::TimeIntervalOptions]])+ [:schema+ {:decode/normalize #'normalize-time-interval}+ :any]])(defmethod options-style-method :time-interval [_tag] ::options-style.last-unless-empty)-(defclause ^:sugar during- field FieldOrExpressionRef+(defclause during+ field [:ref ::FieldOrExpressionRef]value [:or ::lib.schema.literal/date ::lib.schema.literal/datetime]unit ::DateTimeUnit)-(defclause ^:sugar relative-time-interval- col FieldOrExpressionRef+(defclause relative-time-interval+ col [:ref ::FieldOrExpressionRef]value :intbucket [:ref ::RelativeDatetimeUnit]offset-value :int@@ -938,11 +1133,8 @@;;;; It can also be used for GA, which looks something like `[:segment "gaid::-11"]`. GA segments aren't actually MBQL;; segments and pass-thru to GA.-(def ^:private SegmentID- [:ref ::lib.schema.id/segment])--(defclause ^:sugar segment- segment-id [:or SegmentID ::lib.schema.common/non-blank-string])+(defclause segment+ segment-id [:or ::lib.schema.id/segment ::lib.schema.common/non-blank-string])(mr/def ::BooleanExpression(one-of@@ -952,6 +1144,7 @@in not-in does-not-contain inside is-empty not-empty is-null not-null relative-time-interval time-interval during))(mr/def ::Filter+ "Schema for a valid MBQL `:filter` clause."[:multi{:error/message "valid filter expression":dispatch (fn [x]@@ -963,43 +1156,52 @@(is-clause? :value x) :value(is-clause? :segment x) :segment:else :else))}- [:datetime DatetimeExpression]- [:numeric NumericExpression]- [:string StringExpression]- [:boolean BooleanExpression]- [:value value]- [:segment segment]- [:else FieldOrExpressionRef]])--(def ^:private CaseClause- [:tuple {:error/message ":case subclause"} Filter ExpressionArg])--(def ^:private CaseClauses- [:maybe [:sequential CaseClause]])--(def ^:private CaseOptions+ [:datetime [:ref ::DatetimeExpression]]+ [:numeric [:ref ::NumericExpression]]+ [:string [:ref ::StringExpression]]+ [:boolean [:ref ::BooleanExpression]]+ [:value [:ref ::value]]+ [:segment [:ref ::segment]]+ [:else [:ref ::FieldOrExpressionRef]]])++(mr/def ::CaseSubclause+ [:tuple {:error/message ":case subclause"+ :decode/normalize (fn [x]+ (when (sequential? x)+ ;; in some of the weird FE e2e tests `:case` has an empty third arg (unsure why), if+ ;; we see extra args just drop them.+ (into [] (take 2) x)))}+ [:ref ::Filter]+ [:ref ::ExpressionArg]])++(mr/def ::CaseSubclauses+ [:sequential {:min 1} ::CaseSubclause])++(mr/def ::CaseOptions[:map- {:error/message ":case options"}- [:default {:optional true} ExpressionArg]])+ {:decode/normalize lib.schema.common/normalize-map+ :error/message ":case options"}+ [:default {:optional true} [:ref ::ExpressionArg]]])-(defclause ^{:requires-features #{:basic-aggregations}} case- clauses CaseClauses, options (optional CaseOptions))+(defclause case+ clauses [:ref ::CaseSubclauses], options (optional [:ref ::CaseOptions]))(defmethod options-style-method :case [_tag] ::options-style.last-unless-empty)-(defclause ^:sugar ^{:requires-features #{:basic-aggregations}} [case:if if]- clauses CaseClauses, options (optional CaseOptions))+(defclause if+ clauses [:ref ::CaseSubclauses], options (optional [:ref ::CaseOptions]))(defmethod options-style-method :if [_tag] ::options-style.last-unless-empty)(mr/def ::NumericExpression- (one-of + - / * coalesce length floor ceil round abs power sqrt exp log case case:if datetime-diff integer float+ "Schema for the definition of a numeric expression. All numeric expressions evaluate to numeric values."+ (one-of + - / * coalesce length floor ceil round abs power sqrt exp log case if datetime-diff integer floattemporal-extract get-year get-quarter get-month get-week get-day get-day-of-weekget-hour get-minute get-secondaggregation))(mr/def ::StringExpression- (one-of substring trim ltrim rtrim replace lower upper concat regex-match-first coalesce case case:if host domain+ (one-of substring trim ltrim rtrim replace lower upper concat regex-match-first coalesce case if host domainsubdomain path month-name quarter-name day-name text split-part))(mr/def ::FieldOrExpressionDef@@ -1018,24 +1220,22 @@(is-clause? :offset x) :offset(is-clause? :value x) :value:else :else))}- [:numeric NumericExpression]- [:string StringExpression]- [:boolean BooleanExpression]- [:datetime DatetimeExpression]- [:case case]- [:if case:if]- [:offset offset]- [:value value]- [:else FieldOrExpressionRef]])--;;; -------------------------------------------------- Aggregations --------------------------------------------------+ [:numeric [:ref ::NumericExpression]]+ [:string [:ref ::StringExpression]]+ [:boolean [:ref ::BooleanExpression]]+ [:datetime [:ref ::DatetimeExpression]]+ [:case [:ref ::case]]+ [:if [:ref ::if]]+ [:offset [:ref ::offset]]+ [:value [:ref ::value]]+ [:else [:ref ::FieldOrExpressionRef]]]);; For all of the 'normal' Aggregations below (excluding Metrics) fields are implicit Field IDs;; cum-sum and cum-count are SUGAR because they're implemented in middleware. The clauses are swapped out with;; `count` and `sum` aggregations respectively and summation is done in Clojure-land-(defclause ^{:requires-features #{:basic-aggregations}} ^:sugar count, field (optional FieldOrExpressionRef))-(defclause ^{:requires-features #{:basic-aggregations}} ^:sugar cum-count, field (optional FieldOrExpressionRef))+(defclause count, field (optional [:ref ::FieldOrExpressionRef]))+(defclause cum-count, field (optional [:ref ::FieldOrExpressionRef]));; technically aggregations besides count can also accept expressions as args, e.g.;;@@ -1045,36 +1245,36 @@;;;; SUM(field_1 + field_2)-(defclause ^{:requires-features #{:basic-aggregations}} avg, field-or-expression [:ref ::FieldOrExpressionDef])-(defclause ^{:requires-features #{:basic-aggregations}} cum-sum, field-or-expression [:ref ::FieldOrExpressionDef])-(defclause ^{:requires-features #{:basic-aggregations}} distinct, field-or-expression [:ref ::FieldOrExpressionDef])-(defclause ^{:requires-features #{:basic-aggregations}} sum, field-or-expression [:ref ::FieldOrExpressionDef])-(defclause ^{:requires-features #{:basic-aggregations}} min, field-or-expression [:ref ::FieldOrExpressionDef])-(defclause ^{:requires-features #{:basic-aggregations}} max, field-or-expression [:ref ::FieldOrExpressionDef])+(defclause avg, field-or-expression [:ref ::FieldOrExpressionDef])+(defclause cum-sum, field-or-expression [:ref ::FieldOrExpressionDef])+(defclause distinct, field-or-expression [:ref ::FieldOrExpressionDef])+(defclause sum, field-or-expression [:ref ::FieldOrExpressionDef])+(defclause min, field-or-expression [:ref ::FieldOrExpressionDef])+(defclause max, field-or-expression [:ref ::FieldOrExpressionDef])-(defclause ^{:requires-features #{:distinct-where}} distinct-where- field-or-expression [:ref ::FieldOrExpressionDef], pred Filter)+(defclause distinct-where+ field-or-expression [:ref ::FieldOrExpressionDef], pred [:ref ::Filter])-(defclause ^{:requires-features #{:basic-aggregations}} sum-where- field-or-expression [:ref ::FieldOrExpressionDef], pred Filter)+(defclause sum-where+ field-or-expression [:ref ::FieldOrExpressionDef], pred [:ref ::Filter])-(defclause ^{:requires-features #{:basic-aggregations}} count-where- pred Filter)+(defclause count-where+ pred [:ref ::Filter])-(defclause ^{:requires-features #{:basic-aggregations}} share- pred Filter)+(defclause share+ pred [:ref ::Filter])-(defclause ^{:requires-features #{:standard-deviation-aggregations}} stddev+(defclause stddevfield-or-expression [:ref ::FieldOrExpressionDef])-(defclause ^{:requires-features #{:standard-deviation-aggregations}} [ag:var var]+(defclause varfield-or-expression [:ref ::FieldOrExpressionDef])-(defclause ^{:requires-features #{:percentile-aggregations}} median+(defclause medianfield-or-expression [:ref ::FieldOrExpressionDef])-(defclause ^{:requires-features #{:percentile-aggregations}} percentile- field-or-expression [:ref ::FieldOrExpressionDef], percentile NumericExpressionArg)+(defclause percentile+ field-or-expression [:ref ::FieldOrExpressionDef], percentile [:ref ::NumericExpressionArg]);;; V1 (Legacy) Metrics (which lived in their own table) do not exist anymore! A V2 Metric is just a subtype of a Card.(defclause metric@@ -1091,39 +1291,95 @@(if (is-clause? numeric-functions x):numeric-expression:else))}- [:numeric-expression NumericExpression]+ [:numeric-expression [:ref ::NumericExpression]][:else (one-of aggregation avg cum-sum distinct distinct-where stddev sum min max metric share count-where- sum-where case case:if median percentile ag:var cum-count count offset)]])+ sum-where case if median percentile var cum-count count offset)]])-(def ^:private UnnamedAggregation- ::UnnamedAggregation)--(def ^:private AggregationOptions+(mr/def ::AggregationOptionsOptions"Additional options for any aggregation clause when wrapping it in `:aggregation-options`."[:map- {:error/message ":aggregation-options options"}+ {:error/message ":aggregation-options options"+ :decode/normalize lib.schema.common/normalize-map};; name to use for this aggregation in the native query instead of the default name (e.g. `count`)[:name {:optional true} ::lib.schema.common/non-blank-string];; user-facing display name for this aggregation instead of the default one[:display-name {:optional true} ::lib.schema.common/non-blank-string]])-(defclause aggregation-options- aggregation UnnamedAggregation- options AggregationOptions)+(defclause* aggregation-options+ [:and+ [:schema+ {:decode/normalize (fn [x]+ ;; [:named <subclase> <name>] was the MBQL 3 (?) version of `:aggregation-options`+ ;;+ ;; [:named <subclause> <name> {:use-as-display-name? false}] means set `:name` instead of+ ;; `:display-name`+ (if (= (helpers/actual-clause-tag x) :named)+ (let [[_tag subclause display-name {:keys [use-as-display-name?], :or {use-as-display-name? true}}] x]+ [:aggregation-options subclause {(if use-as-display-name? :display-name :name) display-name}])+ x))}+ :any]+ (helpers/clause+ :aggregation-options+ "aggregation" [:ref ::UnnamedAggregation]+ "options" [:ref ::AggregationOptionsOptions])])(defmethod options-style-method :aggregation-options [_tag] ::options-style.last-always)(mr/def ::Aggregation+ "Schema for anything that is a valid `:aggregation` clause."[:multi{:error/message "aggregation clause or numeric expression":dispatch (fn [x]- (if (is-clause? :aggregation-options x)+ (if (is-clause? #{:aggregation-options :named} x):aggregation-options:unnamed-aggregation))}- [:aggregation-options aggregation-options]- [:unnamed-aggregation UnnamedAggregation]])--;;; ---------------------------------------------------- Order-By ----------------------------------------------------+ [:aggregation-options [:ref ::aggregation-options]]+ [:unnamed-aggregation [:ref ::UnnamedAggregation]]])++(defn- normalize-aggregations [x]+ (let [xs (cond+ ;; handle MBQL 1/2 :aggregation which could be just a single string or keyword like+ ;;+ ;; {:aggregation "COUNT"}+ ((some-fn simple-keyword? string?) x)+ [[x]]++ ;; anything else that is not sequential is invalid at this point.+ (not (sequential? x))+ nil++ (empty? x)+ nil++ ;; handle really messed up stuff like [:count :count] or [:rows :count]+ (and (not (sequential? (first x)))+ (every? (some-fn simple-keyword? string?) x))+ (into []+ (map (fn [subclause]+ [subclause]))+ x)++ ;; handle MBQL 2 aggregations when they are a single aggregation that is a vector, e.g.+ ;;+ ;; {:aggregation [:sum 10]} => {:aggregation [[:sum [:field 10 nil]]]}+ (and (not (sequential? (first x)))+ ((some-fn simple-keyword? string?) (first x)))+ [x]++ :else+ x)]+ ;; {:aggregation "ROWS"} was the default in MBQL 1 + 2 for when we had no aggregations... I (Cam) removed it in+ ;; MBQL 3. Strip these out.+ (not-empty+ (remove+ #(= (helpers/actual-clause-tag %) :rows)+ xs))))++(mr/def ::Aggregations+ [:sequential+ {:min 1+ :decode/normalize #'normalize-aggregations}+ [:ref ::Aggregation]]);; order-by is just a series of `[<direction> <field>]` clauses like;;@@ -1131,27 +1387,38 @@;;;; Field ID is implicit in these clauses-(defclause asc, field Reference)-(defclause desc, field Reference)+(defclause asc, field [:ref ::Reference])+(defclause desc, field [:ref ::Reference])(mr/def ::OrderBy"Schema for an `order-by` clause subclause."- (one-of asc desc))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | Queries |-;;; +----------------------------------------------------------------------------------------------------------------+--;;; ---------------------------------------------- Native [Inner] Query ------------------------------------------------(def ^:private TemplateTagType+ [:schema+ {:decode/normalize (fn [subclause]+ ;; handle MBQL 2 clauses which looked like [10 "desc"] or [10 "descending"]+ (when (sequential? subclause)+ (let [[x y] subclause+ [x y] (if (and ((some-fn simple-keyword? string?) y)+ (#{:asc :desc :ascending :descending} (helpers/normalize-keyword y)))+ [(case (helpers/normalize-keyword y)+ (:asc :ascending) :asc+ (:desc :descending) :desc)+ x]+ [x y])]+ ;; handle unwrapped raw Field IDs+ [x (if (pos-int? y)+ [:field y nil]+ y)])))}+ (one-of asc desc)])++(mr/def ::TemplateTagType"Schema for valid values of template tag `:type`."- [:enum :snippet :card :dimension :number :text :date])+ [:enum {:decode/normalize keyword} :snippet :card :dimension :number :text :date])-(def ^:private TemplateTag:Common+(mr/def ::TemplateTag.Common"Things required by all template tag types."[:map- [:type TemplateTagType]+ {:decode/normalize lib.schema.common/normalize-map}+ [:type [:ref ::TemplateTagType]][:name ::lib.schema.common/non-blank-string][:display-name ::lib.schema.common/non-blank-string];; TODO -- `:id` is actually 100% required but we have a lot of tests that don't specify it because this constraint@@ -1166,12 +1433,12 @@;; :type :snippet;; :snippet-name "select";; :snippet-id 1}-(mr/def ::TemplateTag:Snippet+(mr/def ::TemplateTag.Snippet"Schema for a native query snippet template tag."[:merge- TemplateTag:Common+ ::TemplateTag.Common[:map- [:type [:= :snippet]]+ [:type [:= {:decode/normalize helpers/normalize-keyword} :snippet]][:snippet-name ::lib.schema.common/non-blank-string][:snippet-id ::lib.schema.id/snippet];; database to which this Snippet belongs. Doesn't always seen to be specified.@@ -1184,18 +1451,18 @@;; :display-name "#1635";; :type :card;; :card-id 1635}-(mr/def ::TemplateTag:SourceQuery+(mr/def ::TemplateTag.SourceQuery"Schema for a source query template tag."[:merge- TemplateTag:Common+ ::TemplateTag.Common[:map- [:type [:= :card]]+ [:type [:= {:decode/normalize helpers/normalize-keyword} :card]][:card-id ::lib.schema.id/card]]])-(def ^:private TemplateTag:Value:Common+(mr/def ::TemplateTag.Value.Common"Stuff shared between the Field filter and raw value template tag schemas."[:merge- TemplateTag:Common+ ::TemplateTag.Common[:map;; default value for this parameter[:default {:optional true} :any]@@ -1210,16 +1477,17 @@;; :type :dimension,;; :dimension [:field 4 nil];; :widget-type :date/all-options}-(mr/def ::TemplateTag:FieldFilter+(mr/def ::TemplateTag.FieldFilter"Schema for a field filter template tag."[:merge- TemplateTag:Value:Common+ ::TemplateTag.Value.Common[:map- [:type [:= :dimension]]- [:dimension field]- [:alias {:optional true} :string]+ [:type [:= {:decode/normalize helpers/normalize-keyword} :dimension]]+ [:dimension [:ref ::field]]+ [:alias {:optional true} :string][:widget-type+ {:default :category}[:ref{:description"which type of widget the frontend should show for this Field Filter; this also affects which parameter types@@ -1237,13 +1505,13 @@;; :type "temporal-unit",;; :name "unit",;; :display-name "Unit"}-(mr/def ::TemplateTag:TemporalUnit+(mr/def ::TemplateTag.TemporalUnit"Schema for a temporal unit template tag."[:merge- TemplateTag:Value:Common+ ::TemplateTag.Value.Common[:map- [:type [:= :temporal-unit]]- [:dimension field]+ [:type [:= {:decode/normalize helpers/normalize-keyword} :temporal-unit]]+ [:dimension [:ref ::field]][:alias {:optional true} :string]]]);; Example:@@ -1254,10 +1522,10 @@;; :type :number;; :required true;; :default "1"}-(mr/def ::TemplateTag:RawValue+(mr/def ::TemplateTag.RawValue"Schema for a raw value template tag."[:merge- TemplateTag:Value:Common+ ::TemplateTag.Value.Common[:map[:type[:ref@@ -1299,26 +1567,49 @@Field filters and raw values usually have their value specified by `:parameters`."[:multi- {:dispatch :type}- [:dimension [:ref ::TemplateTag:FieldFilter]]- [:snippet [:ref ::TemplateTag:Snippet]]- [:card [:ref ::TemplateTag:SourceQuery]]- [:temporal-unit [:ref ::TemplateTag:TemporalUnit]]- [::mc/default [:ref ::TemplateTag:RawValue]]])--(def TemplateTag- "Alias for ::TemplateTag; prefer that going forward."- [:ref ::TemplateTag])+ {:dispatch (comp keyword :type)}+ [:dimension [:ref ::TemplateTag.FieldFilter]]+ [:snippet [:ref ::TemplateTag.Snippet]]+ [:card [:ref ::TemplateTag.SourceQuery]]+ [:temporal-unit [:ref ::TemplateTag.TemporalUnit]]+ [::mc/default [:ref ::TemplateTag.RawValue]]])(mr/def ::TemplateTagMap"Schema for the `:template-tags` map passed in as part of a native query.Map of template tag name -> template tag definition"[:and- [:map-of ::lib.schema.common/non-blank-string TemplateTag]+ [:map-of+ {:decode/normalize (fn [m]+ (when (and (map? m)+ (seq m))+ (update-keys m (fn [k]+ (cond-> k+ (keyword? k) u/qualified-name)))))}+ ::lib.schema.common/non-blank-string+ [:ref ::TemplateTag]][:ref ::lib.schema.template-tag/template-tag-map.validate-names]])-(def ^:private NativeQuery:Common+(defn- remove-empty-keys [m {:keys [non-empty-keys non-nil-keys]}]+ (when (map? m)+ (reduce-kv+ (fn [m k v]+ (if (or (and (non-empty-keys k)+ (empty? v))+ (and (non-nil-keys k)+ (nil? v)))+ (dissoc m k)+ m))+ m+ m)))++(defn- remove-empty-keys-from-native-inner-query [m]+ (when (map? m)+ (let [m (lib.schema.common/normalize-map m)]+ (remove-empty-keys m {:non-empty-keys #{:template-tags}+ :non-nil-keys #{:collection}}))))++(mr/def ::NativeQuery.Common[:and[:map[:template-tags {:optional true} [:ref ::TemplateTagMap]]@@ -1330,26 +1621,23 @@:source-table ":source-table is only allowed in MBQL inner queries.":fields ":fields is only allowed in MBQL inner queries."})])-(def NativeQuery+(mr/def ::NativeQuery"Schema for a valid, normalized native [inner] query."[:merge- NativeQuery:Common+ {:decode/normalize #'remove-empty-keys-from-native-inner-query}+ ::NativeQuery.Common[:map[:query :some]]])(mr/def ::NativeSourceQuery[:merge- NativeQuery:Common+ {:decode/normalize #'remove-empty-keys-from-native-inner-query}+ ::NativeQuery.Common[:map[:native :some]]])-;;; ----------------------------------------------- MBQL [Inner] Query -------------------------------------------------(def MBQLQuery- "Schema for a valid, normalized MBQL [inner] query."- [:ref ::MBQLQuery])-(mr/def ::SourceQuery+ "Schema for a valid value for a `:source-query`."[:multi{:dispatch (fn [x](if ((every-pred map? :native) x)@@ -1359,13 +1647,9 @@;; `:query` for reasons I do not fully remember (perhaps to make it easier to differentiate them from MBQL source;; queries).[:native [:ref ::NativeSourceQuery]]- [:mbql MBQLQuery]])+ [:mbql [:ref ::MBQLQuery]]])-(def SourceQuery- "Schema for a valid value for a `:source-query` clause."- [:ref ::SourceQuery])--(defn- lib-normalize-legacy-column+(defn- normalize-legacy-column"Normalize legacy column metadata when using [[metabase.lib.normalize/normalize]]."[m](when (map? m)@@ -1375,8 +1659,8 @@(dissoc :ident);; set `display_name` to `name` if it's unset.(as-> $m (cond-> $m- (core/and (:name $m)- (core/not (contains? $m :display_name)))+ (and (:name $m)+ (not (contains? $m :display_name)))(assoc :display_name (:name $m)))))))(mr/def ::legacy-column-metadata.binning-info@@ -1386,8 +1670,8 @@(when (map? m)(let [m (lib.schema.common/normalize-map-no-kebab-case m)](cond-> m- (core/and (:binning_strategy m)- (core/not (:strategy m)))+ (and (:binning_strategy m)+ (not (:strategy m)))(assoc :strategy (:binning_strategy m))))))}[:strategy [:ref ::lib.schema.binning/strategy]][:binning_strategy {:optional true} [:ref ::lib.schema.binning/strategy]]@@ -1396,13 +1680,13 @@[:fn{:error/message "bin_width is a required key when strategy is bin-width"}(fn [m]- (if (core/= (:strategy m) :bin-width)+ (if (= (:strategy m) :bin-width)(contains? m :bin_width)true))][:fn{:error/message "num_bins is a required key when strategy is num-bins"}(fn [m]- (if (core/= (:strategy m) :num-bins)+ (if (= (:strategy m) :num-bins)(contains? m :num_bins)true))][:fn@@ -1413,7 +1697,7 @@(assoc :binning_strategy (:strategy m))))}(fn [m](if (:binning_strategy m)- (core/= (:binning_strategy m) (:strategy m))+ (= (:binning_strategy m) (:strategy m))true))]])(defn- legacy-column-metadata-qualified-keys-schema@@ -1422,10 +1706,10 @@[](let [schema (mr/resolve-schema ::lib.schema.metadata/column)find-map-schema (fn find-map-schema [schema]- (if (core/= (mc/type schema) :map)+ (if (= (mc/type schema) :map)schema(some find-map-schema (mc/children schema))))- map-schema (find-map-schema schema)]+ map-schema (find-map-schema schema)](into [:map](keep (fn [[k opts schema]](when (qualified-keyword? k)@@ -1444,10 +1728,11 @@[:merge[:map;; this schema is allowed for Card `result_metadata` in Lib so `:decode/normalize` is used for those Lib use cases.- {:decode/normalize lib-normalize-legacy-column}+ {:decode/normalize #'normalize-legacy-column}[:base_type {:default :type/*} ::lib.schema.common/base-type][:display_name :string][:name :string]+ [:description {:optional true} [:maybe :string]][:binning_info {:optional true} [:maybe [:ref ::legacy-column-metadata.binning-info]]][:effective_type {:optional true} ::lib.schema.common/base-type][:converted_timezone {:optional true} [:maybe [:ref ::lib.schema.expression.temporal/timezone-id]]]@@ -1462,13 +1747,14 @@[:visibility_type {:optional true} [:maybe [:ref ::lib.schema.metadata/column.visibility-type]]]][:ref ::legacy-column-metadata.qualified-keys]](lib.schema.common/disallowed-keys- {:lib/type "Legacy results metadata should not have :lib/type, use :metabase.lib.schema.metadata/column for Lib metadata"})+ {:lib/type "Legacy results metadata should not have :lib/type, use :metabase.lib.schema.metadata/column for Lib metadata"+ :model/inner_ident ":model/inner_ident is deprecated"})(letfn [(disallowed-key? [k]- (core/or (core/not (keyword? k))- (let [disallowed-char (if (qualified-keyword? k)- "_"- "-")]- (str/includes? (str k) disallowed-char))))]+ (or (not (keyword? k))+ (let [disallowed-char (if (qualified-keyword? k)+ "_"+ "-")]+ (str/includes? (str k) disallowed-char))))][:fn{:error/message "legacy source query metadata should use snake_case keys (except for namespaced lib keys, which should use kebab-case)":error/fn (fn [{m :value} _]@@ -1476,14 +1762,14 @@(when (map? m)(into #{} (filter disallowed-key?) (keys m)))))}(fn [m]- (core/and (map? m)- (every? (complement disallowed-key?) (keys m))))])])+ (and (map? m)+ (every? (complement disallowed-key?) (keys m))))])])(def source-table-card-id-regex"Pattern that matches `card__id` strings that can be used as the `:source-table` of MBQL queries."#"^card__[1-9]\d*$")-(def ^:private SourceTable+(mr/def ::SourceTable"Schema for a valid value for the `:source-table` clause of an MBQL query."[:or::lib.schema.id/table@@ -1492,25 +1778,11 @@:description "`card__<id>` string Table ID"}source-table-card-id-regex]])-(def join-strategies- "Valid values of the `:strategy` key in a join map."- #{:left-join :right-join :inner-join :full-join})--(def JoinStrategy- "Strategy that should be used to perform the equivalent of a SQL `JOIN` against another table or a nested query.- These correspond 1:1 to features of the same name in driver features lists; e.g. you should check that the current- driver supports `:full-join` before generating a Join clause using that strategy."- (into [:enum] join-strategies))--(def Fields- "Schema for valid values of the MBQL `:fields` clause."- [:ref ::Fields])--(def ^:private JoinFields+(mr/def ::JoinFields[:or{:error/message "Valid join `:fields`: `:all`, `:none`, or a sequence of `:field` clauses that have `:join-alias`."}- [:enum :all :none]- Fields])+ [:enum {:decode/normalize #(cond-> % (string? %) keyword)} :all :none]+ [:ref ::Fields]])(mr/def ::Join"Perform the equivalent of a SQL `JOIN` with another Table or nested `:source-query`. JOINs are either explicitly@@ -1527,13 +1799,14 @@[:field \"my_field\" {:base-type :field/Integer, :join-alias \"my_join_alias\"}]"[:and[:map+ {:decode/normalize lib.schema.common/normalize-map}[:source-table{:optional true:description "*What* to JOIN. Self-joins can be done by using the same `:source-table` as in the query wherethis is specified. YOU MUST SUPPLY EITHER `:source-table` OR `:source-query`, BUT NOT BOTH!"}- SourceTable]+ [:ref ::SourceTable]]- [:source-query {:optional true} SourceQuery]+ [:source-query {:optional true} [:ref ::SourceQuery]][:condition{:description@@ -1541,14 +1814,14 @@JOINs this is usually something like[:= <source-table-fk-field> [:field <dest-table-pk-field> {:join-alias <join-table-alias>}]]"}- Filter]+ [:ref ::Filter]][:strategy{:optional true:description "Defaults to `:left-join`; used for all automatically-generated JOINsDriver implementations: this is guaranteed to be present after pre-processing."}- JoinStrategy]+ [:ref ::lib.schema.join/strategy]][:fields{:optional true@@ -1567,7 +1840,7 @@Driver implementations: you can ignore this clause. Relevant fields will be added to top-level `:fields` clause withappropriate aliases."}- JoinFields]+ [:ref ::JoinFields]][:alias{:optional true@@ -1608,14 +1881,10 @@:expressions "A join should not have top-level 'inner' query keys like :expressions":joins "A join should not have top-level 'inner' query keys like :joins"})])-(def Join- "Alias for ::Join. Prefer that going forward."- [:ref ::Join])-(mr/def ::Joins"Schema for a valid sequence of `Join`s. Must be a non-empty sequence, and `:alias`, if specified, must be unique."[:and- (helpers/non-empty [:sequential Join])+ (helpers/non-empty [:sequential [:ref ::Join]])[:fn{:error/message "All join aliases must be unique."}#(helpers/empty-or-distinct? (filter some? (map :alias %)))]])@@ -1623,49 +1892,104 @@(mr/def ::Fields[:schema{:error/message "Distinct, non-empty sequence of Field clauses"}- (helpers/distinct [:sequential {:min 1} FieldOrExpressionRef])])+ (helpers/distinct [:sequential {:min 1} [:ref ::FieldOrExpressionRef]])])(mr/def ::OrderBys(helpers/distinct [:sequential {:min 1} [:ref ::OrderBy]]))+(mr/def ::Breakouts+ [:sequential+ {:min 1+ :decode/normalize (fn [x]+ ;; handle MBQL 2 (?) where we had one single breakout e.g.+ ;;+ ;; {:breakout 10} => {:breakout [[:field 10 nil]]}+ ;; {:breakout [:field-id 3]} => {:breakout [[:field 10 nil]]}+ (cond+ (pos-int? x)+ [[:field x nil]]++ (and (sequential? x)+ ((some-fn simple-keyword? string?) (first x)))+ [x]++ :else+ x))}+ [:ref ::FieldOrExpressionRef]])++(defn- remove-empty-keys-from-mbql-inner-query [query]+ (remove-empty-keys query {:non-empty-keys #{:aggregation :breakout :expressions :fields :filter :order-by :joins}+ :non-nil-keys #{:limit}}))++(mr/def ::Expressions+ [:map-of+ [:ref ::ExpressionName]+ [:ref ::FieldOrExpressionDef]])++(mr/def ::RemoveFieldRefsFromFieldsAlreadyInBreakout+ (letfn [(without-temporal-unit [a-ref]+ (if (= (helpers/actual-clause-tag a-ref) :field)+ (let [[_tag id-or-name opts] a-ref]+ [:field id-or-name (not-empty (dissoc opts :temporal-unit))])+ a-ref))]+ [:fn+ {:error/message "Fields specified in `:breakout` should not be specified in `:fields`; this is implied."+ :decode/normalize (fn [{:keys [breakout fields], :as query}]+ (let [breakout-fields (into #{} (map without-temporal-unit) breakout)]+ (u/assoc-dissoc query :fields (not-empty+ (into []+ (remove (fn [a-ref]+ (breakout-fields (without-temporal-unit a-ref))))+ fields)))))}+ (fn [{:keys [breakout fields]}]+ (empty? (set/intersection+ (into #{} (map without-temporal-unit) breakout)+ (into #{} (map without-temporal-unit) fields))))]))+(mr/def ::MBQLQuery[:and[:map- [:source-query {:optional true} SourceQuery]- [:source-table {:optional true} SourceTable]- [:aggregation {:optional true} [:sequential {:min 1} Aggregation]]- [:breakout {:optional true} [:sequential {:min 1} FieldOrExpressionRef]]- [:expressions {:optional true} [:map-of ::lib.schema.common/non-blank-string [:ref ::FieldOrExpressionDef]]]- [:fields {:optional true} Fields]- [:filter {:optional true} Filter]+ {:decode/normalize lib.schema.common/normalize-map}+ [:source-query {:optional true} [:ref ::SourceQuery]]+ [:source-table {:optional true} [:ref ::SourceTable]]+ [:aggregation {:optional true} [:ref ::Aggregations]]+ [:breakout {:optional true} [:ref ::Breakouts]]+ [:expressions {:optional true} [:ref ::Expressions]]+ [:fields {:optional true} [:ref ::Fields]]+ [:filter {:optional true} [:ref ::Filter]][:limit {:optional true} ::lib.schema.common/int-greater-than-or-equal-to-zero][:order-by {:optional true} [:ref ::OrderBys]][:page {:optional true} [:ref :metabase.lib.schema/page]][:joins {:optional true} [:ref ::Joins]][:source-metadata- {:optional true+ {:optional true:description "Info about the columns of the source query. Added in automatically by middleware. This metadata isprimarily used to let power things like binning when used with Field Literals instead of normal Fields."}[:maybe [:sequential [:ref ::legacy-column-metadata]]]]]+ ;; remove empty query keys; this is done AFTER the map schema above because normalizing things like+ ;; `::Aggregations` will remove things like the `ROWS` aggregation which was removed in MBQL 4.+ ;; e.g. the schema above will normalize+ ;;+ ;; {:aggregation "ROWS"} => {:aggregation nil}+ ;;+ ;; but not actually remove that key; so we need this second pass to remove it.+ [:schema+ {:decode/normalize #'remove-empty-keys-from-mbql-inner-query}+ :map];;;; CONSTRAINTS;;[:fn{:error/message "Query must specify either `:source-table` or `:source-query`, but not both."}(fn [query]- (core/= 1 (core/count (select-keys query [:source-query :source-table]))))]- [:fn- {:error/message "Fields specified in `:breakout` should not be specified in `:fields`; this is implied."}- (fn [{:keys [breakout fields]}]- (empty? (set/intersection (set breakout) (set fields))))]+ (= 1 (count (select-keys query [:source-query :source-table]))))]+ [:ref ::RemoveFieldRefsFromFieldsAlreadyInBreakout](lib.schema.common/disallowed-keys{:lib/type "Legacy MBQL inner queries must not have :lib/type":type "An inner query must not include :type, this will cause us to mix it up with an outer query"})])-;;; ----------------------------------------------------- Params ------------------------------------------------------(mr/def ::WidgetType- "Schema for valid values of `:widget-type` for a [[::TemplateTag:FieldFilter]]."+ "Schema for valid values of `:widget-type` for a [[::TemplateTag.FieldFilter]]."[:ref :metabase.lib.schema.parameter/widget-type]);; this is the reference like [:template-tag <whatever>], not the [[TemplateTag]] schema for when it's declared in@@ -1676,24 +2000,18 @@[:map[:id ::lib.schema.common/non-blank-string]]])-(mr/def ::dimension+(defclause* dimension[:and[:fn {:error/message "must be a `:dimension` clause"} (partial helpers/is-clause? :dimension)][:catn[:tag [:= :dimension]]- [:target [:schema [:or [:ref ::field-or-expression-ref] [:ref ::template-tag]]]]+ [:target [:schema [:or [:ref ::FieldOrExpressionRef] [:ref ::template-tag]]]][:options [:? [:maybe [:map {:error/message "dimension options"} [:stage-number {:optional true} :int]]]]]]])-(def dimension- "Schema for a valid dimension clause."- (with-meta [:ref ::dimension] {:clause-name :dimension}))-(defmethod options-style-method :dimension [_tag] ::options-style.last-unless-empty)(defclause variable- target template-tag)--;;; --------------------------------------------- Metabase [Outer] Query ---------------------------------------------+ target [:ref ::template-tag]);; To the reader: yes, this seems sort of hacky, but one of the goals of the Nested Query Initiative™ was to minimize;; if not completely eliminate any changes to the frontend. After experimenting with several possible ways to do this@@ -1711,7 +2029,7 @@[:ref ::lib.schema.id/database]]);;; Make sure we have the combo of query `:type` and `:native`/`:query`-(mr/def ::check-keys-for-query-type+(mr/def ::CheckKeysForQueryType[:and[:fn{:error/message "Query must specify at most one of `:native` or `:query`, but not both."}@@ -1720,12 +2038,12 @@[:fn{:error/message "Native queries must not specify `:query`; MBQL queries must not specify `:native`."}(fn [{native :native, mbql :query, query-type :type}]- (core/case query-type- :native (core/not mbql)- :query (core/not native)+ (case query-type+ :native (not mbql)+ :query (not native)false))]])-(mr/def ::check-query-does-not-have-source-metadata+(mr/def ::CheckQueryDoesNotHaveSourceMetadata"`:source-metadata` is added to queries when `card__id` source queries are resolved. It contains info about thecolumns in the source query.@@ -1735,25 +2053,46 @@This should automatically be fixed by `normalize`; if we encounter it, it means some middleware is not functioningproperly."- (lib.schema.common/disallowed-keys- {:source-metadata "`:source-metadata` should be added in the same level as `:source-query` (i.e., the 'inner' MBQL query.)"}))--(def Query- "Schema for an [outer] query, e.g. the sort of thing you'd pass to the query processor or save in `Card.dataset_query`."- [:ref ::Query])+ [:fn+ {:error/message "`:source-metadata` should be added in the same level as `:source-query` (i.e., the 'inner' MBQL query.)"+ :decode/normalize (fn [m]+ (when (map? m)+ (cond-> m+ (:source-metadata m)+ (-> (dissoc :source-metadata)+ (assoc-in [:query :source-metadata] (:source-metadata m))))))}+ (complement :source-metadata)])++(defn- normalize-query [query]+ (when (map? query)+ (let [query (lib.schema.common/normalize-map query)]+ (reduce-kv+ (fn [m k v]+ ;; remove `:parameters` which is not allowed to be nil/empty+ (if (and (= k :parameters)+ (empty? v))+ (dissoc m :parameters)+ m))+ query+ query))))(mr/def ::Query[:and+ [:map+ {:decode/normalize #'normalize-query}]+ ;; need to move source metadata to the correct location FIRST so it gets normalized by the schema below+ [:ref ::CheckQueryDoesNotHaveSourceMetadata][:map[:database {:optional true} ::DatabaseID][:type[:enum- {:description "Type of query. `:query` = MBQL; `:native` = native."}+ {:decode/normalize helpers/normalize-keyword+ :description "Type of query. `:query` = MBQL; `:native` = native."}:query :native]]- [:native {:optional true} NativeQuery]- [:query {:optional true} MBQLQuery]+ [:native {:optional true} [:ref ::NativeQuery]]+ [:query {:optional true} [:ref ::MBQLQuery]][:parameters {:optional true} [:maybe [:ref ::lib.schema.parameter/parameters]]];;;; OPTIONS@@ -1779,8 +2118,7 @@[:update-row {:optional true} [:maybe [:ref ::lib.schema.actions/row]]]];;;; CONSTRAINTS- [:ref ::check-keys-for-query-type]- [:ref ::check-query-does-not-have-source-metadata]+ [:ref ::CheckKeysForQueryType](lib.schema.common/disallowed-keys{:lib/type "Legacy MBQL queries must not have :lib/type":source-table "An outer query must not include inner-query keys like :source-table; this might cause us to confuse it with an inner query"diff --git a/src/metabase/legacy_mbql/schema/helpers.cljc b/src/metabase/legacy_mbql/schema/helpers.cljcindex 1b1e19a62cf..63b7dda57a2 100644--- a/src/metabase/legacy_mbql/schema/helpers.cljc+++ b/src/metabase/legacy_mbql/schema/helpers.cljc@@ -4,20 +4,43 @@#?(:clj [metabase.util.performance :refer [for]])[clojure.string :as str][metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.util :as lib.util][metabase.types.core][metabase.util.malli.registry :as mr]))(comment metabase.types.core/keep-me)+(defn clause-registry-name+ "The schema name in the Malli registry for an MBQL 4 clause."+ [clause-name]+ (keyword "metabase.legacy-mbql.schema" (name clause-name)))++(defn defclause+ "Impl for [[defclause*]]."+ [clause-name schema]+ (assert (not (vector? clause-name)))+ (let [clause-name (keyword clause-name)+ registry-name (clause-registry-name clause-name)]+ (mr/def+ registry-name+ (lib.util/format "Schema for a valid MBQL 4 %s clause." clause-name)+ schema)))+(defn mbql-clause?"True if `x` is an MBQL clause (a sequence with a keyword as its first arg).Deprecated: Use [[metabase.lib.core/clause?]] going forward."- {:deprecated "0.57.0"}[x](and (sequential? x)(not (map-entry? x))- (keyword? (first x))))+ ((some-fn simple-keyword? string?) (first x))))++(defn normalize-keyword+ "Like [[lib.schema.common/normalize-keyword]] but also converts the keyword to lower-kebabcase (this is needed in+ legacy MBQL because MBQL 1 used uppercase/snake_case keywords in some cases; Lib does not accept MBQL 1 as an input+ directly (it is normalized to MBQL 4 first)."+ [x]+ (some-> x lib.schema.common/memoized-kebab-key))(defn is-clause?"If `x` is an MBQL clause, and an instance of clauses defined by keyword(s) `k-or-ks`?@@ -26,26 +49,13 @@(is-clause? #{:+ :- :* :/} [:+ 10 20]) ; -> trueDeprecated: use [[metabase.lib.core/clause-of-type?]] going forward."- {:deprecated "0.57.0"}[k-or-ks x](and(mbql-clause? x)- (if (coll? k-or-ks)- ((set k-or-ks) (first x))- (= k-or-ks (first x)))))--(defn check-clause- "Returns `x` if it's an instance of a clause defined by keyword(s) `k-or-ks`-- (check-clause :count [:count 10]) ; => [:count 10]- (check-clause? #{:+ :- :* :/} [:+ 10 20]) ; -> [:+ 10 20]- (check-clause :sum [:count 10]) ; => nil-- DEPRECATED: use [[metabase.lib.core/clause-of-type?]] going forward"- {:deprecated "0.57.0"}- [k-or-ks x]- (when (is-clause? k-or-ks x)- x))+ (let [k (normalize-keyword (first x))]+ (if (coll? k-or-ks)+ ((set k-or-ks) k)+ (= k-or-ks k)))))(defn- wrap-clause-arg-schema [arg-schema][:schema (if (qualified-keyword? arg-schema)@@ -66,40 +76,50 @@"Impl of [[metabase.legacy-mbql.schema.macros/defclause]] macro. Creates a Malli schema."[tag & arg-schemas][:and- {:description (str "schema for a valid legacy MBQL " :tag " clause")}+ {:description (str "schema for a valid MBQL 4 " tag " clause")}[:fn{:error/message (str "must be a `" tag "` clause"):decode/normalize (fn [x](when (and (sequential? x)- ((some-fn keyword? string?) (first x)))- (update (vec x) 0 keyword)))}+ ((some-fn simple-keyword? string?) (first x)))+ (update (vec x) 0 normalize-keyword)))}(partial is-clause? tag)](into[:catn- ["tag" [:= {:decode/normalize lib.schema.common/normalize-keyword} tag]]]+ ["tag" [:= {:decode/normalize normalize-keyword} tag]]](for [[arg-name arg-schema] (partition 2 arg-schemas)][arg-name (clause-arg-schema arg-schema)]))])-(defn- clause-tag [a-clause]- (when (and (vector? a-clause)- (keyword? (first a-clause)))- (first a-clause)))+(defn actual-clause-tag+ "Actual tag of the clause, even if it's one of the MBQL 3 tags removed in MBQL 4. In most cases you want to+ use [[effective-clause-tag]] instead."+ [a-clause]+ (when (mbql-clause? a-clause)+ (normalize-keyword (first a-clause))))++(defn effective-clause-tag+ "Corresponding MBQL 4 clause tag -- MBQL 3 clauses removed in MBQL 4 are mapped to the equivalent MBQL 4 clause tags."+ [a-clause]+ (when-let [tag (actual-clause-tag a-clause)]+ ;; MBQL 3 field clauses should all get treated as `:field`, which is the MBQL 4 equivalent of all of them.+ (case tag+ :field-literal :field+ :datetime-field :field+ :field-id :field+ :binning-strategy :field+ :fk-> :field+ :joined-field :field+ :named :aggregation-options+ tag)))(defn one-of*"Interal impl of `one-of` macro."- [& tags+schemas]+ [& schemas](into- [:multi {:dispatch clause-tag- :error/message (str "valid instance of one of these MBQL clauses: " (str/join ", " (map first tags+schemas)))- :doc/schema (into- [:or- {:description "valid instance of one of these MBQL clauses:"}]- (map second)- tags+schemas)}]- (for [[tag schema] tags+schemas]- [tag (if (qualified-keyword? schema)- [:ref schema]- schema)])))+ [:multi {:dispatch effective-clause-tag+ :error/message (str "valid instance of one of these MBQL clauses: " (str/join ", " (map name schemas)))}]+ (for [schema schemas]+ [(keyword (name schema)) [:ref schema]])))(defn non-empty"Add an addditonal constraint to `schema` (presumably an array) that requires it to be non-empty@@ -115,7 +135,8 @@[:andschema[:fn- {:error/message "non-empty"}+ {:error/message "non-empty"+ :decode/normalize not-empty}seq]]))(defn empty-or-distinct?@@ -127,8 +148,11 @@(mr/def ::distinct[:fn- {:description "values must be distinct"- :error/message "distinct"}+ {:description "values must be distinct"+ :error/message "distinct"+ :decode/normalize (fn [x]+ (when (sequential? x)+ (into [] (clojure.core/distinct) x)))}empty-or-distinct?])(defn distinctdiff --git a/src/metabase/legacy_mbql/schema/macros.clj b/src/metabase/legacy_mbql/schema/macros.cljindex 817b699ffdb..84c8782ebe6 100644--- a/src/metabase/legacy_mbql/schema/macros.clj+++ b/src/metabase/legacy_mbql/schema/macros.clj@@ -1,8 +1,7 @@(ns metabase.legacy-mbql.schema.macros(:refer-clojure :exclude [run!])(:require- [metabase.legacy-mbql.schema.helpers]- [metabase.util.malli.registry :as mr]+ [metabase.legacy-mbql.schema.helpers :as helpers][metabase.util.performance :refer [run!]]))(defn- stringify-names [arg-names-and-schemas]@@ -15,33 +14,26 @@schema)])))arg-names-and-schemas))+(defmacro defclause*+ "Like [[defclause]] but takes the schema to register directly instead of building it from arguments."+ [clause-name schema]+ `(helpers/defclause ~(keyword clause-name) ~schema))+(defmacro defclause- "Define a new MBQL clause.+ "Define a new MBQL clause. This builds and registers a schema like `:metabase.legacy-mbql.schema/clause-name`.- (defclause field-id, id su/IntGreaterThanZero)+ (defclause field-id, id ::lib.schema.id/field)The first arg is the name of the clause, and should be followed by pairs of arg name, arg schema. Arg schemas mayoptionally be wrapped in `optional` or `rest` to signify that the arg is optional, or to accept varargs:(defclause count, field (optional Field))- (defclause and, filters (rest Filter))-- Since there are some cases where clauses should be parsed differently in MBQL (such as expressions in the- `expressions` clause vs in aggregations), you can give the actual symbol produced by this macro a different name as- follows:-- (defclause [ag:+ +] ...) ; define symbol `ag:+` to be used for a `[:+ ...]` clause"+ (defclause and, filters (rest Filter))"[clause-name & arg-names-and-schemas]- (let [[symb-name clause-name] (if (vector? clause-name)- clause-name- [clause-name clause-name])- clause-name-kw (keyword clause-name)- clause-registry-name (keyword "metabase.legacy-mbql.schema" (name clause-name))]- `(do- (mr/register! ~clause-registry-name- (metabase.legacy-mbql.schema.helpers/clause ~clause-name-kw ~@(stringify-names arg-names-and-schemas)))- (def ~(vary-meta symb-name assoc :doc (format "Schema for a valid %s clause." clause-name))- (with-meta [:ref ~clause-registry-name] {:clause-name ~clause-name-kw})))))+ (let [clause-name-kw (keyword clause-name)]+ `(defclause*+ ~clause-name+ (helpers/clause ~clause-name-kw ~@(stringify-names arg-names-and-schemas)))))(defmacro one-of"Define a schema that accepts one of several different MBQL clauses.@@ -49,10 +41,7 @@(one-of field-id field-literal)"[& clauses](run! #(assert (symbol? %)) clauses)- `(metabase.legacy-mbql.schema.helpers/one-of*+ `(helpers/one-of*~@(for [clause clauses- ;; Ensure that the symbol we inline into the code has no metadata to reduce CLJS bundle size.- :let [sym (with-meta clause {})]]- [`(or (:clause-name (meta ~clause))- '~sym)- clause])))+ :let [k (helpers/clause-registry-name clause)]]+ k)))diff --git a/src/metabase/legacy_mbql/util.cljc b/src/metabase/legacy_mbql/util.cljcindex 5800087d4c8..b4be1575556 100644--- a/src/metabase/legacy_mbql/util.cljc+++ b/src/metabase/legacy_mbql/util.cljc@@ -10,7 +10,6 @@[clojure.string :as str][metabase.legacy-mbql.predicates :as mbql.preds][metabase.legacy-mbql.schema :as mbql.s]- [metabase.legacy-mbql.schema.helpers :as schema.helpers][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.ref :as lib.schema.ref]@@ -19,15 +18,46 @@[metabase.util.i18n :as i18n][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.namespaces :as shared.ns][metabase.util.performance :refer [some mapv every? #?(:clj for)]][metabase.util.time :as u.time]))-(shared.ns/import-fns- [schema.helpers- mbql-clause?- is-clause?- check-clause])+(defn mbql-clause?+ "True if `x` is an MBQL clause (a sequence with a keyword as its first arg).++ Deprecated: Use [[metabase.lib.core/clause?]] going forward."+ {:deprecated "0.57.0"}+ [x]+ (and (sequential? x)+ (not (map-entry? x))+ (keyword? (first x))))++(defn is-clause?+ "If `x` is an MBQL clause, and an instance of clauses defined by keyword(s) `k-or-ks`?++ (is-clause? :count [:count 10]) ; -> true+ (is-clause? #{:+ :- :* :/} [:+ 10 20]) ; -> true++ Deprecated: use [[metabase.lib.core/clause-of-type?]] going forward."+ {:deprecated "0.57.0"}+ [k-or-ks x]+ (and+ (mbql-clause? x)+ (if (coll? k-or-ks)+ ((set k-or-ks) (first x))+ (= k-or-ks (first x)))))++(defn check-clause+ "Returns `x` if it's an instance of a clause defined by keyword(s) `k-or-ks`++ (check-clause :count [:count 10]) ; => [:count 10]+ (check-clause? #{:+ :- :* :/} [:+ 10 20]) ; -> [:+ 10 20]+ (check-clause :sum [:count 10]) ; => nil++ DEPRECATED: use [[metabase.lib.core/clause-of-type?]] going forward"+ {:deprecated "0.57.0"}+ [k-or-ks x]+ (when (is-clause? k-or-ks x)+ x))(mu/defn normalize-token :- [:or :keyword :string]"Convert a string or keyword in various cases (`lisp-case`, `snake_case`, or `SCREAMING_SNAKE_CASE`) to a lisp-cased@@ -118,7 +148,7 @@;; simplify the elements of the vector(mapv simplify-compound-filter x)))))-(mu/defn combine-filter-clauses :- mbql.s/Filter+(mu/defn combine-filter-clauses :- ::mbql.s/Filter"Combine two filter clauses into a single clause in a way that minimizes slapping a bunch of `:and`s together ifpossible.@@ -456,7 +486,7 @@(cond-> clause(mbql.preds/FieldOrExpressionDef? clause) desugar-expression))-(mu/defn desugar-filter-clause :- mbql.s/Filter+(mu/defn desugar-filter-clause :- ::mbql.s/Filter"Rewrite various 'syntatic sugar' filter clauses like `:time-interval` and `:inside` as simpler, logicallyequivalent clauses. This can be used to simplify the number of filter clauses that need to be supported by anythingthat needs to enumerate all the possible filter types (such as driver query processor implementations, or the@@ -465,7 +495,7 @@DEPRECATED: This will be removed in a future release. Use [[metabase.lib.core/desugar-filter-clause]] instead goingforward."{:deprecated "0.57.0"}- [filter-clause :- mbql.s/Filter]+ [filter-clause :- ::mbql.s/Filter]#_{:clj-kondo/ignore [:deprecated-var]}(-> filter-clausedesugar-current-relative-datetime@@ -505,14 +535,14 @@(defmethod negate* :starts-with [clause] [:not clause])(defmethod negate* :ends-with [clause] [:not clause])-(mu/defn negate-filter-clause :- mbql.s/Filter+(mu/defn negate-filter-clause :- ::mbql.s/Filter"Return the logical compliment of an MBQL filter clause, generally without using `:not` (except for the stringfilter clause types). Useful for generating highly optimized filter clauses and for drivers that do not supporttop-level `:not` filter clauses.Deprecated: use [[metabase.lib.core/negate-boolean-expression]] going forward."{:deprecated "0.57.0"}- [filter-clause :- mbql.s/Filter]+ [filter-clause :- ::mbql.s/Filter]#_{:clj-kondo/ignore [:deprecated-var]}(-> filter-clause desugar-filter-clause negate* simplify-compound-filter))@@ -583,7 +613,7 @@([query index](aggregation-at-index query index 0))- ([query :- mbql.s/Query+ ([query :- ::mbql.s/Queryindex :- ::lib.schema.common/int-greater-than-or-equal-to-zeronesting-level :- ::lib.schema.common/int-greater-than-or-equal-to-zero](if (zero? nesting-level)@@ -645,12 +675,12 @@[[_ _ opts]]opts)-(mu/defn update-field-options :- mbql.s/Reference+(mu/defn update-field-options :- ::mbql.s/Reference"Like [[clojure.core/update]], but for the options in a `:field`, `:expression`, or `:aggregation` clause.DEPRECATED: Use MBQL 5 + [[metabase.lib.core/update-options]] going forward."{:arglists '([field-or-ag-ref-or-expression-ref f & args]), :deprecated "0.57.0"}- [[clause-type id-or-name opts] :- mbql.s/Reference f & args]+ [[clause-type id-or-name opts] :- ::mbql.s/Reference f & args](let [opts (not-empty (remove-empty (apply f opts args)))];; `:field` clauses should have a `nil` options map if there are no options. `:aggregation` and `:expression`;; should get the arg removed if it's `nil` or empty. (For now. In the future we may change this if we make thediff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljcindex 461664b30b7..5465afe936d 100644--- a/src/metabase/lib/field/resolution.cljc+++ b/src/metabase/lib/field/resolution.cljc@@ -614,7 +614,7 @@"Resolve a ref like `CATEGORY_2` to `CATEGORY` if the query only has the latter."[query stage-number id-or-name](when (string? id-or-name)- (when-let [[_match original-name suffix] (re-matches #"^(\w+)_([1-9][0-9]*)$" id-or-name)]+ (when-let [[_match original-name suffix] (re-matches #"^(\w+)_([1-9]\d*)$" id-or-name)](let [suffix (parse-long suffix)new-suffix (dec suffix);; e.g. `CATEGORY_3` becomes `CATEGORY_2`; `CATEGORY_2` becomes `CATEGORY`diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljcindex 3563ba4306b..9ea46f1217e 100644--- a/src/metabase/lib/schema.cljc+++ b/src/metabase/lib/schema.cljc@@ -69,9 +69,13 @@{:decode/normalize #(->> %normalize-stage-common;; filter out null :collection keys -- see #59675+ ;;+ ;; also filter out empty `:template-tags` maps.(m/filter-kv (fn [k v]- (not (and (= k :collection)- (nil? v))))))+ (case k+ :collection (some? v)+ :template-tags (seq v)+ true)))):encode/for-hashing #'common/encode-map-for-hashing}[:lib/type [:= {:decode/normalize common/normalize-keyword} :mbql.stage/native]];; the actual native query, depends on the underlying database. Could be a raw SQL string or something like that.diff --git a/src/metabase/lib/schema/common.cljc b/src/metabase/lib/schema/common.cljcindex df27ee626af..209eff123bc 100644--- a/src/metabase/lib/schema/common.cljc+++ b/src/metabase/lib/schema/common.cljc@@ -48,7 +48,13 @@(fn [k];; sanity check: make sure we're not accidentally using this on a base type(assert (not= k :type/Text))- (u/->kebab-case-en k))))+ ;; don't use [[u/->kebab-case-en]] here because it converts stuff like `:SuM` to `:su-m` which is not really what+ ;; we want+ (-> k+ u/qualified-name+ (str/replace #"_" "-")+ u/lower-case-en+ keyword))))(defn map->kebab-case"Convert a map to kebab case, for use with `:decode/normalize`."diff --git a/src/metabase/lib/schema/expression/string.cljc b/src/metabase/lib/schema/expression/string.cljcindex ce0cd309c48..e1403ed9ee0 100644--- a/src/metabase/lib/schema/expression/string.cljc+++ b/src/metabase/lib/schema/expression/string.cljc@@ -33,8 +33,8 @@#_replace [:schema :string])(mbql-clause/define-catn-mbql-clause :substring :- :type/Text- [:str [:schema [:ref ::expression/string]]]- [:start [:schema [:ref ::expression/integer]]]+ [:str [:schema [:ref ::expression/string]]]+ [:start [:schema [:ref ::expression/integer]]][:length [:? [:schema [:ref ::expression/integer]]]])(mbql-clause/define-tuple-mbql-clause :split-part :- :type/Textdiff --git a/src/metabase/lib/schema/expression/temporal.cljc b/src/metabase/lib/schema/expression/temporal.cljcindex f4eb0a7a40e..54fbaab08e8 100644--- a/src/metabase/lib/schema/expression/temporal.cljc+++ b/src/metabase/lib/schema/expression/temporal.cljc@@ -115,7 +115,7 @@[:cat[:merge::common/options- [:map [:mode {:optional true} ;; no mode should be iso string+ [:map [:mode {:optional true} ; no mode should be iso string(into [:enum {:decode/normalize normalize-datetime-mode}]datetime-string-modes)]]][:schema [:ref ::expression/string]]]@@ -134,7 +134,7 @@::common/options[:map [:mode (into [:enum {:decode/normalize normalize-datetime-mode}]datetime-binary-modes)]]]- :any]]])+ [:schema [:ref ::expression/expression]]]]]);; doesn't contain `:millisecond`(mr/def ::datetime-diff-unitdiff --git a/src/metabase/lib/schema/info.cljc b/src/metabase/lib/schema/info.cljcindex 53a6238e45f..5ad54586bf5 100644--- a/src/metabase/lib/schema/info.cljc+++ b/src/metabase/lib/schema/info.cljc@@ -8,8 +8,8 @@[metabase.lib.schema.metadata :as lib.schema.metadata][metabase.util.malli.registry :as mr]))-;;; Schema for `info.context`; used for informational purposes to record how a query was executed.(mr/def ::context+ "Schema for `info.context`; used for informational purposes to record how a query was executed."[:enum;; do not decode, since this should not get written to the app DB or come in from the REST API.{:decode/normalize identity}diff --git a/src/metabase/lib/schema/join.cljc b/src/metabase/lib/schema/join.cljcindex f3373576670..efb29219c8e 100644--- a/src/metabase/lib/schema/join.cljc+++ b/src/metabase/lib/schema/join.cljc@@ -5,9 +5,10 @@[metabase.lib.schema.common :as common][metabase.lib.schema.expression :as expression][metabase.lib.schema.util :as lib.schema.util]+ [metabase.util :as u][metabase.util.i18n :as i18n][metabase.util.malli.registry :as mr]- [metabase.util.performance :refer [mapv every?]]))+ [metabase.util.performance :refer [every? mapv]]))(mr/def ::fields"The Fields to include in the results *if* a top-level `:fields` clause *is not* specified. This can be either@@ -36,7 +37,8 @@`table__via__field`. You can specify this yourself if you need to reference a joined field with a `:join-alias` inthe options."[:schema- {:gen/fmap #(str % "-" (random-uuid))}+ {:gen/fmap #(str % "-" (random-uuid))+ :decode/normalize #(cond-> % (keyword? %) u/qualified-name)}::common/non-blank-string])(mr/def ::conditiondiff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljcindex ddbd03990c6..046e81c83bc 100644--- a/src/metabase/lib/schema/metadata.cljc+++ b/src/metabase/lib/schema/metadata.cljc@@ -571,10 +571,9 @@;; if this has `:lib/type` we know FOR SURE that it's lib-style metadata; but we should also be able;; to infer this fact automatically if it's using `kebab-case` keys. `:base-type` is required for both;; styles so look at that.- (let [col (lib.schema.common/normalize-map-no-kebab-case col)]- (if ((some-fn :lib/type :base-type) col)- :lib- :legacy)))}+ (if ((some-fn :lib/type #(get % "lib/type") :base-type #(get % "base-type")) col)+ :lib+ :legacy))}[:lib[:merge[:ref ::column]diff --git a/src/metabase/lib/schema/parameter.cljc b/src/metabase/lib/schema/parameter.cljcindex ffb4a243376..cb25a123b38 100644--- a/src/metabase/lib/schema/parameter.cljc+++ b/src/metabase/lib/schema/parameter.cljc@@ -207,20 +207,11 @@;;; update [[metabase.lib.convert]] to convert `:parameters` back and forth and add UUIDs and what not. But parameters;;; is not ported to MLv2 yet, so conversion isn't implemented YET.-(defn- normalize-legacy-ref [legacy-ref]- (if (pos-int? legacy-ref)- [:field legacy-ref nil]- ((#?(:clj requiring-resolve :cljs resolve) 'metabase.legacy-mbql.normalize/normalize-field-ref) legacy-ref)))-(mr/def ::target.legacy-field-ref- [:ref- {:decode/normalize normalize-legacy-ref}- :metabase.legacy-mbql.schema/field])+ [:ref :metabase.legacy-mbql.schema/field])(mr/def ::target.legacy-expression-ref- [:ref- {:decode/normalize normalize-legacy-ref}- :metabase.legacy-mbql.schema/expression])+ [:ref :metabase.legacy-mbql.schema/expression])(mr/def ::dimension.target[:multi {:dispatch lib.schema.common/mbql-clause-tagdiff --git a/src/metabase/lib/schema/template_tag.cljc b/src/metabase/lib/schema/template_tag.cljcindex 75081968738..9a372c0fc15 100644--- a/src/metabase/lib/schema/template_tag.cljc+++ b/src/metabase/lib/schema/template_tag.cljc@@ -134,7 +134,7 @@;; Valid values of `:type` for raw value template tags.(mr/def ::raw-value.type- (into [:enum] raw-value-template-tag-types))+ (into [:enum {:decode/normalize keyword}] raw-value-template-tag-types));; Example:;;diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.cljindex d2ceaa9f677..ba06f382ec0 100644--- a/src/metabase/models/interface.clj+++ b/src/metabase/models/interface.clj@@ -15,6 +15,7 @@[clojure.walk :as walk][medley.core :as m]^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]+ ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s][metabase.lib.binning :as lib.binning][metabase.lib.core :as lib][metabase.lib.normalize :as lib.normalize]@@ -314,12 +315,41 @@{:in encrypted-json-in:out cached-encrypted-json-out})+;;; TODO (Cam 10/27/25) -- this stuff should be moved into a different module instead of the general models interface,+;;; either `queries` or a new module along with [[metabase.models.visualization-settings]].+(mr/def ::viz-settings-ref+ "Apparently in some cases legacy viz settings keys can be wrapped in `[:ref ...]` e.g.++ [:ref [:field 1 nil]]"+ [:tuple+ {:decode/normalize vec}+ [:= {:decode/normalize keyword} :ref]+ [:ref ::mbql.s/Reference]])++(mr/def ::viz-settings-name+ "Apparently in some cases legacy viz settings keys can be wrapped in `[:ref ...]` e.g.++ [:ref [:field 1 nil]]"+ [:tuple+ {:decode/normalize vec}+ [:= {:decode/normalize keyword} :name]+ :string])+(defn normalize-visualization-settings"The frontend uses JSON-serialized versions of MBQL clauses as keys in `:column_settings`. This normalizes themto MBQL 4 clauses so things work correctly."[viz-settings](letfn [(normalize-column-settings-key [k]- (some-> k u/qualified-name json/decode mbql.normalize/normalize json/encode))+ (some-> k+ u/qualified-name+ json/decode+ ((fn [x]+ (cond+ (not (sequential? x)) x+ (= (first x) "ref") (lib/normalize ::viz-settings-ref x)+ (= (first x) "name") (lib/normalize ::viz-settings-name x)+ :else (mbql.normalize/normalize x))))+ json/encode))(normalize-column-settings [column-settings](into {} (for [[k v] column-settings][(normalize-column-settings-key k) (walk/keywordize-keys v)])))diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex c82db3876ae..dd705b7bb3b 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -1111,12 +1111,15 @@[form](mbql.normalize/is-clause? #{:field :field-id :fk-> :dimension :metric :segment} form))+(defn- normalize [mbql]+ (if-not (mbql-entity-reference? mbql)+ mbql+ (into [(keyword (first mbql))] (map normalize) (rest mbql))))+(defn- mbql-id->fully-qualified-name[mbql](-> mbql- ;; legacy usages -- do not use in new code- #_{:clj-kondo/ignore [:deprecated-var]}- mbql.normalize/normalize-tokens+ normalize(lib.util.match/replace;; `integer?` guard is here to make the operation idempotent[:field (id :guard integer?) opts]diff --git a/src/metabase/models/visualization_settings.cljc b/src/metabase/models/visualization_settings.cljcindex 3cdd30f47e4..0d91c9e5a6e 100644--- a/src/metabase/models/visualization_settings.cljc+++ b/src/metabase/models/visualization_settings.cljc@@ -22,10 +22,9 @@(:require#?@(:clj [[metabase.util.json :as json]])[clojure.set :as set]+ [clojure.walk :as walk][malli.core :as mc][medley.core :as m]- ;; legacy usage -- do not use in new code- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib.schema.id :as lib.schema.id][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]))@@ -367,6 +366,15 @@[mapping](= "dimension" (get-in mapping [:target :type])))+(defn- normalize [form]+ (walk/postwalk+ (fn [form]+ (if (and (sequential? form)+ (string? (first form)))+ (into [(keyword (first form))] (map normalize) (rest form))+ form))+ form))+(defn db->norm-param-mapping"Converts a `parameter-mapping` (i.e. value of `:parameterMapping`) from DB to normalized form"{:added "0.40.0"}@@ -380,9 +388,7 @@(if (dimension-param-mapping? v)(let [parsed-id (-> (if (keyword? k) (keyname k) k)parse-json-string- ;; legacy usage -- do not use in new code- #_{:clj-kondo/ignore [:deprecated-var]}- mbql.normalize/normalize-tokens)]+ normalize)][parsed-id (cond-> v(:source v) (assoc ::param-mapping-source(db->norm-param-ref parsed-id (:source v)))@@ -487,9 +493,9 @@[settings](reduce-kv (fn [m k v](try- (let [k1 (parse-db-column-ref k)- v1 (db->norm-column-settings-entries v)]- (assoc m k1 v1))+ (let [k' (parse-db-column-ref k)+ v' (db->norm-column-settings-entries v)]+ (assoc m k' v'))(catch #?(:clj Throwable :cljs js/Error) _em))){}diff --git a/src/metabase/queries/schema.clj b/src/metabase/queries/schema.cljindex 5176bb708ff..b0ed3e354ff 100644--- a/src/metabase/queries/schema.clj+++ b/src/metabase/queries/schema.clj@@ -6,6 +6,7 @@[metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.parameters.schema :as parameters.schema]+ [metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr][potemkin :as p]))@@ -36,6 +37,20 @@[:map[:lib/metadata ::lib.schema.metadata/metadata-provider]]]]]])+(mr/def ::card.result-metadata+ [:and+ [:sequential ::lib.schema.metadata/lib-or-legacy-column]+ ;; if the metadata could not be normalized into something valid, then just set it to `nil`. Ideally we shouldn't+ ;; have to do this -- we should try to fix flaws in metadata thru normalization if at all possible.+ [:schema+ {:decode/normalize (fn [xs]+ (if-not (mr/validate [:sequential ::lib.schema.metadata/lib-or-legacy-column] xs)+ (do+ (log/warn "Ignoring invalid Card result_metadata")+ nil)+ xs))}+ :any]])+;;; TODO (Cam 9/29/25) -- fill this out more, `:metabase.lib.schema.metadata/card` has a lot of stuff and there's also;;; stuff sprinkled thruout this module. For example [[metabase.queries.api.card/CardUpdateSchema]] should get merged;;; into this@@ -50,7 +65,7 @@[:parameters {:optional true} [:maybe [:ref ::parameters.schema/parameters]]][:parameter_mappings {:optional true} [:maybe [:ref ::parameters.schema/parameter-mappings]]][:type {:optional true} [:maybe ::lib.schema.metadata/card.type]]- [:result_metadata {:optional true} [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]]])+ [:result_metadata {:optional true} [:maybe [:ref ::card.result-metadata]]]])(mu/defn normalize-card :- [:maybe ::card]"Normalize a `card` so it satisfies the `::card` schema."diff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.cljindex 4dc5815a56e..d38906f3418 100644--- a/src/metabase/query_processor/api.clj+++ b/src/metabase/query_processor/api.clj@@ -6,15 +6,12 @@[metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.events.core :as events]- ;; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use- ;; Lib soon- ^{:clj-kondo/ignore [:discouraged-namespace]}- [metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.info :as lib.schema.info][metabase.model-persistence.core :as model-persistence]+ [metabase.models.interface :as mi][metabase.models.visualization-settings :as mb.viz][metabase.parameters.chain-filter :as chain-filter][metabase.parameters.custom-values :as custom-values]@@ -33,11 +30,9 @@[metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu]- ^{:clj-kondo/ignore [:discouraged-namespace]}- [metabase.util.malli.schema :as ms]+ ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.util.malli.schema :as ms][steffan-westcott.clj-otel.api.trace.span :as span]- ^{:clj-kondo/ignore [:discouraged-namespace]}- [toucan2.core :as t2]))+ ^{:clj-kondo/ignore [:discouraged-namespace]} [toucan2.core :as t2]));;; -------------------------------------------- Running a Query Normally --------------------------------------------@@ -145,7 +140,7 @@[:format_rows {:default false} ms/BooleanValue][:pivot_results {:default false} ms/BooleanValue]]](let [viz-settings (-> visualization-settings- (update :table.columns mbql.normalize/normalize)+ mi/normalize-visualization-settingsmb.viz/norm->db)query (-> query(assoc :viz-settings viz-settings)diff --git a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.cljindex 5ead9ac39cd..ef37ed0d824 100644--- a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj+++ b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj@@ -6,15 +6,14 @@which point they can use MLv2 directly)."(:require;; existing legacy usage -- don't use legacy MBQL namespaces in QP going forward- ^{:clj-kondo/ignore [:discouraged-namespace]}- [metabase.legacy-mbql.normalize :as mbql.normalize]+ ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]+ ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema][metabase.lib.schema.common :as lib.schema.common][metabase.query-processor.error-type :as qp.error-type]- ^{:clj-kondo/ignore [:deprecated-namespace]}- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.util.add-alias-info :as-alias add][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu]))@@ -34,7 +33,7 @@(lib/query-from-legacy-inner-query(qp.store/metadata-provider)(:id (lib.metadata/database (qp.store/metadata-provider)))- (mbql.normalize/normalize-fragment [:query] inner-query))+ (mbql.normalize/normalize ::mbql.s/MBQLQuery inner-query))(catch Throwable e(throw (ex-info (tru "Error converting query to pMBQL: {0}" (ex-message e)){:inner-query inner-query, :type qp.error-type/qp}diff --git a/src/metabase/query_processor/util.clj b/src/metabase/query_processor/util.cljindex 87a562cff05..86c28a67ac8 100644--- a/src/metabase/query_processor/util.clj+++ b/src/metabase/query_processor/util.clj@@ -8,8 +8,7 @@[metabase.driver :as driver];; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use;; Lib only soon- ^{:clj-kondo/ignore [:discouraged-namespace]}- [metabase.legacy-mbql.normalize :as mbql.normalize]+ ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s][metabase.lib-be.core][metabase.lib.core :as lib][metabase.query-processor.schema :as qp.schema]@@ -84,8 +83,8 @@(defn- field-normalizer{:deprecated "0.57.0"}[field]- #_{:clj-kondo/ignore [:deprecated-var]}- (let [[type id-or-name options] (mbql.normalize/normalize-tokens field)]+ (let [[type id-or-name options] (lib/normalize ::mbql.s/field field)]+ #_{:clj-kondo/ignore [:deprecated-var]}[type id-or-name (select-keys options field-options-for-identification)]));;; TODO (Cam 9/10/25) -- this logic is all wrong and needs to use Lib insteaddiff --git a/src/metabase/segments/api.clj b/src/metabase/segments/api.cljindex c81d0ec3398..0573a47d443 100644--- a/src/metabase/segments/api.clj+++ b/src/metabase/segments/api.clj@@ -1,11 +1,11 @@(ns metabase.segments.api"/api/segment endpoints."(:require- ;; TODO (Cam 10/1/25) -- update segments to persist MBQL 5[metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.events.core :as events]^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]+ ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s][metabase.models.interface :as mi][metabase.util :as u][metabase.util.log :as log]@@ -61,7 +61,7 @@:present #{:description :caveats :points_of_interest}:non-nil #{:archived :definition :name :show_in_getting_started});; TODO (Cam 10/1/25) -- update segments to persist MBQL 5- new-def (->> clean-body :definition #_{:clj-kondo/ignore [:deprecated-var]} (mbql.normalize/normalize-fragment [:query]))+ new-def (->> clean-body :definition (mbql.normalize/normalize ::mbql.s/MBQLQuery))new-body (merge(dissoc clean-body :revision_message)(when new-def {:definition new-def}))diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.cljindex 17e85e549c5..9e26ac23d26 100644--- a/src/metabase/segments/models/segment.clj+++ b/src/metabase/segments/models/segment.clj@@ -7,6 +7,7 @@[metabase.api.common :as api];; existing usages, do not use legacy MBQL utils in new code^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]+ ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]^{:clj-kondo/ignore [:discouraged-namespace :deprecated-namespace]} [metabase.legacy-mbql.util :as mbql.u][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]@@ -47,7 +48,7 @@[definition](when (seq definition);; TODO (Cam 10/1/25) -- update segments to persist MBQL 5- (u/prog1 #_{:clj-kondo/ignore [:deprecated-var]} (mbql.normalize/normalize-fragment [:query] definition)+ (u/prog1 #_{:clj-kondo/ignore [:deprecated-var]} (mbql.normalize/normalize ::mbql.s/MBQLQuery definition)(validate-segment-definition <>))))(def ^:private transform-segment-definitiondiff --git a/src/metabase/segments/schema.clj b/src/metabase/segments/schema.cljindex faaeda03f10..ca868e4b101 100644--- a/src/metabase/segments/schema.clj+++ b/src/metabase/segments/schema.clj@@ -6,5 +6,5 @@(mr/def ::segment[:map- [:filter {:optional true} [:maybe mbql.s/Filter]]- [:aggregation {:optional true} [:maybe [:sequential ::mbql.s/Aggregation]]]])+ [:filter {:optional true} [:maybe [:ref ::mbql.s/Filter]]]+ [:aggregation {:optional true} [:maybe [:sequential [:ref ::mbql.s/Aggregation]]]]])diff --git a/src/metabase/util/malli/registry.cljc b/src/metabase/util/malli/registry.cljcindex dbeeb7679dd..7ce9fba1d8e 100644--- a/src/metabase/util/malli/registry.cljc+++ b/src/metabase/util/malli/registry.cljc@@ -154,7 +154,6 @@([type schema]`(register! ~type ~schema))([type docstring schema]- (assert (string? docstring))`(metabase.util.malli.registry/def ~type~(macros/case:clj `(-with-doc ~schema ~docstring)diff --git a/src/metabase/xrays/domain_entities/specs.clj b/src/metabase/xrays/domain_entities/specs.cljindex 7aab35cc84f..ac081885b09 100644--- a/src/metabase/xrays/domain_entities/specs.clj+++ b/src/metabase/xrays/domain_entities/specs.clj@@ -1,21 +1,57 @@(ns metabase.xrays.domain-entities.specs(:require+ [clojure.walk :as walk][malli.core :as mc][malli.transform :as mtx][medley.core :as m];; legacy usages, do not use legacy MBQL stuff in new code.^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][metabase.util.yaml :as yaml]))+(mr/def ::xrays-dimension+ "X-rays has its own special `:dimension` psuedo-MBQL clause in templates; it's different from the `:dimension` clause+ we normally see in template tags/parameter targets."+ [:cat+ ;;; TODO (Cam 10/28/25) -- I think we should consider normalizing this key to something like `:xrays/dimension` to+ ;;; make sure it's clear that it's an X-Rays-only thing and that it's different from the standard MBQL `:dimension`+ ;;; clause+ [:= {:decode/normalize keyword} :dimension]+ :string+ [:? [:map+ {:decode/normalize lib.schema.common/normalize-map}+ [:temporal-unit {:optional true} [:keyword {:decode/normalize keyword}]]]]])++;;; it would be cool if there was a way to override `::mbql.s/dimension` and be able to use+;;; `::mbql.normalize/normalize` normally but I can't figure out how to make it work. So we'll have to normalize+;;; things the usual way and then go back and manually normalize our special dimension clauses.++(defn- normalize-xrays-dimension [x]+ (let [schema ::xrays-dimension+ decoder (mr/cached ::decoder schema #(mc/decoder schema (mtx/transformer {:name :normalize})))]+ (decoder x)))++(defn normalize-mbql-clause+ "Normalize the legacy MBQL fragment saved in X-Rays templates. This handles the special X-Rays version of `:dimension`+ clauses as well."+ [x]+ (walk/postwalk+ (fn [form]+ (if (and (sequential? form)+ (= (first form) "dimension"))+ (normalize-xrays-dimension form)+ form))+ (mbql.normalize/normalize x)))+(def MBQL"MBQL clause (ie. a vector starting with a keyword)"[:fn- {:decode/domain-entity-spec mbql.normalize/normalize- :decode/transform-spec mbql.normalize/normalize+ {:decode/domain-entity-spec normalize-mbql-clause+ :decode/transform-spec normalize-mbql-clause:error/message "valid MBQL clause"};;; TODO (Cam 10/10/25) -- update this to use [[metabase.lib.core/clause?]] once we convert X-Rays to MBQL 5#(and (vector? %) (keyword? (first %)))])@@ -116,7 +152,7 @@(mtx/transformermtx/string-transformermtx/json-transformer- (mtx/transformer {:name :domain-entity-spec}))))+ {:name :domain-entity-spec})))(def ^:private domain-entities-dir "domain_entity_specs/")diff --git a/src/metabase/xrays/transforms/specs.clj b/src/metabase/xrays/transforms/specs.cljindex 92f5514a512..185f3ec63d0 100644--- a/src/metabase/xrays/transforms/specs.clj+++ b/src/metabase/xrays/transforms/specs.clj@@ -1,16 +1,17 @@(ns metabase.xrays.transforms.specs(:require[malli.core :as mc]+ [malli.error :as me][malli.transform :as mtx][medley.core :as m];; legacy usages, do not use legacy MBQL stuff in new code.- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]+ [metabase.lib.schema.join :as lib.schema.join][metabase.lib.util.match :as lib.util.match][metabase.util :as u][metabase.util.malli.registry :as mr][metabase.util.yaml :as yaml]- [metabase.xrays.domain-entities.specs :refer [MBQL]]))+ [metabase.xrays.domain-entities.specs :as domain-entities.specs :refer [MBQL]]))(mr/def ::query"Schema for MBQL queries as handled by the X-Rays transforms code. (Currently, legacy queries; in the very near future@@ -39,7 +40,7 @@(defn- extract-dimensions[mbql]- (lib.util.match/match (mbql.normalize/normalize mbql) [:dimension dimension & _] dimension))+ (lib.util.match/match (domain-entities.specs/normalize-mbql-clause mbql) [:dimension dimension & _] dimension))(def ^:private ^{:arglists '([m])} stringify-keys(partial m/map-keys name))@@ -66,7 +67,7 @@(def ^:private JoinStrategy[:schema{:decode/transform-spec keyword}- mbql.s/JoinStrategy])+ [:ref ::lib.schema.join/strategy]])(def ^:private Joins[:sequential@@ -135,7 +136,14 @@(mtx/transformermtx/string-transformermtx/json-transformer- (mtx/transformer {:name :transform-spec}))))+ {:name :transform-spec})+ #_respond identity+ (fn raise [{:keys [value explain], :as _error}]+ (let [humanized (me/humanize explain)]+ (throw (ex-info (format "Failed to coerce Transform spec: %s" (pr-str humanized))+ {:original spec+ :coerced value+ :error humanized}))))))(def ^:private transforms-dir "transforms/")diff --git a/test/metabase/app_db/custom_migrations/metrics_v2_test.clj b/test/metabase/app_db/custom_migrations/metrics_v2_test.cljindex bd95ab0b743..7898841fe88 100644--- a/test/metabase/app_db/custom_migrations/metrics_v2_test.clj+++ b/test/metabase/app_db/custom_migrations/metrics_v2_test.clj@@ -148,10 +148,10 @@json/decode+kw)))))(def query-validator- (mr/validator mbql.s/MBQLQuery))+ (mr/validator ::mbql.s/MBQLQuery))(def query-explainer- (mr/explainer mbql.s/MBQLQuery))+ (mr/explainer ::mbql.s/MBQLQuery))(deftest migrate-metrics-to-v2-test(impl/test-migrations ["v51.2024-05-13T15:30:57" "v51.2024-05-13T16:00:00"] [migrate!]diff --git a/test/metabase/dashboards/api_test.clj b/test/metabase/dashboards/api_test.cljindex 30efbfd6234..db85cdfd5e9 100644--- a/test/metabase/dashboards/api_test.clj+++ b/test/metabase/dashboards/api_test.clj@@ -2666,31 +2666,31 @@(deftest ^:parallel dashcard->query-hashes-test(doseq [[dashcard expected][[{:card {:dataset_query (fake-query 1)}}- ["RNcJA21084AYlVPIqlxDxMILKsyU6iE9hiUDMYBIrUw="- "GsfPch7sQZ60fotaEuN6fghF82iX3dUHLlvWtkJXOU0="]]+ ["S7xKRDQIVA4k/rzNGAc6PyMCvMiYs2MTkAJK5gwBGHU="+ "F2yzgei1xfnQhNcakBq9c/q3lg0K9QDtWUHYOKGpBsM="]][{:card {:dataset_query (fake-query 2)}:series [{:dataset_query (fake-query 3)}{:dataset_query (fake-query 4)}]}- ["LyJKZ0rm2YyPWGXIfP/V6SFcHlRvdalSDD55qn/Avng="- "zYehm4Huu7AD/0GP6XpMhav7+/ystaIeKjax4fqSHUg="- "nDuISCLB6o8sToOR4GHQ/Po6X3A3nE7oPj+e1g2kfCE="- "/yzJQIPSdxWHnREe9F2orYWDbPS5ns8WOaIf6fGix1k="- "oqG69TgkGdzPFaxDFymLAVnb9iuyGnvlX0aiiL1Eojc="- "GKQrcQTNICx7QOYDQVHW7BIi7gB3/oBUWVel+AvOq0Q="]]]]+ ["Kwc08fDZ5ei23pNS0lwowiWGfBi10DWnTDWYy77Ytp8="+ "Vr+1nZAZnqppFuXuxTZSo4tD4QbKzHXF/zeXsYMw8sg="+ "LTCeTqzWT5nW1vzfopiEmat+O0dxfB493ufMf/XGRiU="+ "uYWx5Pm/jhmibG/8+DM44fiPeWTSs5YXYN159yOr1Vs="+ "SIMWVAbMGgvozr1bSIWol6uCKKUDEqCL469OsN4dlsI="+ "thIvyA4Q/suKFAQiq0xJJLtSML2i5UHgw9WoTOdRGs4="]]]](testing (pr-str dashcard)(is (= expected(base-64-encode-byte-arrays (#'api.dashboard/dashcard->query-hashes dashcard)))))))(deftest ^:parallel dashcards->query-hashes-test- (is (= ["RNcJA21084AYlVPIqlxDxMILKsyU6iE9hiUDMYBIrUw="- "GsfPch7sQZ60fotaEuN6fghF82iX3dUHLlvWtkJXOU0="- "LyJKZ0rm2YyPWGXIfP/V6SFcHlRvdalSDD55qn/Avng="- "zYehm4Huu7AD/0GP6XpMhav7+/ystaIeKjax4fqSHUg="- "nDuISCLB6o8sToOR4GHQ/Po6X3A3nE7oPj+e1g2kfCE="- "/yzJQIPSdxWHnREe9F2orYWDbPS5ns8WOaIf6fGix1k="- "oqG69TgkGdzPFaxDFymLAVnb9iuyGnvlX0aiiL1Eojc="- "GKQrcQTNICx7QOYDQVHW7BIi7gB3/oBUWVel+AvOq0Q="]+ (is (= ["S7xKRDQIVA4k/rzNGAc6PyMCvMiYs2MTkAJK5gwBGHU="+ "F2yzgei1xfnQhNcakBq9c/q3lg0K9QDtWUHYOKGpBsM="+ "Kwc08fDZ5ei23pNS0lwowiWGfBi10DWnTDWYy77Ytp8="+ "Vr+1nZAZnqppFuXuxTZSo4tD4QbKzHXF/zeXsYMw8sg="+ "LTCeTqzWT5nW1vzfopiEmat+O0dxfB493ufMf/XGRiU="+ "uYWx5Pm/jhmibG/8+DM44fiPeWTSs5YXYN159yOr1Vs="+ "SIMWVAbMGgvozr1bSIWol6uCKKUDEqCL469OsN4dlsI="+ "thIvyA4Q/suKFAQiq0xJJLtSML2i5UHgw9WoTOdRGs4="](base-64-encode-byte-arrays(#'api.dashboard/dashcards->query-hashes[{:card {:dataset_query (fake-query 1)}}@@ -2709,14 +2709,14 @@{:card {:dataset_query (fake-query 2)}:series [{:dataset_query (fake-query 3)}{:dataset_query (fake-query 4)}]}]- (into {} (for [[k v] {"RNcJA21084AYlVPIqlxDxMILKsyU6iE9hiUDMYBIrUw=" 111- "GsfPch7sQZ60fotaEuN6fghF82iX3dUHLlvWtkJXOU0=" 222- "LyJKZ0rm2YyPWGXIfP/V6SFcHlRvdalSDD55qn/Avng=" 333- "zYehm4Huu7AD/0GP6XpMhav7+/ystaIeKjax4fqSHUg=" 444- "nDuISCLB6o8sToOR4GHQ/Po6X3A3nE7oPj+e1g2kfCE=" 555- "/yzJQIPSdxWHnREe9F2orYWDbPS5ns8WOaIf6fGix1k=" 666- "oqG69TgkGdzPFaxDFymLAVnb9iuyGnvlX0aiiL1Eojc=" 777- "GKQrcQTNICx7QOYDQVHW7BIi7gB3/oBUWVel+AvOq0Q=" 888}]+ (into {} (for [[k v] {"S7xKRDQIVA4k/rzNGAc6PyMCvMiYs2MTkAJK5gwBGHU=" 111+ "F2yzgei1xfnQhNcakBq9c/q3lg0K9QDtWUHYOKGpBsM=" 222+ "Kwc08fDZ5ei23pNS0lwowiWGfBi10DWnTDWYy77Ytp8=" 333+ "Vr+1nZAZnqppFuXuxTZSo4tD4QbKzHXF/zeXsYMw8sg=" 444+ "LTCeTqzWT5nW1vzfopiEmat+O0dxfB493ufMf/XGRiU=" 555+ "uYWx5Pm/jhmibG/8+DM44fiPeWTSs5YXYN159yOr1Vs=" 666+ "SIMWVAbMGgvozr1bSIWol6uCKKUDEqCL469OsN4dlsI=" 777+ "thIvyA4Q/suKFAQiq0xJJLtSML2i5UHgw9WoTOdRGs4=" 888}][(mapv int (codec/base64-decode k)) v]))))));;; +----------------------------------------------------------------------------------------------------------------+diff --git a/test/metabase/legacy_mbql/normalize_test.cljc b/test/metabase/legacy_mbql/normalize_test.cljcindex 733835c87e3..015d7462cfe 100644--- a/test/metabase/legacy_mbql/normalize_test.cljc+++ b/test/metabase/legacy_mbql/normalize_test.cljc@@ -3,234 +3,296 @@(:require#?@(:cljs ([metabase.test-runner.assert-exprs.approximately-equal]))[clojure.set :as set]- [clojure.test :as t]- [metabase.legacy-mbql.normalize :as mbql.normalize]))+ [clojure.test :refer [are deftest is testing]]+ [metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.legacy-mbql.schema :as mbql.s]+ [metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint]))#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))-(defn- tests [f-symb f group->input->expected]+(defn- tests [_f-symb f group->input->expected](doseq [[group input->expected] group->input->expected]- (t/testing group+ (testing group(doseq [[input expected] input->expected]- (t/testing (str "\n" (pr-str (list f-symb input)))- (t/is (= expected- (f input))))))))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | NORMALIZE TOKENS |-;;; +----------------------------------------------------------------------------------------------------------------++ (testing (str "\n" (pr-str (list f input)))+ (is (= expected+ (f input))))))))(defn- normalize-tests"Convenience for generating a bunch normalization tests. Args should alternate a [[t/testing]] context string and mapsof input query -> expected normalized query."[& {:as group-and-input->expected-pairs}]- (tests 'normalize-tokens #'mbql.normalize/normalize-tokens group-and-input->expected-pairs))--(t/deftest ^:parallel normalize-tokens-test- (normalize-tests- "Query type should get normalized"- {{:type "NATIVE"}- {:type :native}}-- "native queries should NOT get normalized"- {{:type "NATIVE", :native {"QUERY" "SELECT COUNT(*) FROM CANS;"}}- {:type :native, :native {:query "SELECT COUNT(*) FROM CANS;"}}-- {:native {:query {:NAME "FAKE_QUERY"- :description "Theoretical fake query in a JSON-based query lang"}}}- {:native {:query {:NAME "FAKE_QUERY"- :description "Theoretical fake query in a JSON-based query lang"}}}}-- "METRICS shouldn't get normalized in some kind of wacky way"- {{:aggregation ["+" ["METRIC" 10] 1]}- {:aggregation [:+ [:metric 10] 1]}}-- "Nor should SEGMENTS"- {{:filter ["=" ["+" ["SEGMENT" 10] 1] 10]}- {:filter [:= [:+ [:segment 10] 1] 10]}}-- "field literals should be exempt too"- {{:order-by [[:desc [:field-literal "SALES_TAX" :type/Number]]]}- {:order-by [[:desc [:field-literal "SALES_TAX" :type/Number]]]}}-- "... but they should be converted to strings if passed in as a KW for some reason"- {{:order-by [[:desc ["field_literal" :SALES/TAX "type/Number"]]]}- {:order-by [[:desc [:field-literal "SALES/TAX" :type/Number]]]}}-- "modern :field clauses should get normalized"- {[:field 2 {"temporal-unit" "day"}]- [:field 2 {:temporal-unit :day}]-- [:field 2 {"inherited-temporal-unit" "day"}]- [:field 2 {:inherited-temporal-unit :day}]-- [:field 2 {"binning" {"strategy" "default"}}]- [:field 2 {:binning {:strategy :default}}]}-- ":value clauses should keep snake_case keys in the options"- ;; See https://github.com/metabase/metabase/issues/23354 for details- {[:value "some value" {:some_key "some key value"}]- [:value "some value" {:some_key "some key value"}]}-- ":value clauses should keep snake_case keys in the type info args"- ;; See https://github.com/metabase/metabase/issues/23354 for details- {[:value "some value" {"base_type" "type/Text"}]- [:value "some value" {:base_type :type/Text}]}-- "nil options in aggregation and expression references should be removed"- {[:aggregation 0 nil] [:aggregation 0]- [:expression "CE" nil] [:expression "CE"]}))--;;; -------------------------------------------------- aggregation -----------------------------------------------------(t/deftest ^:parallel normalize-aggregations-test+ (tests 'normalize #'mbql.normalize/normalize group-and-input->expected-pairs))++(deftest ^:parallel normalize-test+ (testing "Query type should get normalized"+ (is (= {:type :native}+ (mbql.normalize/normalize {:type "NATIVE"})))))++(deftest ^:parallel normalize-test-2+ (testing "native queries should NOT get normalized"+ (is (= {:type :native, :native {:query "SELECT COUNT(*) FROM CANS;"}}+ (mbql.normalize/normalize {:type "NATIVE", :native {"QUERY" "SELECT COUNT(*) FROM CANS;"}})))+ (let [query {:native {:query {:NAME "FAKE_QUERY"+ :description "Theoretical fake query in a JSON-based query lang"}}}]+ (is (= query+ (mbql.normalize/normalize query))))))++(deftest ^:parallel normalize-test-3+ (testing "METRICS shouldn't get normalized in some kind of wacky way"+ (is (= {:aggregation [[:+ [:metric 10] 1]]}+ (mbql.normalize/normalize ::mbql.s/MBQLQuery {:aggregation ["+" ["METRIC" 10] 1]})))))++(deftest ^:parallel normalize-test-4+ (testing "Nor should SEGMENTS"+ (is (= {:filter [:= [:+ [:segment 10] 1] 10]}+ (mbql.normalize/normalize ::mbql.s/MBQLQuery {:filter ["=" ["+" ["SEGMENT" 10] 1] 10]})))))++(deftest ^:parallel normalize-test-5+ (testing "field literals should be exempt too"+ (is (= {:order-by [[:desc [:field "SALES_TAX" {:base-type :type/Number}]]]}+ (mbql.normalize/normalize ::mbql.s/MBQLQuery {:order-by [[:desc [:field-literal "SALES_TAX" :type/Number]]]})))))++(deftest ^:parallel normalize-test-6+ (testing "... but they should be converted to strings if passed in as a KW for some reason"+ (is (= {:order-by [[:desc [:field "SALES/TAX" {:base-type :type/Number}]]]}+ (mbql.normalize/normalize ::mbql.s/MBQLQuery {:order-by [[:desc ["field_literal" :SALES/TAX "type/Number"]]]})))))++(deftest ^:parallel normalize-test-7+ (testing "modern :field clauses should get normalized"+ (are [x expected] (= expected+ (mbql.normalize/normalize ::mbql.s/field x))+ [:field 2 {"temporal-unit" "day"}]+ [:field 2 {:temporal-unit :day}]++ [:field 2 {"inherited-temporal-unit" "day"}]+ [:field 2 {:inherited-temporal-unit :day}]++ [:field 2 {"binning" {"strategy" "default"}}]+ [:field 2 {:binning {:strategy :default}}])))++(deftest ^:parallel normalize-test-8+ (testing ":value clauses should keep snake_case keys in the options (#23354)"+ (let [clause [:value "some value" {:some_key "some key value"}]]+ (is (= clause+ (mbql.normalize/normalize ::mbql.s/value [:value "some value" {:some_key "some key value"}]))))))++(deftest ^:parallel normalize-test-9+ (testing ":value clauses should keep snake_case keys in the type info args (#23354)"+ (is (= [:value "some value" {:base_type :type/Text}]+ (mbql.normalize/normalize ::mbql.s/value [:value "some value" {"base_type" "type/Text"}])))))++(deftest ^:parallel normalize-test-10+ (testing "nil options in aggregation and expression references should be removed"+ (is (= [:aggregation 0]+ (mbql.normalize/normalize ::mbql.s/aggregation [:aggregation 0 nil])+ (mbql.normalize/normalize ::mbql.s/aggregation [:aggregation 0 {}])))+ (is (= [:expression "CE"]+ (mbql.normalize/normalize ::mbql.s/expression [:expression "CE" nil])+ (mbql.normalize/normalize ::mbql.s/expression [:expression "CE" {}])))))++(deftest ^:parallel normalize-aggregations-test(normalize-tests"Legacy 'rows' aggregations"{{:query {"AGGREGATION" "ROWS"}}- {:query {:aggregation :rows}}+ {:query {}}{:query {"AGGREGATION" ["ROWS"]}}- {:query {:aggregation [:rows]}}}+ {:query {}}}))+(deftest ^:parallel normalize-aggregations-test-2+ (normalize-tests"Other uppercase tokens"{{:query {"AGGREGATION" ["COUNT" 10]}}- {:query {:aggregation [:count 10]}}+ {:query {:aggregation [[:count [:field 10 nil]]]}}{:query {"AGGREGATION" [["COUNT" 10]]}}- {:query {:aggregation [[:count 10]]}}}+ {:query {:aggregation [[:count [:field 10 nil]]]}}}))- "make sure we normalize ag tokens properly when there's wacky MBQL 95 ag syntax"+(deftest ^:parallel normalize-aggregations-test-3+ (normalize-tests+ "make sure we normalize ag tokens properly when there's wacky MBQL 2 ag syntax"{{:query {:aggregation ["rows" "count"]}}- {:query {:aggregation [:rows :count]}}+ {:query {:aggregation [[:count]]}}{:query {:aggregation ["count" "count"]}}- {:query {:aggregation [:count :count]}}}+ {:query {:aggregation [[:count] [:count]]}}}))+(deftest ^:parallel normalize-aggregations-test-4+ (normalize-tests"don't normalize names of expression refs!"- {{:query {:aggregation ["count" ["count" ["expression" "ABCDEF"]]]}}- {:query {:aggregation [:count [:count [:expression "ABCDEF"]]]}}}+ {{:query {:aggregation [["count"] ["count" ["expression" "ABCDEF"]]]}}+ {:query {:aggregation [[:count]+ [:count [:expression "ABCDEF"]]]}}}))+(deftest ^:parallel normalize-aggregations-test-5+ (normalize-tests"make sure binning-strategy clauses get normalized the way we'd expect"{{:query {:breakout [["BINNING_STRATEGY" 10 "BIN-WIDTH" 2000]]}}- {:query {:breakout [[:binning-strategy 10 :bin-width 2000]]}}}+ {:query {:breakout [[:field 10 {:binning {:strategy :bin-width, :bin-width 2000}}]]}}}))+(deftest ^:parallel normalize-aggregations-test-6+ (normalize-tests"or field literals!"- {{:query {:aggregation ["count" ["count" ["field_literal" "ABCDEF" "type/Text"]]]}}- {:query {:aggregation [:count [:count [:field-literal "ABCDEF" :type/Text]]]}}}+ {{:query {:aggregation [["count"] ["count" ["field_literal" "ABCDEF" "type/Text"]]]}}+ {:query {:aggregation [[:count]+ [:count [:field "ABCDEF" {:base-type :type/Text}]]]}}}))+(deftest ^:parallel normalize-aggregations-test-7+ (normalize-tests"event if you try your best to break things it should handle it"- {{:query {:aggregation ["count" ["sum" 10] ["count" 20] "count"]}}- {:query {:aggregation [:count [:sum 10] [:count 20] :count]}}}+ {{:query {:aggregation [["count"] ["sum" 10] ["count" 20] ["count"]]}}+ {:query {:aggregation [[:count]+ [:sum [:field 10 nil]]+ [:count [:field 20 nil]]+ [:count]]}}}))+(deftest ^:parallel normalize-aggregations-test-8+ (normalize-tests"try an ag that is named using legacy `:named` clause"{{:query {:aggregation ["named" ["SuM" 10] "My COOL AG"]}}- {:query {:aggregation [:named [:sum 10] "My COOL AG"]}}+ {:query {:aggregation [[:aggregation-options [:sum [:field 10 nil]] {:display-name "My COOL AG"}]]}}{:query {:aggregation ["named" ["SuM" 10] "My COOL AG" {:use-as-display-name? false}]}}- {:query {:aggregation [:named [:sum 10] "My COOL AG" {:use-as-display-name? false}]}}}+ {:query {:aggregation [[:aggregation-options [:sum [:field 10 nil]] {:name "My COOL AG"}]]}}}))+(deftest ^:parallel normalize-aggregations-test-9+ (normalize-tests"try w/ `:aggregation-options`, the replacement for `:named`"{{:query {:aggregation ["aggregation_options" ["SuM" 10] {"display_name" "My COOL AG"}]}}- {:query {:aggregation [:aggregation-options [:sum 10] {:display-name "My COOL AG"}]}}}+ {:query {:aggregation [[:aggregation-options [:sum [:field 10 nil]] {:display-name "My COOL AG"}]]}}}))+(deftest ^:parallel normalize-aggregations-test-10+ (normalize-tests"try an expression ag"{{:query {:aggregation ["+" ["sum" 10] ["*" ["SUM" 20] 3]]}}- {:query {:aggregation [:+ [:sum 10] [:* [:sum 20] 3]]}}}+ {:query {:aggregation [[:++ [:sum [:field 10 nil]]+ [:* [:sum [:field 20 nil]] 3]]]}}}))+(deftest ^:parallel normalize-aggregations-test-11+ (normalize-tests"expression ags should handle varargs"{{:query {:aggregation ["+" ["sum" 10] ["SUM" 20] ["sum" 30]]}}- {:query {:aggregation [:+ [:sum 10] [:sum 20] [:sum 30]]}}}+ {:query {:aggregation [[:++ [:sum [:field 10 nil]]+ [:sum [:field 20 nil]]+ [:sum [:field 30 nil]]]]}}}))+(deftest ^:parallel normalize-aggregations-test-12+ (normalize-tests"expression references should be exempt too"- {{:order-by [[:desc [:expression "SALES_TAX"]]]}- {:order-by [[:desc [:expression "SALES_TAX"]]]}}+ {{:query {:order-by [[:desc [:expression "SALES_TAX"]]]}}+ {:query {:order-by [[:desc [:expression "SALES_TAX"]]]}}}))+(deftest ^:parallel normalize-aggregations-test-13+ (normalize-tests"... but they should be converted to strings if passed in as a KW for some reason. Make sure we preserve namespace!"- {{:order-by [[:desc ["expression" :SALES/TAX]]]}- {:order-by [[:desc [:expression "SALES/TAX"]]]}}+ {{:query {:order-by [[:desc ["expression" :SALES/TAX]]]}}+ {:query {:order-by [[:desc [:expression "SALES/TAX"]]]}}}))+(deftest ^:parallel normalize-aggregations-test-14+ (normalize-tests"case"- {{:query {:aggregation ["sum" ["case"- [[[">" ["field-id" 12] 10] 10]- [[">" ["field-id" 12] 100] ["field-id" 1]]- [["=" ["field-id" 2] 1] "foo"]]- {:default ["field-id" 2]}]]}}- {:query {:aggregation [:sum [:case- [[[:> [:field-id 12] 10] 10]- [[:> [:field-id 12] 100] [:field-id 1]]- [[:= [:field-id 2] 1] "foo"]]- {:default [:field-id 2]}]]}}}-+ {{:query+ {:aggregation+ ["sum"+ ["case"+ [[[">" ["field-id" 12] 10] 10]+ [[">" ["field-id" 12] 100] ["field-id" 1]]+ [["=" ["field-id" 2] 1] "foo"]]+ {:default ["field-id" 2]}]]}}+ {:query+ {:aggregation+ [[:sum+ [:case+ [[[:> [:field 12 nil] 10] 10]+ [[:> [:field 12 nil] 100] [:field 1 nil]]+ [[:= [:field 2 nil] 1] "foo"]]+ {:default [:field 2 nil]}]]]}}}))++(deftest ^:parallel normalize-aggregations-test-15+ (normalize-tests"various other new ag types"{{:query {:aggregation ["median" ["field-id" 13]]}}- {:query {:aggregation [:median [:field-id 13]]}}+ {:query {:aggregation [[:median [:field 13 nil]]]}}{:query {:aggregation ["var" ["field-id" 13]]}}- {:query {:aggregation [:var [:field-id 13]]}}+ {:query {:aggregation [[:var [:field 13 nil]]]}}{:query {:aggregation ["percentile" ["field-id" 13] 0.9]}}- {:query {:aggregation [:percentile [:field-id 13] 0.9]}}}))+ {:query {:aggregation [[:percentile [:field 13 nil] 0.9]]}}}))-;;; ---------------------------------------------------- order-by ------------------------------------------------------(t/deftest ^:parallel normalize-order-by-test+(deftest ^:parallel normalize-order-by-test(normalize-tests"does order-by get properly normalized?"{{:query {"ORDER_BY" [[10 "ASC"]]}}- {:query {:order-by [[10 :asc]]}}+ {:query {:order-by [[:asc [:field 10 nil]]]}}{:query {"ORDER_BY" [["ASC" 10]]}}- {:query {:order-by [[:asc 10]]}}+ {:query {:order-by [[:asc [:field 10 nil]]]}}{:query {"ORDER_BY" [[["field_id" 10] "ASC"]]}}- {:query {:order-by [[[:field-id 10] :asc]]}}+ {:query {:order-by [[:asc [:field 10 nil]]]}}{:query {"ORDER_BY" [["DESC" ["field_id" 10]]]}}- {:query {:order-by [[:desc [:field-id 10]]]}}}))--;;; ----------------------------------------------------- filter -----------------------------------------------------+ {:query {:order-by [[:desc [:field 10 nil]]]}}}))-(t/deftest ^:parallel normalize-filter-test+(deftest ^:parallel normalize-filter-test(normalize-tests"the unit & amount in time interval clauses should get normalized"{{:query {"FILTER" ["time-interval" 10 "current" "day"]}}- {:query {:filter [:time-interval 10 :current :day]}}}+ {:query {:filter [:time-interval [:field 10 nil] :current :day]}}}))+(deftest ^:parallel normalize-filter-test-2+ (normalize-tests"but amount should not get normalized if it's an integer"{{:query {"FILTER" ["time-interval" 10 -10 "day"]}}- {:query {:filter [:time-interval 10 -10 :day]}}}+ {:query {:filter [:time-interval [:field 10 nil] -10 :day]}}}))+(deftest ^:parallel normalize-filter-test-3+ (normalize-tests"relative-time-interval is correctly normalized"- {{:query {"FILTER" ["relative-time-interval" 10 "week" -10 "week"]}}- {:query {:filter [:relative-time-interval 10 :week -10 :week]}}}+ {{:query {"FILTER" ["relative-time-interval" 1 10 "week" -10 "week"]}}+ {:query {:filter [:relative-time-interval [:field 1 nil] 10 :week -10 :week]}}}))+(deftest ^:parallel normalize-filter-test-4+ (normalize-tests"make sure we support time-interval options"{["TIME_INTERVAL" 10 -30 "DAY" {"include_current" true}]- [:time-interval 10 -30 :day {:include-current true}]}+ [:time-interval [:field 10 nil] -30 :day {:include-current true}]}))+(deftest ^:parallel normalize-filter-test-5+ (normalize-tests"the unit in relative datetime clauses should get normalized"{{:query {"FILTER" ["=" [:field-id 10] ["RELATIVE_DATETIME" -31 "DAY"]]}}- {:query {:filter [:= [:field-id 10] [:relative-datetime -31 :day]]}}}+ {:query {:filter [:= [:field 10 nil] [:relative-datetime -31 :day]]}}}))+(deftest ^:parallel normalize-filter-test-6+ (normalize-tests"should work if we do [:relative-datetime :current] as well"{{:query {"FILTER" ["=" [:field-id 10] ["RELATIVE_DATETIME" "CURRENT"]]}}- {:query {:filter [:= [:field-id 10] [:relative-datetime :current]]}}}+ {:query {:filter [:= [:field 10 nil] [:relative-datetime :current]]}}}))- "and in datetime-field clauses (MBQL 98+)"+(deftest ^:parallel normalize-filter-test-7+ (normalize-tests+ "and in datetime-field clauses (MBQL 3+)"{{:query {"FILTER" ["=" [:datetime-field ["field_id" 10] "day"] "2018-09-05"]}}- {:query {:filter [:= [:datetime-field [:field-id 10] :day] "2018-09-05"]}}}+ {:query {:filter [:= [:field 10 {:temporal-unit :day}] "2018-09-05"]}}}))- "(or in long-since-deprecated MBQL 95 format)"+(deftest ^:parallel normalize-filter-test-8+ (normalize-tests+ "(or in long-since-deprecated MBQL 2 format)"{{:query {"FILTER" ["=" [:datetime-field 10 "as" "day"] "2018-09-05"]}}- {:query {:filter [:= [:datetime-field 10 :as :day] "2018-09-05"]}}}+ {:query {:filter [:= [:field 10 {:temporal-unit :day}] "2018-09-05"]}}}))+(deftest ^:parallel normalize-filter-test-9+ (normalize-tests"if string filters have an options map that should get normalized"{{:query {"FILTER" ["starts_with" 10 "ABC" {"case_sensitive" true}]}}- {:query {:filter [:starts-with 10 "ABC" {:case-sensitive true}]}}}))+ {:query {:filter [:starts-with [:field 10 nil] "ABC" {:case-sensitive true}]}}}))-;;; --------------------------------------------------- parameters -----------------------------------------------------(t/deftest ^:parallel normalize-parmaeters-test+(deftest ^:parallel normalize-parmaeters-test(normalize-tests"make sure we're not running around trying to normalize the type in native query params"{{:type :native@@ -240,8 +302,10 @@{:type :native:parameters [{:type :date/range:target [:dimension [:template-tag "checkin_date"] {:stage-number -2}]- :value "2015-04-01~2015-05-01"}]}}+ :value "2015-04-01~2015-05-01"}]}}))+(deftest ^:parallel normalize-parmaeters-test-2+ (normalize-tests"`:parameters` `:type` should get normalized, but `:value` should not."{{:type "native":parameters [{:type "text"@@ -250,8 +314,10 @@{:type :native:parameters [{:type :text:target [:dimension [:template-tag "names_list"]]- :value ["BBQ" "Bakery" "Bar"]}]}}+ :value ["BBQ" "Bakery" "Bar"]}]}}))+(deftest ^:parallel normalize-parmaeters-test-3+ (normalize-tests"make sure normalization doesn't try to parse value as an MBQL clause"{{:type "native":parameters [{:type "text"@@ -268,7 +334,7 @@:native {:query "SELECT COUNT(*) FROM \"PUBLIC\".\"CHECKINS\" WHERE {{checkin_date}}":template-tags template-tags}})-(t/deftest ^:parallel normalize-template-tags-test+(deftest ^:parallel normalize-template-tags-test(normalize-tests"`:template-tags` key should get normalized"{{:type :native@@ -285,7 +351,7 @@:dimension [:field 14 nil]:widget-type :category}}}}}))-(t/deftest ^:parallel normalize-template-tags-test-2+(deftest ^:parallel normalize-template-tags-test-2(normalize-tests"Don't try to normalize template-tag name/display name. Names should get converted to strings."{(query-with-template-tags@@ -312,7 +378,7 @@:dimension [:field 14 nil]:widget-type :category}})}))-(t/deftest ^:parallel normalize-template-tags-test-3+(deftest ^:parallel normalize-template-tags-test-3(normalize-tests"Actually, `:name` should just get copied over from the map key if it's missing or different"{(query-with-template-tags@@ -338,7 +404,7 @@:dimension [:field 14 nil]:widget-type :category}})}))-(t/deftest ^:parallel normalize-template-tags-test-4+(deftest ^:parallel normalize-template-tags-test-4(normalize-tests"`:type` should get normalized"{(query-with-template-tags@@ -350,10 +416,10 @@{"names_list" {:name "names_list":display-name "Names List":type :dimension- :dimension [:field-id 49]+ :dimension [:field 49 nil]:widget-type :category}})}))-(t/deftest ^:parallel normalize-template-tags-test-5+(deftest ^:parallel normalize-template-tags-test-5(normalize-tests"`:widget-type` should get normalized"{(query-with-template-tags@@ -367,9 +433,9 @@:display-name "Names List":type :dimension:widget-type :string/=- :dimension [:field-id 49]}})}))+ :dimension [:field 49 nil]}})}))-(t/deftest ^:parallel normalize-template-tags-test-6+(deftest ^:parallel normalize-template-tags-test-6(normalize-tests"`:dimension` should get normalized";; doesn't get converted to `:field` here because that happens during the canonicalization step.@@ -382,10 +448,10 @@{"checkin_date" {:name "checkin_date":display-name "Checkin Date":type :dimension- :dimension [:field-id 14]+ :dimension [:field 14 nil]:widget-type :category}})}))-(t/deftest ^:parallel normalize-template-tags-test-7+(deftest ^:parallel normalize-template-tags-test-7(normalize-tests"Don't normalize `:default` values"{(query-with-template-tags@@ -403,7 +469,7 @@:dimension [:field 1 nil]:default ["a" "b"]}})}))-(t/deftest ^:parallel normalize-template-tags-test-8+(deftest ^:parallel normalize-template-tags-test-8(normalize-tests"Don't keywordize keys that aren't present in template tag maps"{{:database 1@@ -415,7 +481,7 @@:type :native:native {:template-tags {"x" {:name "x"}}}}}))-(t/deftest ^:parallel normalize-template-tags-test-9+(deftest ^:parallel normalize-template-tags-test-9(normalize-tests":dimension (Field filter) template tags with no :widget-type should get :category as a default type (#20643)"{{:database 1@@ -447,9 +513,7 @@:native {:template-tags {"x" {:name "x":type :nonsense}}}}}))-;;; ------------------------------------------------- source queries ---------------------------------------------------(t/deftest ^:parallel normalize-source-queries-test+(deftest ^:parallel normalize-source-queries-test(normalize-tests"Make sure token normalization works correctly on source queries"{{:database 4@@ -474,11 +538,9 @@:query {"source_query" {"source_table" 1, "aggregation" "rows"}}}{:database 4:type :query- :query {:source-query {:source-table 1, :aggregation :rows}}}}))--;;; ----------------------------------------------------- joins ------------------------------------------------------+ :query {:source-query {:source-table 1}}}}))-(t/deftest ^:parallel normalize-joins-test+(deftest ^:parallel normalize-joins-test(normalize-tests"do entries in the `:joins` clause get normalized?"{{:database 4@@ -494,8 +556,10 @@:joins [{:source-table 2:alias "my/table":strategy :left-join- :fields :all}]}}}+ :fields :all}]}}}))+(deftest ^:parallel normalize-joins-test-2+ (normalize-tests"what about with a sequence of :fields?"{{:database 4:type :query@@ -504,9 +568,11 @@{:database 4:type :query:query {:source-table 1- :joins [{:fields [[:field-id 1]- [:field-literal "MY_FIELD" :type/Integer]]}]}}}+ :joins [{:fields [[:field 1 nil]+ [:field "MY_FIELD" {:base-type :type/Integer}]]}]}}}))+(deftest ^:parallel normalize-joins-test-3+ (normalize-tests"do `:joins` inside a nested query get normalized?"{{:database 4:type :query@@ -519,47 +585,35 @@:type :query:query {:source-query {:source-table 1:joins [{:strategy :right-join- :fields [[:field-id 1]- [:field-literal "MY_FIELD" :type/Integer]]}]}}}}))+ :fields [[:field 1 nil]+ [:field "MY_FIELD" {:base-type :type/Integer}]]}]}}}}))-(t/deftest ^:parallel normalize-source-query-in-joins-test- (t/testing "does a `:source-query` in `:joins` get normalized?"+(deftest ^:parallel normalize-source-query-in-joins-test+ (testing "does a `:source-query` in `:joins` get normalized?"(letfn [(query-with-joins [joins]{:database 4:type :query:query {:source-table 1:joins joins}})]- (t/testing "MBQL source query in :joins"- (t/is (= (query-with-joins [{:source-query {:source-table 2}- :fields [[:field-id 1]- [:field-literal "MY_FIELD" :type/Integer]]}])- (#'mbql.normalize/normalize-tokens- (query-with-joins [{"source_query" {"source_table" 2}- "fields" [["field_id" 1]- ["field_literal" :MY_FIELD "type/Integer"]]}])))))- (t/testing "native source query in :joins"- (t/testing "string source query"- (t/is (= (query-with-joins [{:source-query {:native "SELECT *"}}])- (#'mbql.normalize/normalize-tokens- (query-with-joins [{"source_query" {"NATIVE" "SELECT *"}}])))))- (t/testing "map source query"- (t/is (= (query-with-joins [{:source-query {:native {"this_is_a_native_query" "TRUE"}}}])- (#'mbql.normalize/normalize-tokens- (query-with-joins [{"source_query" {"NATIVE" {"this_is_a_native_query" "TRUE"}}}])))))))))--;;; ----------------------------------------------------- other --------------------------------------------------------(t/deftest ^:parallel normalize-execution-context-test- (normalize-tests- "Does the QueryExecution context get normalized?"- {{:context "json-download"}- {:context :json-download}}-- "if `:context` is `nil` it's not our problem"- {{:context nil}- {:context nil}}))--(t/deftest ^:parallel params-normalization-test+ (testing "MBQL source query in :joins"+ (is (= (query-with-joins [{:source-query {:source-table 2}+ :fields [[:field 1 nil]+ [:field "MY_FIELD" {:base-type :type/Integer}]]}])+ (#'mbql.normalize/normalize+ (query-with-joins [{"source_query" {"source_table" 2}+ "fields" [["field_id" 1]+ ["field_literal" :MY_FIELD "type/Integer"]]}])))))+ (testing "native source query in :joins"+ (testing "string source query"+ (is (= (query-with-joins [{:source-query {:native "SELECT *"}}])+ (#'mbql.normalize/normalize+ (query-with-joins [{"source_query" {"NATIVE" "SELECT *"}}])))))+ (testing "map source query"+ (is (= (query-with-joins [{:source-query {:native {"this_is_a_native_query" "TRUE"}}}])+ (#'mbql.normalize/normalize+ (query-with-joins [{"source_query" {"NATIVE" {"this_is_a_native_query" "TRUE"}}}])))))))))++(deftest ^:parallel params-normalization-test(normalize-tests":native :params shouldn't get normalized."{{:native {:query "SELECT * FROM venues WHERE name = ?"@@ -567,7 +621,7 @@{:native {:query "SELECT * FROM venues WHERE name = ?":params ["Red Medicine"]}}}))-(t/deftest ^:parallel normalize-projections-test+(deftest ^:parallel normalize-projections-test(normalize-tests"Native :projections shouldn't get normalized."{{:type :native@@ -576,24 +630,26 @@:native {:projections ["_id" "name" "category_id" "latitude" "longitude" "price"]}}}));; this is also covered-(t/deftest ^:parallel normalize-expressions-test+(deftest ^:parallel normalize-expressions-test(normalize-tests"Expression names should get normalized to strings."- {{:query {"expressions" {:abc ["+" 1 2]}- :fields [["expression" :abc]]}}- {:query {:expressions {"abc" [:+ 1 2]}- :fields [[:expression "abc"]]}}}+ {{:query {"expressions" {:abc ["+" 1 2]}, :fields [["expression" :abc]]}}+ {:query {:expressions {"abc" [:+ 1 2]}, :fields [[:expression "abc"]]}}}))+(deftest ^:parallel normalize-expressions-test-2+ (normalize-tests"are expression names exempt from lisp-casing/lower-casing?"{{"query" {"expressions" {:sales_tax ["-" ["field-id" 10] ["field-id" 20]]}}}- {:query {:expressions {"sales_tax" [:- [:field-id 10] [:field-id 20]]}}}}+ {:query {:expressions {"sales_tax" [:- [:field 10 nil] [:field 20 nil]]}}}}))+(deftest ^:parallel normalize-expressions-test-3+ (normalize-tests"expressions should handle datetime arithemtics"{{:query {:expressions {:prev_month ["+" ["field-id" 13] ["interval" -1 "month"]]}}}- {:query {:expressions {"prev_month" [:+ [:field-id 13] [:interval -1 :month]]}}}+ {:query {:expressions {"prev_month" [:+ [:field 13 nil] [:interval -1 :month]]}}}{:query {:expressions {:prev_month ["-" ["field-id" 13] ["interval" 1 "month"] ["interval" 1 "day"]]}}}- {:query {:expressions {"prev_month" [:- [:field-id 13] [:interval 1 :month] [:interval 1 :day]]}}}+ {:query {:expressions {"prev_month" [:- [:field 13 nil] [:interval 1 :month] [:interval 1 :day]]}}}{:query {:expressions {:datetime-diff ["datetime-diff" ["field" 1 nil] ["field" 2 nil] "month"]}}}{:query {:expressions {"datetime-diff" [:datetime-diff [:field 1 nil] [:field 2 nil] :month]}}}@@ -602,177 +658,187 @@{:query {:expressions {"datetime-add" [:datetime-add [:field 1 nil] 1 :month]}}}{:query {:expressions {:datetime-subtract ["datetime-subtract" ["field" 1 nil] 1 "month"]}}}- {:query {:expressions {"datetime-subtract" [:datetime-subtract [:field 1 nil] 1 :month]}}}}+ {:query {:expressions {"datetime-subtract" [:datetime-subtract [:field 1 nil] 1 :month]}}}}))+(deftest ^:parallel normalize-expressions-test-4+ (normalize-tests"expressions handle namespaced keywords correctly"- {{:query {"expressions" {:abc/def ["+" 1 2]}- :fields [["expression" :abc/def]]}}- {:query {:expressions {"abc/def" [:+ 1 2]}- :fields [[:expression "abc/def"]]}}}+ {{:query {"expressions" {:abc/def ["+" 1 2]}, :fields [["expression" :abc/def]]}}+ {:query {:expressions {"abc/def" [:+ 1 2]}, :fields [[:expression "abc/def"]]}}}))+(deftest ^:parallel normalize-expressions-test-5+ (normalize-tests"expression refs can have opts (#33528)"- {{:query {:expressions {"abc" [+ 1 2]}- :fields [[:expression "abc" {"base-type" "type/Number"}]]}}- {:query {:expressions {"abc" [+ 1 2]}- :fields [[:expression "abc" {:base-type :type/Number}]]}}}+ {{:query {:expressions {"abc" [+ 1 2]}, :fields [[:expression "abc" {"base-type" "type/Number"}]]}}+ {:query {:expressions {"abc" [+ 1 2]}, :fields [[:expression "abc" {:base-type :type/Number}]]}}}))+(deftest ^:parallel normalize-expressions-test-6+ (normalize-tests"expressions can be a literal :value"- {{:query {:expressions {"abc" [:value 123 {"base-type" "type/Integer"}]}- :fields [[:expression "abc" {"base-type" "type/Integer"}]]}}- {:query {:expressions {"abc" [:value 123 {:base_type :type/Integer}]}- :fields [[:expression "abc" {:base-type :type/Integer}]]}}}))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | CANONICALIZE |-;;; +----------------------------------------------------------------------------------------------------------------+--(defn- canonicalize-tests [& {:as group->input->expected}]- (tests 'canonicalize #'mbql.normalize/canonicalize group->input->expected))--(t/deftest ^:parallel wrap-implicit-field-id-test- (t/testing "Does our `wrap-implict-field-id` fn work?"- (doseq [input [10 [:field 10 nil]]]- (t/testing (pr-str (list 'wrap-implicit-field-id input))- (t/is (= [:field 10 nil]- (#'mbql.normalize/wrap-implicit-field-id input)))))- (t/testing "Leave legacy clauses as-is; we'll replace them elsewhere"- (t/is (= [:field-id 10]- (#'mbql.normalize/wrap-implicit-field-id [:field-id 10]))))))--(t/deftest ^:parallel canonicalize-field-test- (canonicalize-tests+ {{:query+ {:expressions {"abc" [:value 123 {"base-type" "type/Integer"}]}+ :fields [[:expression "abc" {"base-type" "type/Integer"}]]}}+ {:query+ {:expressions {"abc" [:value 123 {:base_type :type/Integer}]}+ :fields [[:expression "abc" {:base-type :type/Integer}]]}}}))++(deftest ^:parallel canonicalize-field-test+ (normalize-tests"If someone accidentally nests `:field` clauses, we should fix it for them."{{:query {:fields [[:field [:field 1 {:a 100, :b 200}] {:b 300}]]}}{:query {:fields [[:field 1 {:a 100, :b 300}]]}}{:query {:fields [[:field [:field [:field 1 {:a 100, :b 200}] {:b 300}] {:a 400, :c 500}]]}}- {:query {:fields [[:field 1 {:a 400, :b 300, :c 500}]]}}}+ {:query {:fields [[:field 1 {:a 400, :b 300, :c 500}]]}}}))+(deftest ^:parallel canonicalize-field-test-2+ (normalize-tests"We should remove empty options maps"{[:field 2 {}][:field 2 nil]}))-(t/deftest ^:parallel canonicalize-substring-test- (canonicalize-tests+(deftest ^:parallel canonicalize-substring-test+ (normalize-tests"substring index 0 -> 1"{[:substring "foo" 0 1]- [:substring "foo" 1 1]-- [:substring "foo" 0 1 3]- [:substring "foo" 1 1 3]}))--;;; ------------------------------------------------ binning strategy ------------------------------------------------+ [:substring "foo" 1 1]}))-(t/deftest ^:parallel canonicalize-binning-strategy-test- (canonicalize-tests+(deftest ^:parallel canonicalize-binning-strategy-test+ (normalize-tests"make sure `binning-strategy` wraps implicit Field IDs"{{:query {:breakout [[:binning-strategy 10 :bin-width 2000]]}}{:query {:breakout [[:field 10 {:binning {:strategy :bin-width, :bin-width 2000}}]]}}}))-;;; -------------------------------------------------- aggregation ---------------------------------------------------+(deftest ^:parallel canonicalize-binning-strategy-test-2+ (normalize-tests+ "make sure `binning-strategy` wraps implicit Field IDs"+ {{:query {:breakout [[:binning-strategy 10 :bin-width 2000 {:a :b}]]}}+ {:query {:breakout [[:field 10 {:binning {:strategy :bin-width, :bin-width 2000, :a :b}}]]}}}))-(t/deftest ^:parallel canonicalize-aggregations-test- (canonicalize-tests+(deftest ^:parallel canonicalize-aggregations-test+ (normalize-tests"field ID should get wrapped in field-id and ags should be converted to multiple ag syntax"- {{:query {:aggregation [:count 10]}}- {:query {:aggregation [[:count [:field 10 nil]]]}}}+ {{:query {:aggregation [:count 10]}} {:query {:aggregation [[:count [:field 10 nil]]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-2+ (normalize-tests"ag with no Field ID"- {{:query {:aggregation [:count]}}- {:query {:aggregation [[:count]]}}}+ {{:query {:aggregation [:count]}} {:query {:aggregation [[:count]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-3+ (normalize-tests"if already wrapped in field-id it's ok"- {{:query {:aggregation [:count [:field-id 1000]]}}- {:query {:aggregation [[:count [:field 1000 nil]]]}}}+ {{:query {:aggregation [:count [:field-id 1000]]}} {:query {:aggregation [[:count [:field 1000 nil]]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-4+ (normalize-tests":rows aggregation type, being deprecated since FOREVER, should just get removed"{{:query {:aggregation [:rows]}}- {:query {:aggregation []}}+ {:query {}}{:query {:aggregation :rows}}- {:query {:aggregation []}}}+ {:query {}}}))+(deftest ^:parallel canonicalize-aggregations-test-5+ (normalize-tests"if just a single aggregation is supplied it should always be converted to new-style multiple-aggregation syntax"- {{:query {:aggregation :count}}- {:query {:aggregation [[:count]]}}}+ {{:query {:aggregation :count}} {:query {:aggregation [[:count]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-6+ (normalize-tests"make sure we handle single :count with :field-id correctly"- {{:query {:aggregation [:count [:field-id 10]]}}- {:query {:aggregation [[:count [:field 10 nil]]]}}}+ {{:query {:aggregation [:count [:field-id 10]]}} {:query {:aggregation [[:count [:field 10 nil]]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-7+ (normalize-tests"make sure for multiple aggregations we can handle `:count` that doesn't appear wrapped in brackets"- {{:query {:aggregation [:count [:sum 10]]}}- {:query {:aggregation [[:count] [:sum [:field 10 nil]]]}}}+ {{:query {:aggregation [[:count] [:sum 10]]}}+ {:query {:aggregation [[:count] [:sum [:field 10 nil]]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-8+ (normalize-tests(str "this doesn't make sense, but make sure if someone specifies a `:rows` ag and another one we don't end up ""with a `nil` in the ags list")- {{:query {:aggregation [:rows :count]}}- {:query {:aggregation [[:count]]}}}+ {{:query {:aggregation [:rows :count]}} {:query {:aggregation [[:count]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-9+ (normalize-tests"another stupid aggregation that we need to be able to handle"- {{:query {:aggregation [:count :count]}}- {:query {:aggregation [[:count] [:count]]}}}-- "a mix of unwrapped & wrapped should still work"- {{:query {:aggregation [:count [:sum 10] [:count 20] :count]}}- {:query {:aggregation [[:count] [:sum [:field 10 nil]] [:count [:field 20 nil]] [:count]]}}}+ {{:query {:aggregation [:count :count]}} {:query {:aggregation [[:count] [:count]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-10+ (normalize-tests"legacy `:named` aggregation clauses should get converted to `:aggregation-options`"{{:query {:aggregation [:named [:sum 10] "Sum *TEN*"]}}{:query {:aggregation [[:aggregation-options [:sum [:field 10 nil]] {:display-name "Sum *TEN*"}]]}}{:query {:aggregation [:named [:sum 10] "Sum *TEN*" {:use-as-display-name? false}]}}- {:query {:aggregation [[:aggregation-options [:sum [:field 10 nil]] {:name "Sum *TEN*"}]]}}}+ {:query {:aggregation [[:aggregation-options [:sum [:field 10 nil]] {:name "Sum *TEN*"}]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-11+ (normalize-tests"subclauses of `:aggregation-options` should get canonicalized correctly"{{:query {:aggregation [[:aggregation-options [:sum 10] {}]]}}- {:query {:aggregation [[:aggregation-options [:sum [:field 10 nil]] {}]]}}}+ {:query {:aggregation [[:aggregation-options [:sum [:field 10 nil]] {}]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-12+ (normalize-tests"make sure expression aggregations work correctly"{{:query {:aggregation [:+ [:sum 10] 2]}}{:query {:aggregation [[:+ [:sum [:field 10 nil]] 2]]}}{:query {:aggregation [:+ [:sum 10] [:* [:sum 20] [:sum 30]]]}}- {:query {:aggregation [[:+ [:sum [:field 10 nil]] [:* [:sum [:field 20 nil]] [:sum [:field 30 nil]]]]]}}}+ {:query {:aggregation [[:+ [:sum [:field 10 nil]] [:* [:sum [:field 20 nil]] [:sum [:field 30 nil]]]]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-13+ (normalize-tests"expression ags should handle varargs"{{:query {:aggregation [[:+ [:sum 10] [:sum 20] [:sum 30]]]}}- {:query {:aggregation [[:+ [:sum [:field 10 nil]] [:sum [:field 20 nil]] [:sum [:field 30 nil]]]]}}}+ {:query {:aggregation [[:+ [:sum [:field 10 nil]] [:sum [:field 20 nil]] [:sum [:field 30 nil]]]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-14+ (normalize-tests"METRICS shouldn't get canonicalized in some kind of wacky way"- {{:query {:aggregation [:+ [:metric 1] 2]}}- {:query {:aggregation [[:+ [:metric 1] 2]]}}}+ {{:query {:aggregation [:+ [:metric 1] 2]}} {:query {:aggregation [[:+ [:metric 1] 2]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-15+ (normalize-tests"can cumulative-count be handled with or without a Field?"{{:query {:aggregation [:cum-count]}}{:query {:aggregation [[:cum-count]]}}{:query {:aggregation [:cum-count 10]}}- {:query {:aggregation [[:cum-count [:field 10 nil]]]}}}+ {:query {:aggregation [[:cum-count [:field 10 nil]]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-16+ (normalize-tests"should handle seqs without a problem"{{:query {:aggregation '([:min 1] [:min 2])}}- {:query {:aggregation [[:min [:field 1 nil]] [:min [:field 2 nil]]]}}}+ {:query {:aggregation [[:min [:field 1 nil]] [:min [:field 2 nil]]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-17+ (normalize-tests"make sure canonicalization can handle aggregations with expressions where the Field normally goes"{{:query {:aggregation [[:sum [:* [:field-id 4] [:field-id 1]]]]}}{:query {:aggregation [[:sum [:* [:field 4 nil] [:field 1 nil]]]]}}{:query {:aggregation [[:sum [:* [:field 4 nil] [:field 1 nil]]]]}}- {:query {:aggregation [[:sum [:* [:field 4 nil] [:field 1 nil]]]]}}}+ {:query {:aggregation [[:sum [:* [:field 4 nil] [:field 1 nil]]]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-18+ (normalize-tests"Make sure `:case` expressions get canonicalized correctly"{{:query {:aggregation [:sum [:case [[[:< [:field-id 37331] 2] 2] [[:< [:field-id 37331] 4] 1]]]]}}- {:query {:aggregation [[:sum [:case [[[:< [:field 37331 nil] 2] 2] [[:< [:field 37331 nil] 4] 1]]]]]}}}+ {:query {:aggregation [[:sum [:case [[[:< [:field 37331 nil] 2] 2] [[:< [:field 37331 nil] 4] 1]]]]]}}}))+(deftest ^:parallel canonicalize-aggregations-test-19+ (normalize-tests":percentile"{{:query {:aggregation [[:percentile [:field-id 37809] 0.9]]}}{:query {:aggregation [[:percentile [:field 37809 nil] 0.9]]}}}))-;;; ---------------------------------------------------- breakout ------------------------------------------------------(t/deftest ^:parallel canonicalize-breakout-test- (canonicalize-tests+(deftest ^:parallel canonicalize-breakout-test+ (normalize-tests"implicit Field IDs should get wrapped in [:field-id] in :breakout"{{:query {:breakout [10]}}{:query {:breakout [[:field 10 nil]]}}@@ -790,20 +856,16 @@{:query {:breakout [[:field 1000 nil]]}}{:query {:breakout [[:field 1000 nil]]}}}))-;;; ----------------------------------------------------- expressions --------------------------------------------------(t/deftest ^:parallel canonicalize-expressions-test- (canonicalize-tests- "expressions can be a literal :value"+(deftest ^:parallel canonicalize-expressions-test+ (normalize-tests+ "expressions can xbe a literal :value"{{:query {:expressions {"abc" [:value false {:base_type :type/Boolean}]}:fields [[:expression "abc" {:base-type :type/Boolean}]]}}{:query {:expressions {"abc" [:value false {:base_type :type/Boolean}]}:fields [[:expression "abc" {:base-type :type/Boolean}]]}}}))-;;; ----------------------------------------------------- fields -------------------------------------------------------(t/deftest ^:parallel canonicalize-fields-test- (canonicalize-tests+(deftest ^:parallel canonicalize-fields-test+ (normalize-tests"implicit Field IDs should get wrapped in [:field-id] in :fields"{{:query {:fields [10]}}{:query {:fields [[:field 10 nil]]}}@@ -812,160 +874,161 @@{:query {:fields [[:field 10 nil] [:field 20 nil]]}}{:query {:fields [[:field-id 1000]]}}- {:query {:fields [[:field 1000 nil]]}}}+ {:query {:fields [[:field 1000 nil]]}}}))+(deftest ^:parallel canonicalize-fields-test-2+ (normalize-tests"should handle seqs"{{:query {:fields '(10 20)}}{:query {:fields [[:field 10 nil] [:field 20 nil]]}}}))-;;; ----------------------------------------------------- filter -------------------------------------------------------(t/deftest ^:parallel canonicalize-filter-test- (canonicalize-tests+(deftest ^:parallel canonicalize-filter-test+ (normalize-tests"implicit Field IDs should get wrapped in [:field-id] in filters"- {{:query {:filter [:= 10 20]}}- {:query {:filter [:= [:field 10 nil] 20]}}-+ {{:query {:filter [:= 10 20]}} {:query {:filter [:= [:field 10 nil] 20]}}{:query {:filter [:and [:= 10 20] [:= 20 30]]}}{:query {:filter [:and [:= [:field 10 nil] 20] [:= [:field 20 nil] 30]]}}+ {:query {:filter [:between 10 20 30]}} {:query {:filter [:between [:field 10 nil] 20 30]}}}))- {:query {:filter [:between 10 20 30]}}- {:query {:filter [:between [:field 10 nil] 20 30]}}}-+(deftest ^:parallel canonicalize-filter-test-2+ (normalize-tests"`:inside` filters should get implict Field IDs for the first two args"{{:query {:filter [:inside 1 2 90 -90 90 -90]}}- {:query {:filter [:inside [:field 1 nil] [:field 2 nil] 90 -90 90 -90]}}}+ {:query {:filter [:inside [:field 1 nil] [:field 2 nil] 90 -90 90 -90]}}}))+(deftest ^:parallel canonicalize-filter-test-3+ (normalize-tests"compound filters with only one arg should get automatically de-compounded"- {{:query {:filter [:and [:= 100 2]]}}- {:query {:filter [:= [:field 100 nil] 2]}}-- {:query {:filter [:or [:= 100 2]]}}- {:query {:filter [:= [:field 100 nil] 2]}}}+ {{:query {:filter [:and [:= 100 2]]}} {:query {:filter [:= [:field 100 nil] 2]}}+ {:query {:filter [:or [:= 100 2]]}} {:query {:filter [:= [:field 100 nil] 2]}}}))+(deftest ^:parallel canonicalize-filter-test-4+ (normalize-tests"compound filters should \"pull-up\" any args that are the same compound filter"- {{:query {:filter [:and- [:and- [:= [:field-id 100] 1]- [:= [:field-id 200] 2]]- [:and- [:= [:field-id 300] 3]- [:= [:field-id 400] 4]]]}}- {:query {:filter [:and- [:= [:field 100 nil] 1]- [:= [:field 200 nil] 2]- [:= [:field 300 nil] 3]- [:= [:field 400 nil] 4]]}}-- {:query {:filter [:and- [:> [:field-id 4] 1]- [:is-null [:field-id 7]]- [:and- [:= [:field-id 5] "abc"]- [:between [:field-id 9] 0 25]]]}}- {:query {:filter [:and- [:> [:field 4 nil] 1]- [:is-null [:field 7 nil]]- [:= [:field 5 nil] "abc"]- [:between [:field 9 nil] 0 25]]}}-- {:query {:filter [:or- [:or- [:= 100 1]- [:= 200 2]]- [:or- [:= [:field-id 300] 3]- [:= [:field-id 400] 4]]]}}- {:query {:filter [:or- [:= [:field 100 nil] 1]- [:= [:field 200 nil] 2]- [:= [:field 300 nil] 3]- [:= [:field 400 nil] 4]]}}-- {:query {:filter [:or- [:> [:field-id 4] 1]- [:is-null [:field-id 7]]- [:or- [:= [:field-id 5] "abc"]- [:between [:field-id 9] 0 25]]]}}- {:query {:filter [:or- [:> [:field 4 nil] 1]- [:is-null [:field 7 nil]]- [:= [:field 5 nil] "abc"]- [:between [:field 9 nil] 0 25]]}}}-+ {{:query+ {:filter+ [:and [:and [:= [:field-id 100] 1] [:= [:field-id 200] 2]] [:and [:= [:field-id 300] 3] [:= [:field-id 400] 4]]]}}+ {:query+ {:filter [:and [:= [:field 100 nil] 1] [:= [:field 200 nil] 2] [:= [:field 300 nil] 3] [:= [:field 400 nil] 4]]}}++ {:query+ {:filter+ [:and+ [:> [:field-id 4] 1]+ [:is-null [:field-id 7]]+ [:and [:= [:field-id 5] "abc"] [:between [:field-id 9] 0 25]]]}}+ {:query+ {:filter+ [:and [:> [:field 4 nil] 1] [:is-null [:field 7 nil]] [:= [:field 5 nil] "abc"] [:between [:field 9 nil] 0 25]]}}++ {:query {:filter [:or [:or [:= 100 1] [:= 200 2]] [:or [:= [:field-id 300] 3] [:= [:field-id 400] 4]]]}}+ {:query+ {:filter [:or [:= [:field 100 nil] 1] [:= [:field 200 nil] 2] [:= [:field 300 nil] 3] [:= [:field 400 nil] 4]]}}++ {:query+ {:filter+ [:or [:> [:field-id 4] 1] [:is-null [:field-id 7]] [:or [:= [:field-id 5] "abc"] [:between [:field-id 9] 0 25]]]}}+ {:query+ {:filter+ [:or [:> [:field 4 nil] 1] [:is-null [:field 7 nil]] [:= [:field 5 nil] "abc"] [:between [:field 9 nil] 0 25]]}}}))++(deftest ^:parallel canonicalize-filter-test-5+ (normalize-tests"not inside of a not should get elimated entirely"{{:query {:filter [:not [:not [:= [:field-id 100] 1]]]}}- {:query {:filter [:= [:field 100 nil] 1]}}}+ {:query {:filter [:= [:field 100 nil] 1]}}++ {:query {:filter [:not [:not [:not [:not [:= [:field-id 100] 1]]]]]}}+ {:query {:filter [:= [:field 100 nil] 1]}}}))+(deftest ^:parallel canonicalize-filter-test-6+ (normalize-tests"make sure we don't overwrite options if specified"{{:query {:filter [:contains 10 "ABC" {:case-sensitive false}]}}- {:query {:filter [:contains [:field 10 nil] "ABC" {:case-sensitive false}]}}}+ {:query {:filter [:contains [:field 10 nil] "ABC" {:case-sensitive false}]}}}))+(deftest ^:parallel canonicalize-filter-test-7+ (normalize-tests"or for time-interval options"{[:time-interval 10 -30 :day {:include-current true}]- [:time-interval [:field 10 nil] -30 :day {:include-current true}]}+ [:time-interval [:field 10 nil] -30 :day {:include-current true}]}))+(deftest ^:parallel canonicalize-filter-test-8+ (normalize-tests"make sure empty filter clauses don't explode in canonicalize"- {{:database 1, :type :query, :query {:filter []}}- {:database 1, :type :query, :query {:filter []}}}+ {{:database 1, :type :query, :query {:source-table 1, :filter []}}+ {:database 1, :type :query, :query {:source-table 1}}}))+(deftest ^:parallel canonicalize-filter-test-9+ (normalize-tests"make sure we can handle GA segments"{{:database 1- :type :query- :query {:filter [:and- [:segment "gaid:-11"]- [:time-interval [:field-id 6851] -365 :day {}]]}}+ :type :query+ :query {:filter [:and [:segment "gaid:-11"] [:time-interval [:field-id 6851] -365 :day {}]]}}{:database 1- :type :query- :query {:filter- [:and- [:segment "gaid:-11"]- [:time-interval [:field 6851 nil] -365 :day {}]]}}}+ :type :query+ :query {:filter [:and [:segment "gaid:-11"] [:time-interval [:field 6851 nil] -365 :day {}]]}}}))+(deftest ^:parallel canonicalize-filter-test-10+ (normalize-tests"should handle seqs"- {{:query {:filter '(:and- [:= 100 1]- [:= 200 2])}}- {:query {:filter [:and [:= [:field 100 nil] 1] [:= [:field 200 nil] 2]]}}}+ {{:query {:filter '(:and [:= 100 1] [:= 200 2])}}+ {:query {:filter [:and+ [:= [:field 100 nil] 1]+ [:= [:field 200 nil] 2]]}}}))+(deftest ^:parallel canonicalize-filter-test-11+ (normalize-tests"if you put a `:datetime-field` inside a `:time-interval` we should fix it for you"{{:query {:filter [:time-interval [:datetime-field [:field-id 8] :month] -30 :day]}}- {:query {:filter [:time-interval [:field 8 nil] -30 :day]}}}+ {:query {:filter [:time-interval [:field 8 nil] -30 :day]}}}))- "fk-> clauses should get the field-id treatment"+(deftest ^:parallel canonicalize-filter-test-12+ (normalize-tests+ "MBQL ≤ 3 fk-> clauses should get normalized to MBQL 4"{{:query {:filter [:= [:fk-> 10 20] "ABC"]}}- {:query {:filter [:= [:field 20 {:source-field 10}] "ABC"]}}}+ {:query {:filter [:= [:field 20 {:source-field 10}] "ABC"]}}++ {:query {:fields [["fk->" ["field" 101 nil] ["field" 294 nil]]]}}+ {:query {:fields [[:field 294 {:source-field 101}]]}}}))+(deftest ^:parallel canonicalize-filter-test-13+ (normalize-tests"as should datetime-field clauses..."{{:query {:filter [:= [:datetime-field 10 :day] "2018-09-05"]}}- {:query {:filter [:= [:field 10 {:temporal-unit :day}] "2018-09-05"]}}}+ {:query {:filter [:= [:field 10 {:temporal-unit :day}] "2018-09-05"]}}}))- "MBQL 95 datetime-field clauses ([:datetime-field <field> :as <unit>]) should get converted to MBQL 2000"+(deftest ^:parallel canonicalize-filter-test-14+ (normalize-tests+ "MBQL 2 datetime-field clauses ([:datetime-field <field> :as <unit>]) should get converted to MBQL 4"{{:query {:filter [:= [:datetime-field 10 :as :day] "2018-09-05"]}}- {:query {:filter [:= [:field 10 {:temporal-unit :day}] "2018-09-05"]}}}+ {:query {:filter [:= [:field 10 {:temporal-unit :day}] "2018-09-05"]}}}))+(deftest ^:parallel canonicalize-filter-test-15+ (normalize-tests"if someone is dumb and passes something like a field-literal inside a field-id, fix it for them."{{:query {:filter [:= [:field-id [:field-literal "my_field" "type/Number"]] 10]}}- {:query {:filter [:= [:field "my_field" {:base-type "type/Number"}] 10]}}}+ {:query {:filter [:= [:field "my_field" {:base-type :type/Number}] 10]}}}))+(deftest ^:parallel canonicalize-filter-test-16+ (normalize-tests"we should fix :field-ids inside :field-ids too"- {{:query {:filter [:= [:field-id [:field-id 1]] 10]}}- {:query {:filter [:= [:field 1 nil] 10]}}}+ {{:query {:filter [:= [:field-id [:field-id 1]] 10]}} {:query {:filter [:= [:field 1 nil] 10]}}}))+(deftest ^:parallel canonicalize-filter-test-17+ (normalize-tests"we should handle seqs no prob"- {{:query {:filter '(:= 1 10)}}- {:query {:filter [:= [:field 1 nil] 10]}}}))+ {{:query {:filter '(:= 1 10)}} {:query {:filter [:= [:field 1 nil] 10]}}}));;; ---------------------------------------------------- order-by -----------------------------------------------------(t/deftest ^:parallel canonicalize-order-by-test- (canonicalize-tests- "ORDER BY: MBQL 95 [field direction] should get translated to MBQL 98+ [direction field]"+(deftest ^:parallel canonicalize-order-by-test+ (normalize-tests+ "ORDER BY: MBQL 2 [field direction] should get translated to MBQL 3+ [direction field]"{{:query {:order-by [[[:field-id 10] :asc]]}}{:query {:order-by [[:asc [:field 10 nil]]]}}}- "MBQL 95 old order-by names should be handled"+ "MBQL 2 old order-by names should be handled"{{:query {:order-by [[10 :ascending]]}}{:query {:order-by [[:asc [:field 10 nil]]]}}}@@ -987,10 +1050,8 @@{:query {:order-by [[:asc [:field 1 nil]][:desc [:field 2 nil]]]}}}))-;;; ------------------------------------------------- source queries ---------------------------------------------------(t/deftest ^:parallel canonicalize-source-queries-test- (canonicalize-tests+(deftest ^:parallel canonicalize-source-queries-test+ (normalize-tests"Make sure canonicalization works correctly on source queries"{{:database 4:type :query@@ -1007,373 +1068,315 @@:display-name "Category":type :text:required true- :default "Widget"}}}}}}+ :default "Widget"}}}}}}))+(deftest ^:parallel canonicalize-source-queries-test-2+ (normalize-tests"make sure we recursively canonicalize source queries"{{:database 4:type :query:query {:source-query {:source-table 1, :aggregation :rows}}}{:database 4:type :query- :query {:source-query {:source-table 1, :aggregation []}}}}))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | WHOLE-QUERY TRANSFORMATIONS |-;;; +----------------------------------------------------------------------------------------------------------------+--(t/deftest ^:parallel whole-query-transformations-test- (tests 'perform-whole-query-transformations #'mbql.normalize/perform-whole-query-transformations- {(str "If you specify a field in a breakout and in the Fields clause, we should go ahead and remove it from the "- "Fields clause, because it is (obviously) implied that you should get that Field back.")- {{:type :query- :query {:breakout [[:field 1 nil] [:field 2 nil]]- :fields [[:field 2 nil] [:field 3 nil]]}}- {:type :query- :query {:breakout [[:field 1 nil] [:field 2 nil]]- :fields [[:field 3 nil]]}}}-- "should work with FKs"- {{:type :query- :query {:breakout [[:field 1 nil]- [:field 4 {:source-field 2}]]- :fields [[:field 4 {:source-field 2}]- [:field 3 nil]]}}- {:type :query- :query {:breakout [[:field 1 nil]- [:field 4 {:source-field 2}]]- :fields [[:field 3 nil]]}}}-- "should work if the Field is bucketed in the breakout & in fields"- {{:type :query- :query {:breakout [[:field 1 nil]- [:field 4 {:source-field 2, :temporal-unit :month}]]- :fields [[:field 4 {:source-field 2, :temporal-unit :month}]- [:field 3 nil]]}}- {:type :query- :query {:breakout [[:field 1 nil]- [:field 4 {:source-field 2, :temporal-unit :month}]]- :fields [[:field 3 nil]]}}}-- "should work if the Field is bucketed in the breakout but not in fields"- {{:type :query- :query {:breakout [[:field 1 nil]- [:field 4 {:source-field 2, :temporal-unit :month}]]- :fields [[:field 4 {:source-field 2}]- [:field 3 nil]]}}- {:type :query- :query {:breakout [[:field 1 nil]- [:field 4 {:source-field 2, :temporal-unit :month}]]- :fields [[:field 3 nil]]}}}}))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | REPLACE LEGACY FILTERS |-;;; +----------------------------------------------------------------------------------------------------------------+--(t/deftest ^:parallel replace-relative-date-filters-test- (tests 'replace-relative-date-filters #'mbql.normalize/replace-relative-date-filters- {"date range in the past"- {[:between- [:+ [:field 1 nil] [:interval 7 :year]]- [:relative-datetime -30 :day]- [:relative-datetime 0 :day]]- [:relative-time-interval [:field 1 nil] -30 :day -7 :year]}-- "date range in the future"- {[:between- [:+ [:field 1 nil] [:interval -7 :year]]- [:relative-datetime 0 :day]- [:relative-datetime 30 :day]]- [:relative-time-interval [:field 1 nil] 30 :day 7 :year]}-- "date range that is not entirely in the past or in the future should be skipped"- {[:between- [:+ [:field 1 nil] [:interval -7 :year]]- [:relative-datetime -5 :day]- [:relative-datetime 30 :day]]- [:between- [:+ [:field 1 nil] [:interval -7 :year]]- [:relative-datetime -5 :day]- [:relative-datetime 30 :day]]}-- "date range with different units for start and end of the interval should be skipped"- {[:between- [:+ [:field 1 nil] [:interval -7 :year]]- [:relative-datetime 0 :day]- [:relative-datetime 30 :quarter]]- [:between- [:+ [:field 1 nil] [:interval -7 :year]]- [:relative-datetime 0 :day]- [:relative-datetime 30 :quarter]]}}))--(t/deftest ^:parallel replace-exclude-date-filters-test- (tests 'replace-exclude-date-filters #'mbql.normalize/replace-exclude-date-filters- {"`:hour-of-day`"- {[:!= [:field 1 {:temporal-unit :hour-of-day}] 0 23]- [:!= [:get-hour [:field 1 nil]] 0 23]}-- "`:day-of-week`"- {[:!= [:field 1 {:temporal-unit :day-of-week}] "2024-12-02" "2024-12-08"]- [:!= [:get-day-of-week [:field 1 nil] :iso] 1 7]}-- "`:month-of-year`"- {[:!= [:field 1 {:temporal-unit :month-of-year}] "2024-01-02" "2024-12-08"]- [:!= [:get-month [:field 1 nil]] 1 12]}-- "`:quarter-of-year`"- {[:!= [:field 1 {:temporal-unit :quarter-of-year}] "2024-01-02" "2024-12-08"]- [:!= [:get-quarter [:field 1 nil]] 1 4]}-- "field options should be preserved"- {[:!= [:field 1 {:base-type :type/DateTime :temporal-unit :hour-of-day}] 10]- [:!= [:get-hour [:field 1 {:base-type :type/DateTime}]] 10]}-- "filters with invalid dates should be ignored"- {[:!= [:field 1 {:temporal-unit :quarter-of-year}] "2024-01-99" "2024-12-08"]- [:!= [:field 1 {:temporal-unit :quarter-of-year}] "2024-01-99" "2024-12-08"]}-- "filters with non-date string arguments should be ignored"- {[:!= [:field 1 {:temporal-unit :quarter-of-year}] "2024-01-02" "abc"]- [:!= [:field 1 {:temporal-unit :quarter-of-year}] "2024-01-02" "abc"]}}))--(t/deftest ^:parallel replace-legacy-filters-test- (tests 'replace-legacy-filters-test #'mbql.normalize/replace-legacy-filters- {"exclude date filters"- {{:type :query,- :query {:filters [:and- [:!= [:field 1 {:temporal-unit :hour-of-day}] 0 23]- [:!= [:field 1 {:temporal-unit :quarter-of-year}] "2024-01-02" "2024-12-08"]]}}- {:type :query,- :query {:filters [:and- [:!= [:get-hour [:field 1 nil]] 0 23]- [:!= [:get-quarter [:field 1 nil]] 1 4]]}}}}))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | REMOVE EMPTY CLAUSES |-;;; +----------------------------------------------------------------------------------------------------------------+--(t/deftest ^:parallel remove-empty-clauses-test- (tests 'remove-empty-clauses #'mbql.normalize/remove-empty-clauses- {"empty sequences should get removed"- {{:x [], :y [100]}- {:y [100]}}-- "nil values should get removed"- {{:x nil, :y 100}- {:y 100}}-- "sequences containing only nil should get removed"- {{:a [nil 100], :b [nil nil]}- {:a [nil 100]}}-- "empty maps should get removed"- {{:a {:b 100}, :c {}}- {:a {:b 100}}-- {:a {:b 100}, :c {:d nil}}- {:a {:b 100}}}}))--(t/deftest ^:parallel remove-empty-options-from-field-clause-test- (tests 'remove-empty-clauses #'mbql.normalize/remove-empty-clauses- {"We should remove empty options maps"- {[:field 2 {}]- [:field 2 nil]-- [:field 2 {:binning {}}]- [:field 2 nil]}-- "We should remove nil keys from options maps"- {[:field 2 {:join-alias nil}]- [:field 2 nil]-- [:field 2 {:binning {:strategy nil}}]- [:field 2 nil]}-- "Don't remove false values from options map"- {[:field 2 {:x false}]- [:field 2 {:x false}]}-- "Remove empty sequences from options map"- {[:field 2 {:x []}]- [:field 2 nil]-- [:field 2 {:x [{:y nil}]}]- [:field 2 nil]}}))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | PUTTING IT ALL TOGETHER |-;;; +----------------------------------------------------------------------------------------------------------------+--(t/deftest ^:parallel e2e-mbql-95-query-test- (t/testing "With an ugly MBQL 95 query, does everything get normalized nicely?"- (t/is (= {:type :query- :query {:source-table 10- :breakout [[:field 10 nil] [:field 20 nil]]- :filter [:= [:field 10 nil] [:field 20 {:temporal-unit :day}]]- :order-by [[:desc [:field 10 nil]]]}}- (mbql.normalize/normalize {:type "query"- :query {"source_table" 10- "AGGREGATION" "ROWS"- "breakout" [10 20]- "filter" ["and" ["=" 10 ["datetime-field" 20 "as" "day"]]]- "order-by" [[10 "desc"]]}})))))--(t/deftest ^:parallel e2e-native-query-with-params-test- (t/testing "let's try doing the full normalization on a native query w/ params"- (t/is (= {:native {:query "SELECT * FROM CATEGORIES WHERE {{names_list}}"- :template-tags {"names_list" {:name "names_list"- :display-name "Names List"- :type :dimension- :widget-type :category- :dimension [:field 49 nil]}}}- :parameters [{:type :text- :target [:dimension [:template-tag "names_list"]]- :value ["BBQ" "Bakery" "Bar"]}]}- (mbql.normalize/normalize- {:native {:query "SELECT * FROM CATEGORIES WHERE {{names_list}}"- "template_tags" {:names_list {:name "names_list"- :display_name "Names List"- :type "dimension"- :dimension ["field-id" 49]}}}- :parameters [{:type "text"- :target ["dimension" ["template-tag" "names_list"]]- :value ["BBQ" "Bakery" "Bar"]}]})))))--(t/deftest ^:parallel e2e-big-query-with-segments-test- (t/testing "let's try normalizing a big query with SEGMENTS"- (t/is (= {:database 1- :type :query- :query {:source-table 2- :filter [:and- [:= [:field 3 nil] "Toucan-friendly"]- [:segment 4]- [:segment 5]]}}- (mbql.normalize/normalize- {:database 1- :type :query- :query {:source-table 2- :filter ["AND"- ["=" 3 "Toucan-friendly"]- ["SEGMENT" 4]- ["SEGMENT" 5]]}})))))--(t/deftest ^:parallel e2e-source-queries-test- (t/testing "make sure source queries get normalized properly!"- (t/is (= {:database 4- :type :query- :query {:source-query {:native "SELECT * FROM PRODUCTS WHERE CATEGORY = {{category}} LIMIT 10"- :template-tags {"category" {:name "category"+ :query {:source-query {:source-table 1}}}}))++(deftest ^:parallel whole-query-transformations-test-1+ (normalize-tests+ (str+ "If you specify a field in a breakout and in the Fields clause, we should go ahead and remove it from the "+ "Fields clause, because it is (obviously) implied that you should get that Field back.")+ {{:type :query, :query {:breakout [[:field 1 nil] [:field 2 nil]], :fields [[:field 2 nil] [:field 3 nil]]}}+ {:type :query, :query {:breakout [[:field 1 nil] [:field 2 nil]], :fields [[:field 3 nil]]}}}))++(deftest ^:parallel whole-query-transformations-test-2+ (normalize-tests+ "should work with FKs"+ {{:type :query+ :query {:breakout [[:field 1 nil]+ [:field 4 {:source-field 2}]]+ :fields [[:field 4 {:source-field 2}]+ [:field 3 nil]]}}+ {:type :query+ :query {:breakout [[:field 1 nil]+ [:field 4 {:source-field 2}]]+ :fields [[:field 3 nil]]}}}))++(deftest ^:parallel whole-query-transformations-test-3+ (normalize-tests+ "should work if the Field is bucketed in the breakout & in fields"+ {{:type :query+ :query {:breakout [[:field 1 nil] [:field 4 {:source-field 2, :temporal-unit :month}]]+ :fields [[:field 4 {:source-field 2, :temporal-unit :month}] [:field 3 nil]]}}+ {:type :query+ :query {:breakout [[:field 1 nil] [:field 4 {:source-field 2, :temporal-unit :month}]], :fields [[:field 3 nil]]}}}))++(deftest ^:parallel whole-query-transformations-test-4+ (normalize-tests+ "should work if the Field is bucketed in the breakout but not in fields"+ {{:type :query+ :query+ {:breakout [[:field 1 nil] [:field 4 {:source-field 2, :temporal-unit :month}]]+ :fields [[:field 4 {:source-field 2}] [:field 3 nil]]}}+ {:type :query+ :query {:breakout [[:field 1 nil]+ [:field 4 {:source-field 2, :temporal-unit :month}]]+ :fields [[:field 3 nil]]}}}))++(deftest ^:parallel replace-relative-date-filters-test+ (normalize-tests+ "date range in the past"+ {[:between [:+ [:field 1 nil] [:interval 7 :year]] [:relative-datetime -30 :day] [:relative-datetime 0 :day]]+ [:relative-time-interval [:field 1 nil] -30 :day -7 :year]}))++(deftest ^:parallel replace-relative-date-filters-test-2+ (normalize-tests+ "date range in the future"+ {[:between [:+ [:field 1 nil] [:interval -7 :year]] [:relative-datetime 0 :day] [:relative-datetime 30 :day]]+ [:relative-time-interval [:field 1 nil] 30 :day 7 :year]}))++(deftest ^:parallel replace-relative-date-filters-test-3+ (normalize-tests+ "date range that is not entirely in the past or in the future should be skipped"+ {[:between [:+ [:field 1 nil] [:interval -7 :year]] [:relative-datetime -5 :day] [:relative-datetime 30 :day]]+ [:between [:+ [:field 1 nil] [:interval -7 :year]] [:relative-datetime -5 :day] [:relative-datetime 30 :day]]}))++(deftest ^:parallel replace-relative-date-filters-test-4+ (normalize-tests+ "date range with different units for start and end of the interval should be skipped"+ {[:between [:+ [:field 1 nil] [:interval -7 :year]] [:relative-datetime 0 :day] [:relative-datetime 30 :quarter]]+ [:between [:+ [:field 1 nil] [:interval -7 :year]] [:relative-datetime 0 :day] [:relative-datetime 30 :quarter]]}))++(deftest ^:parallel replace-exclude-date-filters-test+ (normalize-tests+ {"`:hour-of-day`"+ {[:!= [:field 1 {:temporal-unit :hour-of-day}] 0 23]+ [:!= [:get-hour [:field 1 nil]] 0 23]}++ "`:day-of-week`"+ {[:!= [:field 1 {:temporal-unit :day-of-week}] "2024-12-02" "2024-12-08"]+ [:!= [:get-day-of-week [:field 1 nil] :iso] 1 7]}++ "`:month-of-year`"+ {[:!= [:field 1 {:temporal-unit :month-of-year}] "2024-01-02" "2024-12-08"]+ [:!= [:get-month [:field 1 nil]] 1 12]}++ "`:quarter-of-year`"+ {[:!= [:field 1 {:temporal-unit :quarter-of-year}] "2024-01-02" "2024-12-08"]+ [:!= [:get-quarter [:field 1 nil]] 1 4]}++ "field options should be preserved"+ {[:!= [:field 1 {:base-type :type/DateTime :temporal-unit :hour-of-day}] 10]+ [:!= [:get-hour [:field 1 {:base-type :type/DateTime}]] 10]}++ "filters with invalid dates should be ignored"+ {[:!= [:field 1 {:temporal-unit :quarter-of-year}] "2024-01-99" "2024-12-08"]+ [:!= [:field 1 {:temporal-unit :quarter-of-year}] "2024-01-99" "2024-12-08"]}++ "filters with non-date string arguments should be ignored"+ {[:!= [:field 1 {:temporal-unit :quarter-of-year}] "2024-01-02" "abc"]+ [:!= [:field 1 {:temporal-unit :quarter-of-year}] "2024-01-02" "abc"]}}))++(deftest ^:parallel replace-legacy-filters-test+ (normalize-tests+ {"exclude date filters"+ {{:type :query+ :query {:filter [:and+ [:!= [:field 1 {:temporal-unit :hour-of-day}] 0 23]+ [:!= [:field 1 {:temporal-unit :quarter-of-year}] "2024-01-02" "2024-12-08"]]}}+ {:type :query+ :query {:filter [:and+ [:!= [:get-hour [:field 1 nil]] 0 23]+ [:!= [:get-quarter [:field 1 nil]] 1 4]]}}}}))++(deftest ^:parallel remove-empty-options-from-field-clause-test+ (is (= {:query {:fields [[:field 1 nil]+ [:field 2 nil]]}}+ (mbql.normalize/normalize {:query {:fields [[:field 1 {}]+ [:field 2 {}]]}}))))++(deftest ^:parallel e2e-mbql-2-query-test+ (testing "With an ugly MBQL 2 query, does everything get normalized nicely?"+ (is (= {:type :query+ :query {:source-table 10+ :breakout [[:field 10 nil] [:field 20 nil]]+ :filter [:= [:field 10 nil] [:field 20 {:temporal-unit :day}]]+ :order-by [[:desc [:field 10 nil]]]}}+ (mbql.normalize/normalize {:type "query"+ :query {"source_table" 10+ "AGGREGATION" "ROWS"+ "breakout" [10 20]+ "filter" ["and" ["=" 10 ["datetime-field" 20 "as" "day"]]]+ "order-by" [[10 "desc"]]}})+ (mbql.normalize/normalize {:type "query"+ :query {"source_table" 10+ "AGGREGATION" ["ROWS"]+ "breakout" [10 20]+ "filter" ["and" ["=" 10 ["datetime-field" 20 "as" "day"]]]+ "order-by" [[10 "descending"]]}})))))++(deftest ^:parallel e2e-native-query-with-params-test+ (testing "let's try doing the full normalization on a native query w/ params"+ (is (= {:native {:query "SELECT * FROM CATEGORIES WHERE {{names_list}}"+ :template-tags {"names_list" {:name "names_list"+ :display-name "Names List"+ :type :dimension+ :widget-type :category+ :dimension [:field 49 nil]}}}+ :parameters [{:type :text+ :target [:dimension [:template-tag "names_list"]]+ :value ["BBQ" "Bakery" "Bar"]}]}+ (mbql.normalize/normalize+ {:native {:query "SELECT * FROM CATEGORIES WHERE {{names_list}}"+ "template_tags" {:names_list {:name "names_list"+ :display_name "Names List"+ :type "dimension"+ :dimension ["field-id" 49]}}}+ :parameters [{:type "text"+ :target ["dimension" ["template-tag" "names_list"]]+ :value ["BBQ" "Bakery" "Bar"]}]})))))++(deftest ^:parallel e2e-big-query-with-segments-test+ (testing "let's try normalizing a big query with SEGMENTS"+ (is (= {:database 1+ :type :query+ :query {:source-table 2+ :filter [:and+ [:= [:field 3 nil] "Toucan-friendly"]+ [:segment 4]+ [:segment 5]]}}+ (mbql.normalize/normalize+ {:database 1+ :type :query+ :query {:source-table 2+ :filter ["AND"+ ["=" 3 "Toucan-friendly"]+ ["SEGMENT" 4]+ ["SEGMENT" 5]]}})))))++(deftest ^:parallel e2e-source-queries-test+ (testing "make sure source queries get normalized properly!"+ (is (= {:database 4+ :type :query+ :query {:source-query {:native "SELECT * FROM PRODUCTS WHERE CATEGORY = {{category}} LIMIT 10"+ :template-tags {"category" {:name "category"+ :display-name "Category"+ :type :text+ :required true+ :default "Widget"}}}}}+ (mbql.normalize/normalize+ {:database 4+ :type :query+ :query {"source_query" {:native "SELECT * FROM PRODUCTS WHERE CATEGORY = {{category}} LIMIT 10"+ "template_tags" {:category {:name "category":display-name "Category"- :type :text+ :type "text":required true- :default "Widget"}}}}}- (mbql.normalize/normalize- {:database 4- :type :query- :query {"source_query" {:native "SELECT * FROM PRODUCTS WHERE CATEGORY = {{category}} LIMIT 10"- "template_tags" {:category {:name "category"- :display-name "Category"- :type "text"- :required true- :default "Widget"}}}}})))))--(t/deftest ^:parallel e2e-rows-aggregation-test- (t/testing "make sure `rows` aggregations get removed"- (t/is (= {:database 4- :type :query- :query {:source-query {:source-table 1}}}- (mbql.normalize/normalize- {:database 4- :type :query- :query {"source_query" {"source_table" 1, "aggregation" "rows"}}})))))--(t/deftest ^:parallel e2e-parameters-test- (t/testing (str "make sure that parameters get normalized/canonicalized correctly. value should not get normalized, "- "but type should; target should do canonicalization for MBQL clauses")- (t/is (= {:type :query- :query {:source-table 1}- :parameters [{:type :id, :target [:dimension [:field 4575 {:source-field 3265}]], :value ["field-id"]}- {:type :date/all-options, :target [:dimension [:field 3270 nil]], :value "thismonth"}]}+ :default "Widget"}}}}})))))++(deftest ^:parallel e2e-rows-aggregation-test+ (testing "make sure `rows` aggregations get removed"+ (is (= {:database 4+ :type :query+ :query {:source-query {:source-table 1}}}+ (mbql.normalize/normalize+ {:database 4+ :type :query+ :query {"source_query" {"source_table" 1, "aggregation" "rows"}}})))))++(deftest ^:parallel e2e-parameters-test+ (testing (str "make sure that parameters get normalized/canonicalized correctly. value should not get normalized, "+ "but type should; target should do canonicalization for MBQL clauses")+ (is (= {:type :query+ :query {:source-table 1}+ :parameters [{:type :id, :target [:dimension [:field 4575 {:source-field 3265}]], :value ["field-id"]}+ {:type :date/all-options, :target [:dimension [:field 3270 nil]], :value "thismonth"}]}+ (mbql.normalize/normalize+ {:type :query+ :query {:source-table 1}+ :parameters [{:type "id", :target ["dimension" ["fk->" 3265 4575]], :value ["field-id"]}+ {:type "date/all-options", :target ["dimension" ["field-id" 3270]], :value "thismonth"}]})))))++(deftest ^:parallel e2e-parameters-test-2+ (testing "Make sure default values do not get normalized"+ (is (= {:database 1+ ;; tag name not normalized+ :native {:template-tags {"name" {:id "1f56330b-3dcb-75a3-8f3d-5c2c2792b749"+ :name "name"+ :display-name "Name"+ :type :dimension+ :dimension [:field 14 nil] ;; field normalized+ :widget-type :string/= ;; widget-type normalize+ :default ["Hudson Borer"]}} ;; default values not keyworded+ :query "select * from PEOPLE where {{name}}"}+ :type :native}+ (mbql.normalize/normalize+ {:database 1+ :native {:template-tags {"name" {:id "1f56330b-3dcb-75a3-8f3d-5c2c2792b749"+ :name "name"+ :display-name "Name"+ :type "dimension"+ :dimension ["field" 14 nil]+ :widget-type "string/="+ :default ["Hudson Borer"]}}+ :query "select * from PEOPLE where {{name}}"}+ :type "native"+ :parameters []})))))++(deftest ^:parallel e2e-source-metadata-test+ (testing "make sure `:source-metadata` gets normalized the way we'd expect:"+ (testing "1. Type names should get converted to keywords"+ (is (= {:query {:source-metadata+ [{:name "name"+ :display_name "Name"+ :description nil+ :base_type :type/Text+ :semantic_type :type/Name+ :fingerprint {:global {:distinct-count 100}+ :type {:type/Text {:percent-json 0.0+ :percent-url 0.0+ :percent-email 0.0+ :average-length 15.63}}}}]}}(mbql.normalize/normalize- {:type :query- :query {:source-table 1}- :parameters [{:type "id", :target ["dimension" ["fk->" 3265 4575]], :value ["field-id"]}- {:type "date/all-options", :target ["dimension" ["field-id" 3270]], :value "thismonth"}]}))))- (t/testing "Make sure default values do not get normalized"- (t/is (= {:database 1- ;; tag name not normalized- :native {:template-tags {"name" {:id "1f56330b-3dcb-75a3-8f3d-5c2c2792b749"- :name "name"- :display-name "Name"- :type :dimension- :dimension [:field 14 nil] ;; field normalized- :widget-type :string/= ;; widget-type normalize- :default ["Hudson Borer"]}} ;; default values not keyworded- :query "select * from PEOPLE where {{name}}"}- :type :native}+ {:query {:source-metadata [{:name "name"+ :display_name "Name"+ :description nil+ :base_type "type/Text"+ :semantic_type "type/Name"+ :fingerprint {"global" {"distinct-count" 100}+ "type" {"type/Text" {"percent-json" 0.0+ "percent-url" 0.0+ "percent-email" 0.0+ "average-length" 15.63}}}}]}}))))))++(deftest ^:parallel e2e-source-metadata-test-2+ (testing "make sure `:source-metadata` gets normalized the way we'd expect:"+ (testing (str "2. if `:source-metadata` is at the top-level, it should get moved to the correct location inside "+ "the 'inner' MBQL query")+ (is (= {:query {:source-metadata+ [{:name "name"+ :display_name "Name"+ :description nil+ :base_type :type/Text+ :semantic_type :type/Name+ :fingerprint {:global {:distinct-count 100}+ :type {:type/Text {:percent-json 0.0+ :percent-url 0.0+ :percent-email 0.0+ :average-length 15.63}}}}]}}(mbql.normalize/normalize- {:database 1- :native {:template-tags {"name" {:id "1f56330b-3dcb-75a3-8f3d-5c2c2792b749"- :name "name"- :display-name "Name"- :type "dimension"- :dimension ["field" 14 nil]- :widget-type "string/="- :default ["Hudson Borer"]}}- :query "select * from PEOPLE where {{name}}"}- :type "native"- :parameters []})))))--(t/deftest ^:parallel e2e-source-metadata-test- (t/testing "make sure `:source-metadata` gets normalized the way we'd expect:"- (t/testing "1. Type names should get converted to keywords"- (t/is (= {:query {:source-metadata- [{:name "name"- :display_name "Name"- :base_type :type/Text- :semantic_type :type/Name- :fingerprint {:global {:distinct-count 100}- :type {:type/Text {:percent-json 0.0- :percent-url 0.0- :percent-email 0.0- :average-length 15.63}}}}]}}- (mbql.normalize/normalize- {:query {:source-metadata [{:name "name"- :display_name "Name"- :description nil- :base_type "type/Text"- :semantic_type "type/Name"- :fingerprint {"global" {"distinct-count" 100}- "type" {"type/Text" {"percent-json" 0.0- "percent-url" 0.0- "percent-email" 0.0- "average-length" 15.63}}}}]}}))))-- (t/testing (str "2. if `:source-metadata` is at the top-level, it should get moved to the correct location inside "- "the 'inner' MBQL query")- (t/is (= {:query {:source-metadata- [{:name "name"- :display_name "Name"- :base_type :type/Text- :semantic_type :type/Name- :fingerprint {:global {:distinct-count 100}- :type {:type/Text {:percent-json 0.0- :percent-url 0.0- :percent-email 0.0- :average-length 15.63}}}}]}}- (mbql.normalize/normalize- {:source-metadata [{:name "name"- :display_name "Name"- :description nil- :base_type "type/Text"- :semantic_type "type/Name"- :fingerprint {"global" {"distinct-count" 100}- "type" {"type/Text" {"percent-json" 0.0- "percent-url" 0.0- "percent-email" 0.0- "average-length" 15.63}}}}]}))))))--(t/deftest ^:parallel normalize-nil-values-in-native-maps-test- (t/testing "nil values in native query maps (e.g. MongoDB queries) should not get removed during normalization.\n"+ {:source-metadata [{:name "name"+ :display_name "Name"+ :description nil+ :base_type "type/Text"+ :semantic_type "type/Name"+ :fingerprint {"global" {"distinct-count" 100}+ "type" {"type/Text" {"percent-json" 0.0+ "percent-url" 0.0+ "percent-email" 0.0+ "average-length" 15.63}}}}]}))))))++(deftest ^:parallel normalize-nil-values-in-native-maps-test+ (testing "nil values in native query maps (e.g. MongoDB queries) should not get removed during normalization.\n"(letfn [(test-normalization [native-query](let [native-source-query (set/rename-keys native-query {:query :native})](doseq [[message query] {"top-level native query"@@ -1384,11 +1387,11 @@"native source query in join"{:query {:joins [{:source-query native-source-query}]}}}]- (t/testing (str "\n" message)- (t/is (= query- (mbql.normalize/normalize query)))))))]+ (testing (str "\n" message)+ (is (= query+ (mbql.normalize/normalize query)))))))]- (t/testing "Keys in native query maps should not get normalized"+ (testing "Keys in native query maps should not get normalized"(test-normalization{:projections [:count]:query [{"$project" {"price" "$price"}}@@ -1398,25 +1401,25 @@{"$project" {"_id" false, "count" true}}]:collection "venues"}))- (t/testing "`nil` values inside native :params shouldn't get removed"+ (testing "`nil` values inside native :params shouldn't get removed"(test-normalization {:query "SELECT ?" :params [nil]})))))-(t/deftest ^:parallel empty-test- (t/testing "test a query with :is-empty"- (t/is (= {:query {:filter [:and- [:> [:field 4 nil] 1]- [:is-empty [:field 7 nil]]- [:= [:field 5 nil] "abc"]- [:between [:field 9 nil] 0 25]]}}- (#'mbql.normalize/canonicalize {:query {:filter [:and- [:> [:field-id 4] 1]- [:is-empty [:field-id 7]]- [:and- [:= [:field-id 5] "abc"]- [:between [:field-id 9] 0 25]]]}})))))--(t/deftest ^:parallel modernize-fields-test- (t/testing "some extra tests for Field clause canonicalization to the modern `:field` clause."+(deftest ^:parallel empty-test+ (testing "test a query with :is-empty"+ (is (= {:query {:filter [:and+ [:> [:field 4 nil] 1]+ [:is-empty [:field 7 nil]]+ [:= [:field 5 nil] "abc"]+ [:between [:field 9 nil] 0 25]]}}+ (#'mbql.normalize/normalize {:query {:filter [:and+ [:> [:field-id 4] 1]+ [:is-empty [:field-id 7]]+ [:and+ [:= [:field-id 5] "abc"]+ [:between [:field-id 9] 0 25]]]}})))))++(deftest ^:parallel modernize-fields-test+ (testing "some extra tests for Field clause canonicalization to the modern `:field` clause."(doseq [[form expected] {[:=[:datetime-field [:joined-field "source" [:field-id 100]] :month]"2021-02-18"]@@ -1427,75 +1430,94 @@[:binning-strategy [:field 1] :bin-width][:field 1 {:binning {:strategy :bin-width}}]}]- (t/testing (pr-str form)- (t/is (= expected- (#'mbql.normalize/canonicalize-mbql-clauses form)))))))--(t/deftest ^:parallel modernize-fields-e2e-test- (t/testing "Should be able to modernize legacy MBQL '95 Field clauses"- (t/is (= {:database 1- :type :query- :query {:source-table 3- :breakout [[:field 11 {:temporal-unit :month}]]- :aggregation [[:count]]}}- (mbql.normalize/normalize- {:database 1- :type :query- :query {:source-table 3- :breakout [[:datetime-field 11 :month]]- :aggregation [[:count]]}}))))-- (t/testing "Fixes for stuff like old-style breakout with a single clause should still work with the new `:field` clause"- (t/is (= {:database 1- :type :query- :query {:filter [:> [:field "count" {:base-type :type/Integer}] 5]- :source-query {:source-table 2- :aggregation [[:count]]- :breakout [[:field 3 {:temporal-unit :month, :source-field 4}]]}}}- (mbql.normalize/normalize- {:database 1- :type :query- :query {:filter [:> [:field "count" {:base-type :type/Integer}] 5]- :source-query {:source-table 2- :aggregation [[:count]]- :breakout [:field 3 {:temporal-unit :month, :source-field 4}]}}})))))--(t/deftest ^:parallel normalize-fragment-test- (t/testing "normalize-fragment"- (t/testing "shouldn't try to do anything crazy non-standard MBQL clauses like `:dimension` (from automagic dashboards)"- (t/is (= [:time-interval [:dimension "JoinDate"] -30 :day]- (mbql.normalize/normalize-fragment [:query :filter]- ["time-interval" ["dimension" "JoinDate"] -30 "day"]))))-- (t/testing "should be able to modernize Fields anywhere we find them"- (t/is (= [[:> [:field 1 nil] 3]- [:and- [:= [:field 2 nil] 2]- [:segment 1]]- [:metric 1]]- (mbql.normalize/normalize-fragment nil- [[:> [:field-id 1] 3]- ["and" [:= ["FIELD-ID" 2] 2]- ["segment" 1]]- [:metric 1]]))))-- (t/testing "Should be able to modern Field options anywhere"- (t/is (= [:field 2 {:temporal-unit :day}]- (mbql.normalize/normalize-fragment nil- [:field 2 {"temporal-unit" "day"}]))))))--(t/deftest ^:parallel normalize-fragment-filter-test- (t/is (= [:!=- [:expression "expr" {:base-type :type/Date}]- [:field 66302 {:base-type :type/DateTime}]]- (mbql.normalize/normalize-fragment- [:query :filter]- ["!="- [:expression "expr" {:base-type :type/Date}]- [:field 66302 {:base-type :type/DateTime}]]))))--(t/deftest ^:parallel do-not-normalize-fingerprints-test- (t/testing "Numbers in fingerprints shouldn't get normalized"+ (testing (pr-str form)+ (is (= expected+ (#'mbql.normalize/normalize form)))))))++(deftest ^:parallel modernize-fields-e2e-test+ (testing "Should be able to modernize legacy MBQL '95 Field clauses"+ (is (= {:database 1+ :type :query+ :query {:source-table 3+ :breakout [[:field 11 {:temporal-unit :month}]]+ :aggregation [[:count]]}}+ (mbql.normalize/normalize+ {:database 1+ :type :query+ :query {:source-table 3+ :breakout [[:datetime-field 11 :month]]+ :aggregation [[:count]]}})))))++(deftest ^:parallel modernize-fields-e2e-test-2+ (testing "Fixes for stuff like old-style breakout with a single clause should still work with the new `:field` clause"+ (is (= {:database 1+ :type :query+ :query {:filter [:> [:field "count" {:base-type :type/Integer}] 5]+ :source-query {:source-table 2+ :aggregation [[:count]]+ :breakout [[:field 3 {:temporal-unit :month, :source-field 4}]]}}}+ (mbql.normalize/normalize+ {:database 1+ :type :query+ :query {:filter [:> [:field "count" {:base-type :type/Integer}] 5]+ :source-query {:source-table 2+ :aggregation [[:count]]+ :breakout [:field 3 {:temporal-unit :month, :source-field 4}]}}})))))++(deftest ^:parallel normalize-breakout-test+ (are [breakouts expected] (= {:database 1+ :type :query+ :query {:breakout expected}}+ (mbql.normalize/normalize+ {:database 1+ :type :query+ :query {:breakout breakouts}}))+ 1 [[:field 1 nil]]+ [1] [[:field 1 nil]]+ [:field-id 1] [[:field 1 nil]]+ [[:field-id 1]] [[:field 1 nil]]+ [:field 1 nil] [[:field 1 nil]]+ [[:field 1 nil]] [[:field 1 nil]]+ [1 2] [[:field 1 nil] [:field 2 nil]]+ [[:field-id 1] [:field-id 2]] [[:field 1 nil] [:field 2 nil]]))++(deftest ^:parallel normalize-by-schema-test+ (testing "normalize with an explicit schema"+ (is (= [:time-interval [:field 1 nil] -30 :day]+ (mbql.normalize/normalize ::mbql.s/Filter ["time-interval" ["field-id" 1] -30 "day"])))))++(deftest ^:parallel normalize-by-schema-test-2+ (testing "normalize with an explicit schema"+ (testing "should be able to modernize Fields anywhere we find them"+ (is (= [[:> [:field 1 nil] 3]+ [:and+ [:= [:field 2 nil] 2]+ [:segment 1]]+ [:metric 1]]+ (mbql.normalize/normalize [:sequential ::mbql.s/Filter]+ [[:> [:field-id 1] 3]+ ["and" [:= ["FIELD-ID" 2] 2]+ ["segment" 1]]+ [:metric 1]]))))))++(deftest ^:parallel normalize-by-schema-test-3+ (testing "normalize with an explicit schema"+ (testing "Should be able to modern Field options anywhere"+ (is (= [:field 2 {:temporal-unit :day}]+ (mbql.normalize/normalize ::mbql.s/field [:field 2 {"temporal-unit" "day"}]))))))++(deftest ^:parallel normalize-filter-by-schema-test+ (is (= [:!=+ [:expression "expr" {:base-type :type/Date}]+ [:field 66302 {:base-type :type/DateTime}]]+ (mbql.normalize/normalize+ ::mbql.s/Filter+ ["!="+ [:expression "expr" {:base-type :type/Date}]+ [:field 66302 {:base-type :type/DateTime}]]))))++(deftest ^:parallel do-not-normalize-fingerprints-test+ (testing "Numbers in fingerprints shouldn't get normalized"(let [fingerprint {:global {:distinct-count 1, :nil% 0}:type {:type/Number {:min 1:q1 1@@ -1503,147 +1525,156 @@:max 1:sd 0:avg 1}}}]- (t/is (= fingerprint- (mbql.normalize/normalize fingerprint)))+ (is (= fingerprint+ (mbql.normalize/normalize ::lib.schema.metadata.fingerprint/fingerprint fingerprint)))(doseq [path [[:query :source-metadata][:metabase-enterprise.sandbox.query-processor.middleware.sandboxing/original-metadata][:info :metadata/model-metadata]- [:info :pivot/result-metadata][:query :joins 0 :source-metadata]]]- (t/testing (pr-str path)+ (testing (pr-str path)(let [query (-> {:query{:source-query{:native "SELECT USER_ID FROM ORDERS LIMIT 1":parameters [{:type :category:target [:variable [:template-tag "sandbox"]]:value "1"}]}- :joins [{:alias "A"}]- :database 1}}+ :joins [{:alias "A"}]+ :database 1}}(assoc-in path [{:name "USER_ID":display_name "USER_ID":base_type :type/Integer:field_ref [:field "USER_ID" {:base-type :type/Integer}]:fingerprint fingerprint}]))]- (t/is (= query- (mbql.normalize/normalize query)))))))))--(t/deftest ^:parallel error-messages-test- (t/testing "Normalization error messages should be sane"- (let [bad-query {:database 1- :type :native- :native {:template-tags {100 [:field-id "WOW"]}}}]- (t/is (thrown-with-msg?- #?(:clj clojure.lang.ExceptionInfo :cljs cljs.core.ExceptionInfo)- #"Error normalizing query"- (mbql.normalize/normalize bad-query)))- (let [e (try- (mbql.normalize/normalize bad-query)- nil- (catch #?(:clj Throwable :cljs js/Error) e- e))]- (t/testing "Should have meaningful ex-data"- (t/is (instance? #?(:clj clojure.lang.ExceptionInfo :cljs cljs.core.ExceptionInfo) e))- (t/is (= {:query bad-query}- (ex-data e))))- (t/testing "\nParent exception(s) should be even more specific"- (let [cause (ex-cause e)]- (t/is (some? cause))- (t/is (re-find #"Error normalizing form:" (ex-message cause)))- (t/is (= {:form bad-query- :special-fn nil}- (ex-data cause)))))))))--(t/deftest ^:parallel remove-unsuitable-temporal-units-test- (t/testing "Ignore unsuitable temporal units (such as bucketing a Date by minute) rather than erroring (#16485)"+ (is (= query+ (mbql.normalize/normalize query)))))))))++(deftest ^:parallel remove-unsuitable-temporal-units-test+ (testing "Ignore unsuitable temporal units (such as bucketing a Date by minute) rather than erroring (#16485)";; this query is with legacy MBQL syntax. It's just copied directly from the original issue(let [query {:query {:filter ["<"["datetime-field" ["field-literal" "date_seen" "type/Date"] "minute"]"2021-05-01T12:30:00"]}}]- (t/is (= {:query {:filter [:<- [:field "date_seen" {:base-type :type/Date}]- "2021-05-01T12:30:00"]}}- (mbql.normalize/normalize query))))))--(t/deftest ^:parallel normalize-aggregation-ref-test- (t/are [clause] (= {:database 1- :type :query- :query {:order-by [[:asc [:aggregation 0]]]}}- (mbql.normalize/normalize- {:database 1- :type :query- :query {:order-by [[:asc clause]]}}))+ (is (= {:query {:filter [:<+ [:field "date_seen" {:base-type :type/Date}]+ "2021-05-01T12:30:00"]}}+ (mbql.normalize/normalize query))))))++(deftest ^:parallel normalize-aggregation-ref-test+ (are [clause] (= {:database 1+ :type :query+ :query {:order-by [[:asc [:aggregation 0]]]}}+ (mbql.normalize/normalize+ {:database 1+ :type :query+ :query {:order-by [[:asc clause]]}}))[:aggregation 0 nil][:aggregation 0 {}][:aggregation 0]))-(t/deftest ^:parallel normalize-info-test- (t/is (= {:database 1- :type :query- :query {:source-table 2}- :info {:context :collection}}- (mbql.normalize/normalize {:database 1- :type :query- :query {:source-table 2}- :info {:context "collection"}}))))--(t/deftest ^:parallel dont-remove-nil-parameter-values-test- (t/testing "The FE code is extremely dumb and will consider parameters to be changed (haveParametersChanged) if we strip out value: nil"+(deftest ^:parallel dont-remove-nil-parameter-values-test+ (testing "The FE code is extremely dumb and will consider parameters to be changed (haveParametersChanged) if we strip out value: nil"(let [query {:type :native:database 1:parameters [{:id "d98c3875-e0f1-9270-d36a-5b729eef938e":target [:dimension [:template-tag "category"]]:type :category:value nil}]}]- (t/is (=? {:parameters [{:id "d98c3875-e0f1-9270-d36a-5b729eef938e"- :value nil}]}- (mbql.normalize/normalize query))))))+ (is (=? {:parameters [{:id "d98c3875-e0f1-9270-d36a-5b729eef938e"+ :value nil}]}+ (mbql.normalize/normalize query))))))-(t/deftest ^:parallel dont-normalize-actions-row-test+(deftest ^:parallel dont-normalize-actions-row-test(doseq [k [:create-row :update-row]]- (t/testing k+ (testing k(let [query {:database 1:type :query:query {:source-table 2}k {"x" 1, "y" {"z" 2}, "a" nil}}]- (t/is (= query- (mbql.normalize/normalize query)))))))--(t/deftest ^:parallel normalize-offset-test- (t/is (=? [:offset- {:effective-type :type/Float, :lib/uuid string?}- [:field- 1- {:base-type :type/Float, :effective-type :type/Float}]- -1]- (mbql.normalize/normalize- ["offset"- {"effective-type" "type/Float"}- ["field"- 1- {"base-type" "type/Float", "effective-type" "type/Float"}]- -1]))))--(t/deftest ^:parallel normalize-legacy-filters-test- (t/is (= {:database 1- :type :query- :query {:filters [:and- [:!= [:get-hour [:field 1 nil]] 0 23]- [:between [:field 1 nil] "2024-10-05" "2024-12-08"]]}}- (mbql.normalize/normalize {:database 1- :type :query- :query {:filters [:and- [:!= [:field 1 {:temporal-unit :hour-of-day}] 0 23]- [:between [:field 1 nil] "2024-10-05" "2024-12-08"]]}}))))--(t/deftest ^:parallel normalize-datetime-test- (t/is (= [:datetime ""]- (mbql.normalize/normalize-tokens [:datetime ""])))- (t/testing "if we add other options, they are preserved (and don't break anything)"- (t/is (= [:datetime "" {:x "x"}]- (mbql.normalize/normalize-tokens [:datetime "" {"x" "x"}]))))- (t/is (= [:datetime "" {:mode :iso}]- (mbql.normalize/normalize-tokens [:datetime "" {:mode :iso}])))- (t/is (= [:datetime "" {:mode :iso}]- (mbql.normalize/normalize-tokens [:datetime "" {:mode "iso"}])))- (t/is (= [:datetime "" {:mode :iso}]- (mbql.normalize/normalize-tokens ["datetime" "" {"mode" "iso"}]))))+ (is (= query+ (mbql.normalize/normalize query)))))))++(deftest ^:parallel normalize-offset-test+ (is (=? [:offset+ {:effective-type :type/Float, :lib/uuid string?}+ [:field+ 1+ {:base-type :type/Float, :effective-type :type/Float}]+ -1]+ (mbql.normalize/normalize+ ::mbql.s/offset+ ["offset"+ {"effective-type" "type/Float"}+ ["field"+ 1+ {"base-type" "type/Float", "effective-type" "type/Float"}]+ -1]))))++(deftest ^:parallel normalize-legacy-filters-test+ (is (= {:database 1+ :type :query+ :query {:filter [:and+ [:!= [:get-hour [:field 1 nil]] 0 23]+ [:between [:field 1 nil] "2024-10-05" "2024-12-08"]]}}+ (mbql.normalize/normalize {:database 1+ :type :query+ :query {:filter [:and+ [:!= [:field 1 {:temporal-unit :hour-of-day}] 0 23]+ [:between [:field 1 nil] "2024-10-05" "2024-12-08"]]}}))))++(deftest ^:parallel normalize-datetime-test+ (is (= [:datetime ""]+ (mbql.normalize/normalize [:datetime ""])))+ (testing "if we add other options, they are preserved (and don't break anything)"+ (is (= [:datetime "" {:x "x"}]+ (mbql.normalize/normalize [:datetime "" {"x" "x"}]))))+ (is (= [:datetime "" {:mode :iso}]+ (mbql.normalize/normalize [:datetime "" {:mode :iso}])+ (mbql.normalize/normalize [:datetime "" {:mode "iso"}])+ (mbql.normalize/normalize ["datetime" "" {"mode" "iso"}]))))++(deftest ^:parallel normalize-datetime-test-2+ (is (= [:datetime+ [:field "DATE_TIME" {:effective-type :type/*, :base-type :type/*}]+ {:mode :simple-bytes}]+ (mbql.normalize/normalize+ ["datetime"+ ["field" "DATE_TIME" {:effective-type "type/*", :base-type "type/*"}]+ {"mode" "simplebytes"}]))))++(deftest ^:parallel normalize-time-interval-test+ (testing (str "If you specify a `:temporal-unit` for the Field inside a `:time-interval`, remove it. The unit"+ " in `:time-interval` takes precedence.")+ (is (= [:time-interval [:field 1 nil] -10 :minute]+ (mbql.normalize/normalize ::mbql.s/time-interval [:time-interval [:field 1 {:temporal-unit :hour}] -10 :minute])))))++(deftest ^:parallel normalize-incomplete-field-refs-test+ (is (= {:database 1+ :type :query+ :query {:source-table 2+ :fields [[:field 3 nil]]+ :order-by [[:asc [:field 3 nil]]]}}+ (mbql.normalize/normalize {:database 1+ :type "query"+ :query {:source-table 2+ :fields [["field" 3 nil]]+ :order-by [[:asc ["field" 3]]+ [:asc ["field" 3]]]}}))))++(deftest ^:parallel normalize-relative-datetime-test+ (testing "Fix incorrect :relative-datetime clauses that use the wrong key (e.g. :now) instead of :current"+ (is (= [:relative-datetime :current]+ (mbql.normalize/normalize '(:relative-datetime :now))+ (mbql.normalize/normalize '(:relative-datetime "now"))+ (mbql.normalize/normalize '(:relative-datetime "current"))))))++(deftest ^:parallel normalize-case-test+ (is (= [:case+ [[[:> [:field 3 nil] 0]+ [:field 3 nil]]] ; drop the extra 3rd arg in the subclause+ {:default [:field 5 nil]}]+ (mbql.normalize/normalize+ ["case"+ [[[">" ["field" 3 nil] 0]+ ["field" 3]+ nil]]+ {"default" ["field" 5 {}]}]))))diff --git a/test/metabase/legacy_mbql/schema_test.cljc b/test/metabase/legacy_mbql/schema_test.cljcindex 5998fdf249e..e46901d048d 100644--- a/test/metabase/legacy_mbql/schema_test.cljc+++ b/test/metabase/legacy_mbql/schema_test.cljc@@ -67,7 +67,7 @@[:field 1 {:binning {:strategy :fake}}] false}](testing (pr-str clause)(is (= expected- (mr/validate mbql.s/field clause)))))))+ (mr/validate ::mbql.s/field clause)))))))(deftest ^:parallel validate-template-tag-names-test(testing "template tags with mismatched keys/`:names` in definition should be disallowed\n"@@ -80,10 +80,10 @@:type :text}}}}bad-query (assoc-in correct-query [:native :template-tags "foo" :name] "filter")](testing (str "correct-query " (pr-str correct-query))- (is (not (me/humanize (mr/explain mbql.s/Query correct-query))))+ (is (not (me/humanize (mr/explain ::mbql.s/Query correct-query))))(is (mr/validate ::mbql.s/Query correct-query)))(testing (str "bad-query " (pr-str bad-query))- (is (me/humanize (mr/explain mbql.s/Query bad-query)))+ (is (me/humanize (mr/explain ::mbql.s/Query bad-query)))(is (= {:native {:template-tags ["keys in template tag map must match the :name of their values"]}}(me/humanize (mr/explain ::mbql.s/Query bad-query))))))))@@ -100,7 +100,7 @@[:coalesce [:sum [:field 36 {:base-type :type/Float}]] 1]]{:name "Avg discount", :display-name "Avg discount"}]]}:parameters []}]- (is (not (me/humanize (mr/explain mbql.s/Query query)))))))+ (is (not (me/humanize (mr/explain ::mbql.s/Query query)))))))(deftest ^:parallel year-of-era-test(testing "year-of-era aggregations should be recognized"@@ -111,18 +111,18 @@:aggregation [[:count]],:breakout [[:field 49 {:base-type :type/Date, :temporal-unit :year-of-era, :source-field 43}]]}:parameters []}]- (is (not (me/humanize (mr/explain mbql.s/Query query)))))))+ (is (not (me/humanize (mr/explain ::mbql.s/Query query)))))))(deftest ^:parallel aggregation-reference-test(are [schema] (nil? (me/humanize (mr/explain schema [:aggregation 0])))- mbql.s/aggregation- mbql.s/Reference))+ ::mbql.s/aggregation+ ::mbql.s/Reference))(deftest ^:parallel native-query-test(let [parameter-dimension [:dimension [:template-tag "date_range"]]template-tag-dimension [:field 2 nil]]- (is (nil? (me/humanize (mr/explain mbql.s/dimension parameter-dimension))))- (is (nil? (me/humanize (mr/explain mbql.s/field template-tag-dimension))))+ (is (nil? (me/humanize (mr/explain ::mbql.s/dimension parameter-dimension))))+ (is (nil? (me/humanize (mr/explain ::mbql.s/field template-tag-dimension))))(let [parameter {:type :date/range:name "created_at":target parameter-dimension@@ -133,7 +133,7 @@:widget-type :date/all-options:dimension template-tag-dimension}](is (nil? (me/humanize (mr/explain ::lib.schema.parameter/parameter parameter))))- (is (nil? (me/humanize (mr/explain mbql.s/TemplateTag template-tag))))+ (is (nil? (me/humanize (mr/explain ::mbql.s/TemplateTag template-tag))))(let [query {:database 1:type :native:native {:query (str/join \newline ["SELECT dayname(\"TIMESTAMP\") as \"day\""@@ -143,7 +143,7 @@" LIMIT 1"]):template-tags {"date_range" template-tag}:parameters [parameter]}}]- (is (nil? (me/humanize (mr/explain mbql.s/Query query))))))))+ (is (nil? (me/humanize (mr/explain ::mbql.s/Query query))))))))(deftest ^:parallel value-test(let [value [:value@@ -155,12 +155,12 @@:database_type "inet":name "ip"}]](are [schema] (not (me/humanize (mr/explain schema value)))- mbql.s/value- @#'mbql.s/EqualityComparable- [:or mbql.s/absolute-datetime mbql.s/value])))+ ::mbql.s/value+ ::mbql.s/EqualityComparable+ [:or ::mbql.s/absolute-datetime ::mbql.s/value])))(deftest ^:parallel expression-value-wrapped-literals-test- (are [value] (not (me/humanize (mr/explain mbql.s/MBQLQuery+ (are [value] (not (me/humanize (mr/explain ::mbql.s/MBQLQuery{:source-table 1, :expressions {"expr" [:value value nil]}})))"""192.168.1.1"@@ -173,8 +173,8 @@false))(deftest ^:parallel expression-unwrapped-literals-test- (are [value] (= {:expressions {"expr" ["valid instance of one of these MBQL clauses: :expression, :field"]}}- (me/humanize (mr/explain mbql.s/MBQLQuery+ (are [value] (= {:expressions {"expr" ["valid instance of one of these MBQL clauses: expression, field"]}}+ (me/humanize (mr/explain ::mbql.s/MBQLQuery{:source-table 1, :expressions {"expr" value}})))"""192.168.1.1"@@ -189,19 +189,19 @@(deftest ^:parallel or-test(are [schema expected] (= expected(mu.humanize/humanize (mr/explain schema [:value "192.168.1.1" {:base_type :type/FK}])))- mbql.s/absolute-datetime+ ::mbql.s/absolute-datetime"not an :absolute-datetime clause"- [:or mbql.s/absolute-datetime]+ [:or ::mbql.s/absolute-datetime]"not an :absolute-datetime clause"- mbql.s/value+ ::mbql.s/value[nil nil {:base_type "Not a valid base type: :type/FK"}]- [:or mbql.s/value]+ [:or ::mbql.s/value][nil nil {:base_type "Not a valid base type: :type/FK"}]- [:or mbql.s/absolute-datetime :string mbql.s/value]+ [:or ::mbql.s/absolute-datetime :string ::mbql.s/value]["not an :absolute-datetime clause""should be a string"[nil nil {:base_type "Not a valid base type: :type/FK"}]]))@@ -251,7 +251,7 @@[:datetime "" {}][:datetime "" {:mode :iso}][:datetime 10 {:mode :unix-seconds}]]]- (is (mr/validate mbql.s/datetime expr))))+ (is (mr/validate ::mbql.s/datetime expr))))(deftest ^:parallel normalize-source-metadata-test(testing "normalize-source-metadata"@@ -312,3 +312,8 @@:binning_info {"binning_strategy" "default""strategy" "num-bins""num-bins" 100}}))))++(deftest ^:parallel remove-inner-ident-test+ (testing "Remove deprecated keys like :model/inner_ident automatically (GIT-8399)"+ (is (= {:base_type :type/*, :name "X", :display_name "X"}+ (lib/normalize ::mbql.s/legacy-column-metadata {:name "X", :model/inner_ident "wow"})))))diff --git a/test/metabase/lib/query_test.cljc b/test/metabase/lib/query_test.cljcindex a4f497b6005..69aebec9fa9 100644--- a/test/metabase/lib/query_test.cljc+++ b/test/metabase/lib/query_test.cljc@@ -209,7 +209,6 @@:lib/type :mbql/query:lib/metadata lib.metadata.protocols/cached-metadata-provider?:stages [{:lib/type :mbql.stage/native- :template-tags {}:native "select * from products limit 3;"}]:lib.convert/converted? true:type (symbol "nil #_\"key is not present.\"")diff --git a/test/metabase/lib/schema/expression/temporal_test.cljc b/test/metabase/lib/schema/expression/temporal_test.cljcindex 96c709daba0..4d663e680a8 100644--- a/test/metabase/lib/schema/expression/temporal_test.cljc+++ b/test/metabase/lib/schema/expression/temporal_test.cljc@@ -2,6 +2,7 @@(:require[clojure.test :refer [are deftest is testing]][malli.error :as me]+ [metabase.lib.core :as lib][metabase.lib.schema][metabase.lib.schema.expression :as expression][metabase.lib.schema.expression.temporal :as temporal]@@ -149,3 +150,16 @@;; mode is not allowed to be nil[:get-week default-options "2023-05-25" nil][nil nil nil ["should be either :iso, :us or :instance" "Valid :get-week clause"]]))++(deftest ^:parallel normalize-datetime-test+ (is (= [:datetime+ {:lib/uuid "8b343e3b-a549-4d22-87a3-d5888793209b", :mode :simple-bytes, :lib/expression-name "parsed_date"}+ [:field+ {:lib/uuid "9da67f88-9c46-4917-b964-07806f60870c", :effective-type :type/*, :base-type :type/*}+ "DATE_TIME"]]+ (lib/normalize+ ["datetime"+ {:lib/uuid "8b343e3b-a549-4d22-87a3-d5888793209b", :mode "simple-bytes", :lib/expression-name "parsed_date"}+ ["field"+ {:lib/uuid "9da67f88-9c46-4917-b964-07806f60870c", :effective-type "type/*", :base-type "type/*"}+ "DATE_TIME"]]))))diff --git a/test/metabase/lib/schema/metadata_test.cljc b/test/metabase/lib/schema/metadata_test.cljcindex ec44e69d532..5d7f5cef0a5 100644--- a/test/metabase/lib/schema/metadata_test.cljc+++ b/test/metabase/lib/schema/metadata_test.cljc@@ -1,6 +1,6 @@(ns metabase.lib.schema.metadata-test(:require- [clojure.test :refer [are deftest is]]+ [clojure.test :refer [are deftest is testing]][clojure.walk :as walk][metabase.legacy-mbql.schema :as mbql.s][metabase.lib.core :as lib]@@ -121,3 +121,8 @@:type/Number {:q1 1.459}}}}))::mbql.s/legacy-column-metadata::lib.schema.metadata/lib-or-legacy-column))++(deftest ^:parallel remove-inner-ident-test+ (testing "Remove deprecated keys like :model/inner_ident automatically (GIT-8399)"+ (is (= {:lib/type :metadata/column, :base-type :type/*, :name "X"}+ (lib/normalize ::lib.schema.metadata/column {:name "X", :model/inner_ident "wow"})))))diff --git a/test/metabase/queries/schema_test.clj b/test/metabase/queries/schema_test.cljindex 489c71f786d..714858715dc 100644--- a/test/metabase/queries/schema_test.clj+++ b/test/metabase/queries/schema_test.clj@@ -10,3 +10,7 @@:stages [["Initial MBQL stage must have either :source-table or :source-card (but not both)"]]}}(me/humanize (mr/explain ::queries.schema/card{:dataset_query {:lib/type :mbql/query, :database 2378, :stages [{:lib/type :mbql.stage/mbql}]}})))))++(deftest ^:parallel set-invalid-metadata-to-nil-test+ (is (= {:result_metadata nil}+ (queries.schema/normalize-card {:result_metadata [{}]}))))diff --git a/test/metabase/query_processor/card_test.clj b/test/metabase/query_processor/card_test.cljindex 109d6a11a60..e40db80d14f 100644--- a/test/metabase/query_processor/card_test.clj+++ b/test/metabase/query_processor/card_test.clj@@ -3,7 +3,8 @@{:clj-kondo/config '{:linters;; allowing `with-temp` here for now since this tests the REST API which doesn't fully use;; metadata providers.- {:discouraged-var {metabase.test/with-temp {:level :off}}}}}+ {:discouraged-var {metabase.test/with-temp {:level :off}+ toucan2.tools.with-temp/with-temp {:level :off}}}}}(:require[clojure.string :as str][clojure.test :refer :all]diff --git a/test/metabase/query_processor/middleware/reconcile_breakout_and_order_by_bucketing_test.clj b/test/metabase/query_processor/middleware/reconcile_breakout_and_order_by_bucketing_test.cljindex ddcd84a22d8..5f833a91c76 100644--- a/test/metabase/query_processor/middleware/reconcile_breakout_and_order_by_bucketing_test.clj+++ b/test/metabase/query_processor/middleware/reconcile_breakout_and_order_by_bucketing_test.clj@@ -122,7 +122,7 @@(deftest ^:parallel remove-duplicate-order-by-clauses-test(testing (str "if a Field is referenced twice and we bucket an unbucketed reference, creating duplicate order-by "- "clauses, we should remove them, as it is illegal in MBQL 2000+")+ "clauses, we should remove them, as it is illegal in MBQL 4+")(is (=? (mbql-query:breakout [[:field 1 {:temporal-unit :day}]]:order-by [[:asc [:field 1 {:temporal-unit :day}]]])diff --git a/test/metabase/query_processor/streaming_test.clj b/test/metabase/query_processor/streaming_test.cljindex bafa3e537ef..3bfb8c32551 100644--- a/test/metabase/query_processor/streaming_test.clj+++ b/test/metabase/query_processor/streaming_test.clj@@ -427,7 +427,6 @@:type :query:query {:source-table (mt/id :venues):limit 1}}-:viz-settings {:column_settings {},:table.columns[{:name "NAME", :fieldRef [:field (mt/id :venues :name) nil], :enabled true}@@ -436,17 +435,14 @@{:name "LATITUDE", :fieldRef [:field (mt/id :venues :latitude) nil], :enabled false}{:name "LONGITUDE", :fieldRef [:field (mt/id :venues :longitude) nil], :enabled false}{:name "PRICE", :fieldRef [:field (mt/id :venues :price) nil], :enabled true}]}-:assertions {:csv (fn [results](is (= [["Name" "ID" "Category ID" "Price"]["Red Medicine" "1" "4" "3"]](parse-csv-results results))))-:json (fn [results](is (= [["Name" "ID" "Category ID" "Price"]["Red Medicine" "1" "4" "3"]](parse-json-results results))))-:xlsx (fn [results](is (= [["Name" "ID" "Category ID" "Price"]["Red Medicine" 1.0 4.0 3.0]]diff --git a/test/metabase/segments/models/segment_test.clj b/test/metabase/segments/models/segment_test.cljindex d5bb7d45ac7..c23d4d005a5 100644--- a/test/metabase/segments/models/segment_test.clj+++ b/test/metabase/segments/models/segment_test.clj@@ -18,7 +18,7 @@(deftest ^:parallel dont-explode-on-way-out-from-db-test(testing "`segment-definition`s should avoid explosions coming out of the DB..."- (is (= nil+ (is (= {:filter [:field 1000 nil]}((:out @#'segment/transform-segment-definition)(json/encode{:filter 1000}))))@@ -26,7 +26,7 @@(is (thrown?Exception((:in @#'segment/transform-segment-definition)- {:filter 1000}))))))+ {:filter "X"}))))))(deftest update-test(testing "Updating"diff --git a/test/metabase/test/data.clj b/test/metabase/test/data.cljindex 571da5f4b4e..bf362bacd3d 100644--- a/test/metabase/test/data.clj+++ b/test/metabase/test/data.clj@@ -185,7 +185,7 @@#_{:clj-kondo/ignore [:deprecated-var]}{:database (id):type :native- :native (mbql.normalize/normalize-fragment [:native] inner-native-query)})+ :native (mbql.normalize/normalize ::mbql.s/NativeQuery inner-native-query)})(defn run-mbql-query* [query];; catch the Exception and rethrow with the query itself so we can have a little extra info for debugging if it fails.diff --git a/test/metabase/util/malli/humanize_test.cljc b/test/metabase/util/malli/humanize_test.cljcindex cd3fba15264..ec3cdaa1b50 100644--- a/test/metabase/util/malli/humanize_test.cljc+++ b/test/metabase/util/malli/humanize_test.cljc@@ -54,7 +54,7 @@(let [error (mr/explain[:or:int- mbql.s/value]+ ::mbql.s/value][:value "192.168.1.1" {:base_type :type/FK}])](are [f expected] (= expected(f error))@@ -71,7 +71,7 @@[:map[:x [:or:int- mbql.s/value]]]+ ::mbql.s/value]]]{:x [:value "192.168.1.1" {:base_type :type/FK}]})](are [f expected] (= expected(f error))@@ -124,7 +124,7 @@(deftest ^:parallel ref-test-2(are [f expected] (= expected- (f (mr/explain [:or mbql.s/value] [:value "192.168.1.1" {:base_type :type/FK}])))+ (f (mr/explain [:or ::mbql.s/value] [:value "192.168.1.1" {:base_type :type/FK}])))me/humanize[nil nil {:base_type ["Not a valid base type: :type/FK"]}]diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.cljindex 51b30692727..4c6de0b52e4 100644--- a/test/metabase/xrays/automagic_dashboards/core_test.clj+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj@@ -773,7 +773,7 @@(let [model-card (t2/select-one :model/Card model-id)dashboard (magic/automagic-analysis model-card nil)parameter-mappings (eduction (comp (keep :parameter_mappings) cat) (:dashcards dashboard))- dimension? (mr/validator mbql.s/dimension)]+ dimension? (mr/validator ::mbql.s/dimension)](is (every? (comp dimension? :target) parameter-mappings)))))))(deftest test-table-title-testdiff --git a/test/metabase/xrays/transforms/specs_test.clj b/test/metabase/xrays/transforms/specs_test.cljindex 26da1d98a1f..65897588dd3 100644--- a/test/metabase/xrays/transforms/specs_test.clj+++ b/test/metabase/xrays/transforms/specs_test.clj@@ -6,10 +6,11 @@(deftest ^:parallel extract-dimensions-test(are [arg expected] (= expected(#'tf.specs/extract-dimensions arg))- [:dimension "foo"] ["foo"]- [:some-op 12 {:k [:dimension "foo"]}] ["foo"]- nil nil- [1 2 3] nil))+ [:dimension "foo"] ["foo"]+ ;; this is not a real usage but we just want to check that `extract-dimensions` can find the `:dimension`+ [:starts-with 12 {:k [:dimension "foo"]}] ["foo"]+ nil nil+ [1 2 3] nil))(deftest ^:parallel validate-yaml-test(testing "All specs should be valid YAML (the parser will raise an exception if not) and conforming to the schema."