Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[MBQL lib] Export some healthy API functions; drop some API nses (#69666)

Braden Shepherdsond2e8fcac5b1e
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index dc47a3256f6..3f6a2dd1029 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -738,17 +738,10 @@
{:team "Querying"
:api #{metabase.lib.core
metabase.lib.equality
- metabase.lib.field
metabase.lib.init
metabase.lib.metadata
- metabase.lib.metadata.cached-provider
- metabase.lib.metadata.column
- metabase.lib.metadata.invocation-tracker
metabase.lib.metadata.protocols
- metabase.lib.metadata.result-metadata
- metabase.lib.normalize
metabase.lib.options
- metabase.lib.query
metabase.lib.schema
metabase.lib.schema.actions
metabase.lib.schema.aggregation
@@ -772,14 +765,17 @@
metabase.lib.schema.validate
metabase.lib.types.isa
metabase.lib.util
- metabase.lib.util.match
- metabase.lib.walk}
+ metabase.lib.util.match}
:uses #{config
graph
legacy-mbql
types
util}
- :friends #{query-processor legacy-mbql}}
+ :friends #{legacy-mbql ;; Reuses internal schemas and a few helper functions. Deeply coupled to MBQL.
+ lib-be ;; BE-specific extension of lib; tight coupling is acceptable.
+ query-processor ;; Necessarily manipulates MBQL; better to pull most logic into a new module.
+ enterprise/sandbox ;; Really part of the QP, though its lives in the enterprise/ tree.
+ }}
lib-be
{:team "Querying"
diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.clj
index 317797ddc24..505c76b1dad 100644
--- a/src/metabase/driver_api/core.clj
+++ b/src/metabase/driver_api/core.clj
@@ -17,7 +17,6 @@
[metabase.legacy-mbql.util :as mbql.u]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
- [metabase.lib.field :as lib.field]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema.actions :as lib.schema.actions]
[metabase.lib.schema.common :as lib.schema.common]
@@ -93,7 +92,6 @@
database-routing/check-allowed-access!
events/publish-event!
lib-be/start-of-week
- lib.field/json-field?
lib-be/instance->metadata
lib.metadata/database
lib.metadata/field
@@ -115,6 +113,7 @@
lib/->legacy-MBQL
lib/->metadata-provider
lib/duplicate-column-error
+ lib/json-field?
lib/match-and-normalize-tag-name
lib/missing-column-error
lib/missing-table-alias-error
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index 0a7900ed05c..d11eac45891 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -285,6 +285,8 @@
fieldable-columns
fields
find-visible-column-for-ref
+ infer-has-field-values ; Single-use
+ json-field? ; Single-use
remove-field
with-fields]
[metabase.lib.field.util
@@ -437,6 +439,7 @@
can-run
can-save
check-card-overwrite
+ native?
preview-query
query
query-from-legacy-inner-query
diff --git a/src/metabase/lib/field.cljc b/src/metabase/lib/field.cljc
index ec9628ff797..a4e899847a4 100644
--- a/src/metabase/lib/field.cljc
+++ b/src/metabase/lib/field.cljc
@@ -714,9 +714,9 @@
query stage-number)]
(lib.equality/find-matching-column query stage-number field-ref columns))))
-(defn json-field?
+(mu/defn json-field? :- :boolean
"Return true if field is a JSON field, false if not."
- [field]
+ [field :- [:maybe ::lib.schema.metadata/column]]
(some? (:nfc-path field)))
;;; yes, this is intentionally different from the version in `:metabase.lib.schema.metadata/column.has-field-values`.
diff --git a/src/metabase/lib/metadata/column.cljc b/src/metabase/lib/metadata/column.cljc
index ce70ae13b51..973a8d2b5fd 100644
--- a/src/metabase/lib/metadata/column.cljc
+++ b/src/metabase/lib/metadata/column.cljc
@@ -6,24 +6,20 @@
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]))
-(mr/def ::column-unique-key
- [:re
- #"^column-unique-key-v\d+\$.+$"])
-
(mr/def ::column-unique-key-version
nat-int?)
-(mu/defn- pack-unique-key :- ::column-unique-key
+(mu/defn- pack-unique-key :- ::lib.schema/column-unique-key
[version :- ::column-unique-key-version
column-key :- :string]
(str "column-unique-key-v" version "$" column-key))
(mu/defn- unpack-unique-key :- [:tuple #_version ::column-unique-key-version #_rest :string]
- [unique-key :- ::column-unique-key]
+ [unique-key :- ::lib.schema/column-unique-key]
(let [[_match version-string column-key] (re-find #"^column-unique-key-v(\d+)\$(.+$)" unique-key)]
[(parse-long version-string) column-key]))
-(mu/defn column-unique-key :- ::column-unique-key
+(mu/defn column-unique-key :- ::lib.schema/column-unique-key
"Create a unique key for returned column metadata. Treat this key as opaque!"
[col :- ::lib.metadata.calculation/returned-column]
(pack-unique-key 1 (:lib/desired-column-alias col)))
@@ -34,7 +30,7 @@
(column-with-unique-key query -1 unique-key))
([query :- ::lib.schema/query
stage-number :- :int
- unique-key :- ::column-unique-key]
+ unique-key :- ::lib.schema/column-unique-key]
(let [[version column-key] (unpack-unique-key unique-key)]
(m/find-first
(case version
diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljc
index c9936bec5d5..4ffec264570 100644
--- a/src/metabase/lib/schema.cljc
+++ b/src/metabase/lib/schema.cljc
@@ -44,6 +44,10 @@
metabase.lib.schema.expression.window/keep-me
metabase.lib.schema.filter/keep-me)
+(mr/def ::column-unique-key
+ [:re
+ #"^column-unique-key-v\d+\$.+$"])
+
(defn- normalize-stage-common [m]
(when-let [m (common/normalize-map m)]
(reduce
diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.clj
index f8c7a8918ff..d6835f00c3d 100644
--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj
+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj
@@ -5,7 +5,6 @@
[metabase.collections.models.collection :as collection]
[metabase.events.core :as events]
[metabase.lib.core :as lib]
- [metabase.lib.normalize :as lib.normalize]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.native-query-snippets.models.native-query-snippet.permissions :as snippet.perms]
@@ -29,7 +28,7 @@
(t2/deftransforms :model/NativeQuerySnippet
{:template_tags {:in mi/json-in
:out (comp (mi/catch-normalization-exceptions
- #(lib.normalize/normalize :metabase.lib.schema.template-tag/template-tag-map %))
+ #(lib/normalize :metabase.lib.schema.template-tag/template-tag-map %))
mi/json-out-without-keywordization)}})
(defmethod collection/allowed-namespaces :model/NativeQuerySnippet
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 303cef2dc72..d5f39736226 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -19,7 +19,6 @@
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
- [metabase.lib.normalize :as lib.normalize]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.metadata :as lib.schema.metadata]
@@ -810,7 +809,7 @@
(not verified-result-metadata?)
(contains? (t2/changes card) :type))
(card.metadata/populate-result-metadata changes))
- (m/update-existing :result_metadata #(some->> % (lib.normalize/normalize [:sequential ::lib.schema.metadata/lib-or-legacy-column])))))
+ (m/update-existing :result_metadata #(some->> % (lib/normalize [:sequential ::lib.schema.metadata/lib-or-legacy-column])))))
(t2/define-before-update :model/Card
[{:keys [verified-result-metadata?] :as card}]
diff --git a/src/metabase/queries/models/card/metadata.clj b/src/metabase/queries/models/card/metadata.clj
index 1083e01bbab..b8f89cef69d 100644
--- a/src/metabase/queries/models/card/metadata.clj
+++ b/src/metabase/queries/models/card/metadata.clj
@@ -6,7 +6,6 @@
[metabase.api.common :as api]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
- [metabase.lib.normalize :as lib.normalize]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.query-processor.metadata :as qp.metadata]
@@ -247,4 +246,4 @@ saved later when it is ready."
(log/debug "Attempting to infer result metadata for Card")
(assoc card :result_metadata (infer-metadata-with-model-overrides query card))))
;; now normalize the result metadata as needed so it passes the output schema check
- (m/update-existing :result_metadata #(some->> % (lib.normalize/normalize [:sequential ::lib.schema.metadata/lib-or-legacy-column]))))))
+ (m/update-existing :result_metadata #(some->> % (lib/normalize [:sequential ::lib.schema.metadata/lib-or-legacy-column]))))))
diff --git a/src/metabase/query_processor/middleware/fetch_source_query.clj b/src/metabase/query_processor/middleware/fetch_source_query.clj
index e4c72f1a2b4..3bbd82449cc 100644
--- a/src/metabase/query_processor/middleware/fetch_source_query.clj
+++ b/src/metabase/query_processor/middleware/fetch_source_query.clj
@@ -5,8 +5,8 @@
[metabase.driver.ddl.interface :as ddl.i]
[metabase.lib-be.core :as lib-be]
[metabase.lib.card :as lib.card]
+ [metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
- [metabase.lib.query :as lib.query]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.lib.schema.id :as lib.schema.id]
@@ -73,7 +73,7 @@
persisted-info)))]
(conj (vec (butlast stages)) last-stage)))
(update-query [query]
- (-> (lib.query/query metadata-providerable query)
+ (-> (lib/query metadata-providerable query)
;; Now that cards' queries can come out of the AppDB already in MBQL 5, complete with `:lib/uuid`s,
;; a card getting joined twice creates duplicate UUID errors!
;; This safely re-rolls all the `:lib/uuid`s on the card's query so they won't collide.
diff --git a/src/metabase/transforms/schema.clj b/src/metabase/transforms/schema.clj
index 91c43a4ca65..0bda7ccebd0 100644
--- a/src/metabase/transforms/schema.clj
+++ b/src/metabase/transforms/schema.clj
@@ -1,6 +1,6 @@
(ns metabase.transforms.schema
(:require
- [metabase.lib.metadata.column :as lib.metadata.column]
+ [metabase.lib.schema :as lib.schema]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.queries.schema :as queries.schema]
[metabase.util.malli.registry :as mr]
@@ -26,7 +26,7 @@
[:checkpoint-filter {:optional true} :string]
;; for mbql and python
[:checkpoint-filter-unique-key {:optional true}
- ::lib.metadata.column/column-unique-key]])
+ ::lib.schema/column-unique-key]])
(mr/def ::source-incremental-strategy
[:multi {:dispatch :type}
diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.clj
index c20f7d51f1d..f0c61c0d308 100644
--- a/src/metabase/transforms/util.clj
+++ b/src/metabase/transforms/util.clj
@@ -14,7 +14,6 @@
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
- [metabase.lib.query :as lib.query]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.models.interface :as mi]
[metabase.models.transforms.transform-run :as transform-run]
@@ -383,7 +382,7 @@
Returns the query unchanged on first run (no checkpoint) or for native queries without the checkpoint tag."
[query source-incremental-strategy checkpoint]
(if-let [checkpoint-value (next-checkpoint-value checkpoint)]
- (if (lib.query/native? query)
+ (if (lib/native? query)
;; native query with explicit checkpoint filter
(if (get-in query [:stages 0 :template-tags "checkpoint"])
(update query :parameters conj
@@ -401,7 +400,7 @@
Generates SQL that wraps the original query as a subquery and filters by `checkpoint_filter > (checkpoint_query)`. "
[outer-query driver {:keys [source-incremental-strategy] :as source} {checkpoint-query :query :as checkpoint}]
(let [{:keys [checkpoint-filter]} source-incremental-strategy]
- (if (and (lib.query/native? (:query source))
+ (if (and (lib/native? (:query source))
(not (get-in (:query source) [:stages 0 :template-tags "checkpoint"]))
(next-checkpoint-value checkpoint))
(let [wrap-query (fn [query]
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index ac7cf50f6e8..5353d2e1d78 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -6,7 +6,7 @@
[medley.core :as m]
[metabase.app-db.core :as mdb]
[metabase.lib-be.core :as lib-be]
- [metabase.lib.field :as lib.field]
+ [metabase.lib.core :as lib]
[metabase.lib.schema.metadata]
[metabase.models.humanization :as humanization]
[metabase.models.interface :as mi]
@@ -328,15 +328,15 @@
This does one important thing: if `:has_field_values` is already present and set to `:auto-list`, it is replaced by
`:list` -- presumably because the frontend doesn't need to know `:auto-list` even exists?
- See [[lib.field/infer-has-field-values]] for more info."
+ See [[lib/infer-has-field-values]] for more info."
[_model k field]
(when field
- (let [has-field-values (lib.field/infer-has-field-values (lib-be/instance->metadata field :metadata/column))]
+ (let [has-field-values (lib/infer-has-field-values (lib-be/instance->metadata field :metadata/column))]
(assoc field k has-field-values))))
(methodical/defmethod t2.hydrate/needs-hydration? [#_model :default #_k :has_field_values]
"Always (re-)hydrate `:has_field_values`. This is used to convert an existing value of `:auto-list` to
- `:list` (see [[infer-has-field-values]])."
+ `:list` (see [[lib/infer-has-field-values]])."
[_model _k _field]
true)