The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
refactor: reusable typed schema codegen module (#78223)
Phoomparin Manocac4d1e44e9a
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex f5cc9cde6e4..5be10772435 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2517,9 +2517,8 @@typed-schemas{:team "Embedding"- :api #{metabase.typed-schemas.api}+ :api #{metabase.typed-schemas.core}:uses #{actions- apicollectionsliblib-be@@ -2538,6 +2537,13 @@:model/Measure:model/Table}}+ typed-schemas-rest+ {:team "Embedding"+ :api #{metabase.typed-schemas-rest.api}+ :uses #{api+ typed-schemas+ util}}+types{:team "Querying Platform":api #{metabase.types.corediff --git a/skills/metabase-data-app-semantic-layer/SKILL.md b/skills/metabase-data-app-semantic-layer/SKILL.mdindex 95820dd1173..db6901d980a 100644--- a/skills/metabase-data-app-semantic-layer/SKILL.md+++ b/skills/metabase-data-app-semantic-layer/SKILL.md@@ -51,7 +51,7 @@ Before generating, make sure the user has explicitly chosen the library scope th- `include-models=true` for readable models that have actions. When combined with `database=<name-or-id>`, it includes models with actions for that database only.- `database=<name-or-id>` when the app should use tables from one database.-Use `question-collections` when the app needs `schema.questions.*` to be generated. Use `include-models=true` when the app needs any saved action under `schema.models.<model>.actions`; models without executable actions are omitted to keep generated schemas compact. It can be combined with `library-collections`, `include-data-library`, `include-metric-library`, or `question-collections` so one schema can include selected tables/metrics/questions plus all relevant actions.+Use `question-collections` when the app needs `schema.questions.*` to be generated. Use `include-models=true` when the app needs any saved action under `schema.models.<model>.actions`; it includes all readable models with executable actions, unless `database` scopes them to one database. Models without executable actions are omitted to keep generated schemas compact. It can be combined with `library-collections`, `include-data-library`, `include-metric-library`, or `question-collections` so one schema can include selected tables/metrics/questions plus all readable actions.If the user asks for any mutation-like flow, such as creating, updating, deleting, submitting, approving, executing an action, or running a write operation, include `include-models=true` in the typed-schema URL. Do this even when the user names one specific model/action, because actions are only discoverable through generated model entries.diff --git a/src/metabase/api_routes/routes.clj b/src/metabase/api_routes/routes.cljindex bf474669e94..1abcf93f661 100644--- a/src/metabase/api_routes/routes.clj+++ b/src/metabase/api_routes/routes.clj@@ -70,7 +70,7 @@[metabase.transforms-rest.api.transform][metabase.transforms-rest.api.transform-job][metabase.transforms-rest.api.transform-tag]- [metabase.typed-schemas.api]+ [metabase.typed-schemas-rest.api][metabase.upload.api][metabase.user-key-value.api][metabase.users-rest.api]@@ -126,7 +126,7 @@metabase.transforms-rest.api.transform/keep-memetabase.transforms-rest.api.transform-job/keep-memetabase.transforms-rest.api.transform-tag/keep-me- metabase.typed-schemas.api/keep-me+ metabase.typed-schemas-rest.api/keep-memetabase.upload.api/keep-memetabase.user-key-value.api/keep-memetabase.users-rest.api/keep-me@@ -246,7 +246,7 @@"/transform-dag-run" (+auth metabase.transforms-rest.api.transform/transform-dag-run-routes)"/transform-job" (+auth metabase.transforms-rest.api.transform/transform-job-routes)"/transform-tag" (+auth metabase.transforms-rest.api.transform/transform-tag-routes)- "/typed-schemas" (+auth metabase.typed-schemas.api/routes)+ "/typed-schemas" (+auth 'metabase.typed-schemas-rest.api)"/upload" (+auth 'metabase.upload.api)"/user" (+auth 'metabase.users-rest.api)"/user-key-value" (+auth 'metabase.user-key-value.api)diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.cljindex 6ddbade6ef2..700cee243d6 100644--- a/src/metabase/collections/models/collection.clj+++ b/src/metabase/collections/models/collection.clj@@ -1146,6 +1146,28 @@additional-honeysql-where-clauses)})[]))+(mu/defn descendants-flat-for :- [:sequential CollectionWithLocationAndIDOrRoot]+ "Like [[descendants-flat]], but returns the descendants of *any* of+ `collections` in a single query. Also selects `:type`, so callers can+ classify descendants without another lookup."+ [collections :- [:sequential CollectionWithLocationAndIDOrRoot]+ & additional-honeysql-where-clauses]+ (if (empty? collections)+ []+ (t2/select [:model/Collection :name :id :location :description :type]+ {:where (apply+ vector+ :and+ (into [:or]+ (map (fn [collection]+ [:like :location (str (children-location collection) "%")]))+ collections)+ ;; same personal-collection rule as descendants-flat+ [:or+ [:= :personal_owner_id nil]+ [:= :personal_owner_id api/*current-user-id*]]+ additional-honeysql-where-clauses)})))+(mu/defn descendants :- [:set Children]"Return all descendant Collections of a `collection`, including children, grandchildren, and so forth. This is doneprimarily to power the `effective-children` feature below, and thus the descendants are returned in a hierarchy,diff --git a/src/metabase/typed_schemas/api.clj b/src/metabase/typed_schemas/api.cljdeleted file mode 100644index 5d13bfb828a..00000000000--- a/src/metabase/typed_schemas/api.clj+++ /dev/null@@ -1,133 +0,0 @@-(ns metabase.typed-schemas.api- "/api/typed-schemas endpoints."- (:require- [clojure.set :as set]- [metabase.api.common :as api]- [metabase.api.macros :as api.macros]- [metabase.typed-schemas.api.render :as render]- [metabase.typed-schemas.api.schema :as schema]- [metabase.typed-schemas.api.schema.metric :as schema.metric]- [metabase.typed-schemas.api.schema.model :as schema.model]- [metabase.typed-schemas.api.schema.question :as schema.question]- [metabase.typed-schemas.api.schema.table :as schema.table]- [metabase.typed-schemas.api.scope :as scope]- [metabase.util.malli.schema :as ms]))--(set! *warn-on-reflection* true)--(def ^:private typescript-response-headers- {"Content-Type" "text/typescript; charset=utf-8"- "X-Content-Type-Options" "nosniff"- "Cross-Origin-Resource-Policy" "same-origin"- "Referrer-Policy" "no-referrer"- "Cache-Control" "no-store"})--(defn- validate-query-params!- [query-params]- (let [library-value (scope/query-library-value query-params)- library-collection-values (scope/query-library-collection-values query-params)- question-collection-values (scope/query-question-collection-values query-params)- include-library-root? (or (scope/query-include-data-library? query-params)- (scope/query-include-metric-library? query-params))- collection-scoped? (or library-value- library-collection-values- question-collection-values- include-library-root?)- database-value (scope/query-database-value query-params)- questions-only (scope/truthy-query-param? (:questions query-params))]- (api/check-400- (not (and library-value (or library-collection-values include-library-root?)))- "The library query parameter is mutually exclusive with library-collections, include-data-library, and include-metric-library.")- (api/check-400- (not (and collection-scoped? database-value))- "Collection-scoped query parameters and database query parameters are mutually exclusive.")- (api/check-400- (not (and collection-scoped? questions-only))- "Collection-scoped query parameters and the questions query parameter are mutually exclusive.")- (api/check-400- (not (and questions-only (nil? database-value)))- "The questions query parameter requires a database query parameter.")))--(defn- typed-schema-for-library-scope- [library-scope models]- (let [{:keys [metric-collection-ids]} library-scope- metrics (schema.metric/metric-schemas nil metric-collection-ids)- mapped-table-ids (->> metrics (mapcat :mappedTableIds) set)- library-table-ids (->> (schema.table/select-library-tables library-scope) (map :id) set)- table-ids (set/union library-table-ids mapped-table-ids)- tables (schema.table/table-schemas (schema.table/select-tables nil table-ids))]- (schema/base-schema [] models tables metrics)))--(defn- typed-schema- [query-params]- (validate-query-params! query-params)- (let [library-value (scope/query-library-value query-params)- library-collection-values (scope/query-library-collection-values query-params)- question-collection-values (scope/query-question-collection-values query-params)- library-scope (scope/library-scope query-params)- database-ids (scope/database-ids-for-value (scope/query-database-value query-params))- question-collection-ids (scope/collection-scope question-collection-values)- include-models? (scope/query-include-models? query-params)- models (cond- database-ids- (schema.model/model-schemas database-ids)-- include-models?- (schema.model/model-schemas nil)-- :else- [])- questions-only (scope/truthy-query-param? (:questions query-params))]- (cond- (or library-value- library-collection-values- library-scope- question-collection-values- (and include-models? (nil? database-ids)))- (let [questions (if question-collection-values- (schema.question/question-schemas nil question-collection-ids)- [])- library-schema (some-> library-scope- (typed-schema-for-library-scope models))]- (schema/base-schema questions- models- (-> library-schema :tables vals)- (-> library-schema :metrics vals)))-- questions-only- (schema/base-schema (schema.question/question-schemas database-ids) models [] [])-- :else- (let [questions (schema.question/question-schemas database-ids)- metrics (schema.metric/metric-schemas database-ids)- tables (schema.table/table-schemas (schema.table/select-tables database-ids))]- (schema/base-schema questions models tables metrics)))))--(def ^:private TypedSchemaQueryParams- [:map- [:database {:optional true} [:maybe ms/NonBlankString]]- [:database-name {:optional true} [:maybe ms/NonBlankString]]- [:library {:optional true} [:maybe ms/NonBlankString]]- [:library-collections {:optional true} [:maybe ms/NonBlankString]]- [:collections {:optional true} [:maybe ms/NonBlankString]]- [:question-collections {:optional true} [:maybe ms/NonBlankString]]- [:include-data-library {:optional true} [:maybe :boolean]]- [:include-metric-library {:optional true} [:maybe :boolean]]- [:include-models {:optional true} [:maybe :boolean]]- [:questions {:optional true} [:maybe :boolean]]])--(api.macros/defendpoint :get "/v1/typescript" :- :any- "Generate a TypeScript semantic schema module."- [_route-params- query-params :- TypedSchemaQueryParams- _body- _request- respond- _raise]- (respond {:status 200- :headers typescript-response-headers- :body (render/render-typescript (typed-schema query-params))}))--(def ^{:arglists '([request respond raise])} routes- "`/api/typed-schemas/` routes."- (api.macros/ns-handler *ns*))diff --git a/src/metabase/typed_schemas/api/javascript.clj b/src/metabase/typed_schemas/api/javascript.cljdeleted file mode 100644index 7e6c5435867..00000000000--- a/src/metabase/typed_schemas/api/javascript.clj+++ /dev/null@@ -1,172 +0,0 @@-(ns metabase.typed-schemas.api.javascript- "JavaScript syntax rendering helpers for typed schemas."- (:require- [clojure.string :as str]- [metabase.util :as u]- [metabase.util.json :as json]))--(def ^:private js-identifier-pattern- #"[A-Za-z_$][A-Za-z0-9_$]*")--(defn- spaces- [indent]- (apply str (repeat indent " ")))--(defn line-block- "Returns `lines` joined as a JavaScript block followed by a blank line."- [& lines]- (str (str/join "\n" lines) "\n\n"))--(defn reference- "Returns a marker for a raw JavaScript expression.-- [[render-value]] normally renders maps as object literals. Some renderer code- needs to splice an existing JavaScript expression into that object instead,- such as `tables.orders.fields` or `pickFields(...)`."- [path]- {:javascriptReference path})--(defn- entry-value- "Returns the requested entry without invoking map lookup.-- Some rendered schemas can contain sorted maps with string keys; direct keyword- lookup can make those maps compare keywords to strings and throw."- [entries-map entry-key-to-find]- (some (fn [[entry-key entry-value]]- (when (= entry-key entry-key-to-find)- entry-value))- entries-map))--(defn reference?- "Returns true when `value` is a raw JavaScript expression marker created by [[reference]]."- [value]- (and (map? value)- (string? (entry-value value :javascriptReference))))--(defn- javascript-key- "Renders a value as a JavaScript object key.-- Valid identifiers render bare e.g. orders; everything else renders as a quoted- property name e.g. \"orders-by-month\"."- [entry-key]- (let [key-name (u/qualified-name entry-key)]- (if (re-matches js-identifier-pattern key-name)- key-name- (json/encode key-name))))--(defn- javascript-property-access- "Renders property access for a JavaScript reference path segment."- [path-segment]- (let [segment-name (u/qualified-name path-segment)]- (if (re-matches js-identifier-pattern segment-name)- (str "." segment-name)- (str "[" (json/encode segment-name) "]"))))--(defn reference-path- "Returns a JavaScript property path for `path-segments`."- [& path-segments]- (str (u/qualified-name (first path-segments))- (apply str (map javascript-property-access (rest path-segments)))))--(defn- primitive-value?- [value]- (or (nil? value)- (string? value)- (number? value)- (true? value)- (false? value)))--(defn string-vector- "Renders `values` as a JavaScript string array literal."- [values]- (str "[ " (str/join ", " (map json/encode values)) " ]"))--(declare render-value*)--(defn- render-node- "Renders one nested value with its optional leading comments and object key.-- The four-argument arity is for vector items, which have comments but no key.- The five-argument arity is for map entries, which render `key: value`."- ([options indent path value]- (let [{:keys [item-comments]} options- comments (item-comments indent path value)- prefix (spaces indent)]- (str (when (seq comments)- (str (str/join "\n" comments) "\n"))- prefix- (render-value* value indent path options))))- ([options indent path entry-key value]- (let [{:keys [entry-comments]} options- comments (entry-comments indent path entry-key value)- prefix (spaces indent)]- (str (when (seq comments)- (str (str/join "\n" comments) "\n"))- prefix- (javascript-key entry-key)- ": "- (render-value* value indent (conj path entry-key) options)))))--(defn- render-entry- [options indent path entry-key value]- (render-node options indent path entry-key value))--(defn- render-map- "Renders a map as a JavaScript object literal.-- `:map-keys` controls both key ordering and which keys are emitted."- [{:keys [map-keys] :as options} value indent path]- (let [entries (->> (map-keys path value)- (map (fn [entry-key]- (render-entry options (+ indent 2) path entry-key (get value entry-key)))))]- (if (seq entries)- (str "{\n" (str/join ",\n" entries) "\n" (spaces indent) "}")- "{ }")))--(defn- render-vector- "Renders a vector as a compact array for primitive values, or multiline array otherwise."- [options value indent path]- (cond- (empty? value)- "[ ]"-- (every? primitive-value? value)- (str "[ " (str/join ", " (map json/encode value)) " ]")-- :else- (let [entries (map-indexed (fn [index item]- (render-node options (+ indent 2) (conj path index) item))- value)]- (str "[\n" (str/join ",\n" entries) "\n" (spaces indent) "]"))))--(defn- render-value*- "Recursive implementation for [[render-value]].-- `path` tracks where the value lives so callers can choose keys and comments- based on surrounding schema context."- [value indent path options]- (cond- (reference? value) (entry-value value :javascriptReference)- (map? value) (render-map options value indent path)- (vector? value) (render-vector options value indent path)- (seq? value) (render-vector options (vec value) indent path)- :else (json/encode value)))--(defn render-value- "Renders `value` into JavaScript syntax.-- Options:- - `:path` is the current object path.- - `:map-keys` returns the keys to render for a map at `[path value]`.- - `:entry-comments` returns comment lines before a map entry.- - `:item-comments` returns comment lines before a vector item."- ([value]- (render-value value {}))- ([value {:keys [path map-keys entry-comments item-comments]- :or {path []- map-keys (fn [_path value] (keys value))- entry-comments (constantly nil)- item-comments (constantly nil)}}]- (render-value* value 0 path {:map-keys map-keys- :entry-comments entry-comments- :item-comments item-comments})))diff --git a/src/metabase/typed_schemas/api/schema.clj b/src/metabase/typed_schemas/api/schema.cljdeleted file mode 100644index 1bd58780633..00000000000--- a/src/metabase/typed_schemas/api/schema.clj+++ /dev/null@@ -1,21 +0,0 @@-(ns metabase.typed-schemas.api.schema- "Top-level typed schema helpers."- (:require- [metabase.system.core :as system]- [metabase.typed-schemas.api.common :as common])- (:import- (java.time Instant)))--(set! *warn-on-reflection* true)--(defn base-schema- "Returns the top-level typed schema map."- [questions models tables metrics]- (array-map- :schemaVersion 2- :generatedAt (str (Instant/now))- :metabase {:instanceUrl (system/site-url)}- :questions (common/keyed-map questions)- :models (common/keyed-model-map models)- :tables (common/keyed-map tables)- :metrics (common/keyed-map metrics)))diff --git a/src/metabase/typed_schemas/api/scope.clj b/src/metabase/typed_schemas/api/scope.cljdeleted file mode 100644index be7814e8301..00000000000--- a/src/metabase/typed_schemas/api/scope.clj+++ /dev/null@@ -1,218 +0,0 @@-(ns metabase.typed-schemas.api.scope- "Scope resolution for typed-schema endpoints."- (:require- [clojure.string :as str]- [metabase.api.common :as api]- [metabase.collections.models.collection :as collection]- [metabase.models.interface :as mi]- [toucan2.core :as t2]))--(set! *warn-on-reflection* true)--(defn truthy-query-param?- "Returns true when a query param value should be interpreted as enabled."- [v]- (contains? #{true "true" "1"} v))--(def ^:private library-data-entity-id- "Entity id of the root Data library collection."- "librarylibrarydatadat")--(def ^:private library-metrics-entity-id- "Entity id of the root metrics library collection."- "librarylibrarymetrics")--(defn query-database-value- "Returns the requested database id/name query param."- [query-params]- (some-> (or (:database query-params)- (:database-name query-params))- str/trim- not-empty))--(defn database-ids-for-value- "Returns readable database ids matching `database-value`."- [database-value]- (when database-value- (let [database-id (parse-long database-value)]- (->> (if database-id- (t2/select :model/Database :id database-id)- (t2/select :model/Database :name database-value))- (filter mi/can-read?)- (map :id)- set))))--(defn database-id-filter-clause- "Returns a Honey SQL filter clause for optional database ids."- [database-ids column]- (when database-ids- (if (seq database-ids)- [:in column database-ids]- [:= column -1])))--(defn id-filter-clause- "Returns a Honey SQL filter clause for optional ids."- [ids column]- (when ids- (if (seq ids)- [:in column ids]- [:= column -1])))--(defn query-library-value- "Returns the `library` query param."- [query-params]- (some-> (:library query-params)- str/trim- not-empty))--(defn- query-comma-separated-values- [query-params param-keys]- (when-let [value (some-> (->> param-keys- (keep query-params)- first)- str/trim- not-empty)]- (->> (str/split value #",")- (map str/trim)- (remove str/blank?)- seq)))--(defn query-library-collection-values- "Returns requested library collection refs."- [query-params]- (query-comma-separated-values query-params [:library-collections- :collections]))--(defn query-include-data-library?- "Returns true when the root Data library should be included."- [query-params]- (truthy-query-param? (:include-data-library query-params)))--(defn query-include-metric-library?- "Returns true when the root metrics library should be included."- [query-params]- (truthy-query-param? (:include-metric-library query-params)))--(defn query-include-models?- "Returns true when readable model action namespaces should be included."- [query-params]- (truthy-query-param? (:include-models query-params)))--(defn query-question-collection-values- "Returns requested question collection refs."- [query-params]- (query-comma-separated-values query-params [:question-collections]))--(defn- library-collection-for-value- [library-value]- (when library-value- (let [collection-id (parse-long library-value)]- (->> (if collection-id- (t2/select :model/Collection :id collection-id)- (t2/select :model/Collection :name library-value))- (filter #(contains? collection/library-collection-types (:type %)))- (filter mi/can-read?)- first))))--(defn- library-collection-for-ref- [collection-value]- (let [collection-id (parse-long collection-value)]- (->> (if collection-id- (t2/select :model/Collection :id collection-id)- (t2/select :model/Collection :entity_id collection-value))- (filter #(contains? collection/library-collection-types (:type %)))- (filter mi/can-read?)- first)))--(defn- library-collection-for-entity-id- [entity-id]- (->> (t2/select :model/Collection :entity_id entity-id)- (filter #(contains? collection/library-collection-types (:type %)))- (filter mi/can-read?)- first))--(defn- collection-for-ref- [collection-value]- (let [collection-id (parse-long collection-value)]- (->> (if collection-id- (t2/select :model/Collection :id collection-id)- (t2/select :model/Collection :entity_id collection-value))- (filter mi/can-read?)- first)))--(defn collection-scope- "Returns ids for requested normal collections and descendants."- [collection-values]- (when (seq collection-values)- (let [collections (for [collection-value collection-values]- (or (collection-for-ref collection-value)- (api/check-404 false)))]- (->> collections- (mapcat #(cons % (collection/descendants-flat %)))- (map :id)- set))))--(defn- library-collection-scope*- [library-collections]- (let [ids (->> library-collections- (mapcat #(cons % (collection/descendants-flat %)))- (map :id)- set)- rows (t2/select [:model/Collection :id :type] :id [:in ids])- ids-for-type (fn [collection-type]- (->> rows- (filter #(= (:type %) collection-type))- (map :id)- set))]- {:library-collections library-collections- :collection-ids ids- :data-collection-ids (ids-for-type collection/library-data-collection-type)- :metric-collection-ids (ids-for-type collection/library-metrics-collection-type)}))--(defn library-collection-scope- "Returns the library scope for one named or numeric library collection."- [library-value]- (when library-value- (let [library (or (library-collection-for-value library-value)- (api/check-404 false))]- (library-collection-scope* [library]))))--(defn library-collections-scope- "Returns the library scope for requested library collection refs."- [collection-values]- (when (seq collection-values)- (let [collections (for [collection-value collection-values]- (or (library-collection-for-ref collection-value)- (api/check-404 false)))]- (library-collection-scope* collections))))--(defn- included-library-root-collections- [query-params]- (keep (fn [[include? entity-id]]- (when include?- (or (library-collection-for-entity-id entity-id)- (api/check-404 false))))- [[(query-include-data-library? query-params) library-data-entity-id]- [(query-include-metric-library? query-params) library-metrics-entity-id]]))--(defn library-scope- "Returns the requested library scope for data and metric library params."- [query-params]- (let [library-value (query-library-value query-params)- library-collection-values (query-library-collection-values query-params)- included-roots (included-library-root-collections query-params)- collection-scope (cond- library-value- (library-collection-scope library-value)-- library-collection-values- (library-collections-scope library-collection-values))]- (cond- (and collection-scope (seq included-roots))- (library-collection-scope* (concat (:library-collections collection-scope) included-roots))-- (seq included-roots)- (library-collection-scope* included-roots)-- :else- collection-scope)))diff --git a/src/metabase/typed_schemas/build.clj b/src/metabase/typed_schemas/build.cljnew file mode 100644index 00000000000..f6cfda0497f--- /dev/null+++ b/src/metabase/typed_schemas/build.clj@@ -0,0 +1,169 @@+(ns metabase.typed-schemas.build+ "Builds the semantic schema value: interprets [[SemanticSchemaOptions]] into+ SchemaSource reads and assembles the fetched items into the document.++ Internal to the typed-schemas module: the public surface is+ [[metabase.typed-schemas.core]], which composes [[fetch-items]] and+ [[create-schema]] into its public functions. See the core namespace+ docstring for the pipeline shape and the separation rules."+ (:require+ [clojure.set :as set]+ [metabase.system.core :as system]+ [metabase.typed-schemas.common :as common]+ [metabase.typed-schemas.source :as source]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms])+ (:import+ (java.time Instant)))++(set! *warn-on-reflection* true)++(def ^:private DatabaseRef+ "References a database by id or name."+ [:or+ [:map [:id :int]]+ ; remote sync representations expose database names instead of numeric database ids+ [:map [:name ms/NonBlankString]]])++(def ^:private CollectionRef+ "References a collection by numeric or entity id."+ [:or+ [:map [:id :int]]+ [:map [:entity-id ms/NonBlankString]]])++(def SemanticSchemaOptions+ "Schema generation options accepted by [[fetch-items]]."+ [:map {:closed true}+ [:database {:optional true}+ [:maybe {:description (str "Scopes the schema to a database. "+ "Accepts `{:id <database-id>}` or `{:name <database-name>}`.")}+ DatabaseRef]]+ [:library-collection-refs {:optional true}+ [:sequential {:description (str "Limits tables and metrics to library collections. "+ "Each reference accepts `{:id <collection-id>}` or `{:entity-id <collection-entity-id>}`.")}+ CollectionRef]]+ [:question-collection-refs {:optional true}+ [:sequential {:description (str "Includes saved questions from collections. "+ "Each reference accepts `{:id <collection-id>}` or `{:entity-id <collection-entity-id>}`.")}+ CollectionRef]]+ [:include-data-library? {:optional true}+ [:boolean {:description "Whether to include the root data library."}]]+ [:include-metric-library? {:optional true}+ [:boolean {:description "Whether to include the root metrics library."}]]+ [:include-models? {:optional true}+ [:boolean {:description (str "Whether to include readable models with actions when no "+ "database scope is given. A `:database` scope always includes "+ "that database's models, regardless of this option.")}]]])++(def Items+ "Fetched schema entities, ready for pure assembly by [[create-schema]].++ Each entry holds entity maps shaped by the `metabase.typed-schemas.schema.*`+ builders; `:key` on each entity seeds the generated object keys."+ [:map {:closed true}+ [:questions [:sequential :map]]+ [:models [:sequential :map]]+ [:tables [:sequential :map]]+ [:metrics [:sequential :map]]])++(defn- invalid-options!+ [message]+ (throw (ex-info message {:status-code 400})))++(defn- validate-options!+ [{:keys [database library-collection-refs question-collection-refs+ include-data-library? include-metric-library?] :as options}]+ (when-not (mr/validate SemanticSchemaOptions options)+ (invalid-options! "Invalid semantic schema options."))+ (let [include-library-root? (or include-data-library? include-metric-library?)+ collection-scoped? (or (seq library-collection-refs)+ (seq question-collection-refs)+ include-library-root?)]+ (when (and collection-scoped? database)+ (invalid-options!+ "Collection-scoped query parameters and database query parameters are mutually exclusive."))))++(defn- library-items+ "Returns tables and metrics for a library scope.++ Tables mapped by in-scope metrics are included even when they live outside+ the library, so rendered metric dimensions can reference their fields."+ [source library-scope]+ (let [metrics (source/metrics source nil (:metric-collection-ids library-scope))+ table-ids (set/union (into #{} (map :id) (source/library-tables source library-scope))+ (into #{} (mapcat :mappedTableIds) metrics))]+ {:tables (source/tables source nil table-ids)+ :metrics metrics}))++(defn- models-for-scope+ "Returns model schemas scoped to `database-ids`, or all readable models when requested without a database scope."+ [source database-ids include-models?]+ (cond+ database-ids (source/models source database-ids)+ include-models? (source/models source nil)+ :else []))++(defn fetch-items+ "Fetches the schema entities selected by [[SemanticSchemaOptions]].++ This is the only impure stage of the pipeline: everything downstream of the+ returned [[Items]] is pure. All reads go through a+ [[metabase.typed-schemas.source/SchemaSource]] — the application database by+ default, filtered to what the current user can read; tests reify the+ protocol with literal values."+ ([options]+ (fetch-items options source/app-db-source))+ ([options source]+ (let [{:keys [database library-collection-refs question-collection-refs+ include-data-library? include-metric-library? include-models?]+ :or {library-collection-refs []+ question-collection-refs []+ include-data-library? false+ include-metric-library? false+ include-models? false}} options]+ (validate-options! (assoc options+ :library-collection-refs library-collection-refs+ :question-collection-refs question-collection-refs+ :include-data-library? include-data-library?+ :include-metric-library? include-metric-library?+ :include-models? include-models?))+ (let [library-scope (source/library-scope source+ {:library-collection-refs library-collection-refs+ :include-data-library? include-data-library?+ :include-metric-library? include-metric-library?})+ database-ids (source/database-ids source database)+ question-collection-ids (source/collection-ids source question-collection-refs)+ models (models-for-scope source database-ids include-models?)]+ (if (or library-scope+ (seq question-collection-refs)+ (and include-models? (nil? database-ids)))+ (let [{:keys [tables metrics]} (when library-scope+ (library-items source library-scope))]+ {:questions (if (seq question-collection-refs)+ (source/questions source nil question-collection-ids)+ [])+ :models (vec models)+ :tables (vec tables)+ :metrics (vec metrics)})+ {:questions (source/questions source database-ids nil)+ :models (vec models)+ :tables (source/tables source database-ids nil)+ :metrics (source/metrics source database-ids nil)})))))++(defn create-schema+ "Assembles fetched [[Items]] into the semantic schema value rendered by+ [[render-typescript]].++ Pure when `info` provides `:generated-at` and `:instance-url`; they default+ to the current time and the configured site URL."+ ([items]+ (create-schema items nil))+ ([{:keys [questions models tables metrics]} {:keys [generated-at instance-url]}]+ (array-map+ :schemaVersion 2+ :generatedAt (str (or generated-at (Instant/now)))+ :metabase {:instanceUrl (or instance-url (system/site-url))}+ :questions (common/keyed-map questions)+ :models (common/keyed-model-map models)+ :tables (common/keyed-map tables)+ :metrics (common/keyed-map metrics))))diff --git a/src/metabase/typed_schemas/api/common.clj b/src/metabase/typed_schemas/common.cljsimilarity index 99%rename from src/metabase/typed_schemas/api/common.cljrename to src/metabase/typed_schemas/common.cljindex 1d43d59c006..5abc6aa2406 100644--- a/src/metabase/typed_schemas/api/common.clj+++ b/src/metabase/typed_schemas/common.clj@@ -1,4 +1,4 @@-(ns metabase.typed-schemas.api.common+(ns metabase.typed-schemas.common"Shared helpers for typed-schema generation."(:require[clojure.string :as str]diff --git a/src/metabase/typed_schemas/core.clj b/src/metabase/typed_schemas/core.cljnew file mode 100644index 00000000000..0b392f5e6af--- /dev/null+++ b/src/metabase/typed_schemas/core.clj@@ -0,0 +1,62 @@+(ns metabase.typed-schemas.core+ "Typed schemas: TypeScript modules that describe questions, models (with+ executable actions), tables and metrics to coding agents. Runtime objects+ feed the Lib.createTestQuery DSL; `//` comments carry context for humans and+ agents.++ The public surface is deliberately small — the schema value and its+ rendering, mirroring the module's data/print split:++ (-> options build-semantic-schema render-typescript) ; options -> TS string++ These public functions are compositions of the internal pipeline stages;+ their definitions below show the whole pipeline. Fetching filters by what+ the current user can read, so callers outside a request must bind a+ current-user context first.++ Keep the separation when extending this module:++ - New data reads go through [[metabase.typed-schemas.source/SchemaSource]] —+ a protocol method plus its `app-db-source` implementation — never direct+ `t2` calls from the pipeline stages.+ - Everything downstream of fetched items stays pure. Timestamps, site URL,+ and any other environment values enter through the `info` argument, not by+ calling out from assembly or rendering code.+ - Which keys render as runtime data vs comments is policy in+ `metabase.typed-schemas.render`; TypeScript syntax lives only in the+ `metabase.typed-schemas.javascript` printer.+ - REST query-parameter strings are decoded once, in+ `metabase.typed-schemas-rest.api.query-params`; this module only ever sees+ typed [[SemanticSchemaOptions]].+ - New exports here are compatibility promises, and they need not map 1:1 to+ internal functions — expose what consumers need, keep the stages internal."+ (:require+ [metabase.typed-schemas.build :as build]+ [metabase.typed-schemas.javascript :as javascript]+ [metabase.typed-schemas.render :as render]))++(set! *warn-on-reflection* true)++(def SemanticSchemaOptions+ "Schema generation options accepted by [[build-semantic-schema]]."+ build/SemanticSchemaOptions)++(defn build-semantic-schema+ "Builds a semantic schema value from [[SemanticSchemaOptions]].++ `info` optionally pins `:generated-at` and `:instance-url` for deterministic+ output; they default to the current time and the configured site URL."+ ([options]+ (build-semantic-schema options nil))+ ([options info]+ (-> options+ build/fetch-items ; all data access, returns items+ (build/create-schema info)))) ; pure assembly into the schema value++(defn render-typescript+ "Renders a semantic schema value as an ES module of `as const` TypeScript+ constants."+ [schema]+ (-> schema+ render/schema->ast ; pure render policy, returns a TypeScript AST+ javascript/render-js)) ; pure, option-free printerdiff --git a/src/metabase/typed_schemas/javascript.clj b/src/metabase/typed_schemas/javascript.cljnew file mode 100644index 00000000000..e69e5e2a6e8--- /dev/null+++ b/src/metabase/typed_schemas/javascript.clj@@ -0,0 +1,228 @@+(ns metabase.typed-schemas.javascript+ "A small TypeScript AST encoded as tagged vectors, and a printer for it.++ The AST covers exactly the subset of TypeScript that typed-schema modules+ emit. Expressions:++ [:lit 42] ; any JSON-encodable literal+ [:ref \"tables\" \"orders\" :fields] ; property path => tables.orders.fields+ [:call \"pickFields\" expr ...] ; call expression, args printed inline+ [:arr expr-or-item ...] ; array literal+ [:obj entry ...] ; object literal++ Object entries are `[key expr]` or `[key {:comments [...]} expr]`; array+ items are bare expressions or `[:item {:comments [...]} expr]`. Comments are+ plain strings; the printer prefixes them with `//` on their own lines above+ the entry. Statements:++ [:const \"tables\" expr] ; const tables = <expr> as const;+ [:raw \"function helper() {...}\"] ; verbatim TypeScript+ [:export-default expr] ; export default <expr>;+ [:module statement ...] ; one printed file++ For example:++ [:module+ [:const \"tables\"+ [:obj [\"orders\" {:comments [\"Entity ID: abc123\"]}+ [:obj [\"type\" [:lit \"table\"]]+ [\"ids\" [:arr [:lit 1] [:lit 2]]]]]]]+ [:export-default [:ref \"tables\"]]]++ prints as:++ const tables = {+ // Entity ID: abc123+ orders: {+ type: \"table\",+ ids: [ 1, 2 ]+ }+ } as const;++ export default tables;++ Rendering rules: arrays whose items are all literals print on one line;+ `:call` arguments always print inline; object keys print bare when they are+ valid JavaScript identifiers and quoted otherwise. [[render-js]] prints a+ `:module`; [[Module]] is the Malli schema for the whole grammar.++ The printer is deliberately option-free and policy-free: decisions about+ *what* to emit (runtime keys, comments, compaction) belong in+ `metabase.typed-schemas.render`, which builds this AST. If the output needs+ new syntax, add a node type and its printer here rather than concatenating+ TypeScript strings elsewhere."+ (:require+ [clojure.string :as str]+ [metabase.util :as u]+ [metabase.util.json :as json]))++(set! *warn-on-reflection* true)++(def ^:private expression-tags+ #{:lit :ref :call :arr :obj})++(def ^:private key-schema+ [:or :string :keyword])++(def Module+ "Malli schema for the TypeScript AST accepted by [[render-js]]."+ [:schema {:registry+ {::comment-options [:map {:closed true}+ [:comments {:optional true} [:sequential :string]]]+ ::entry [:or+ [:tuple key-schema [:ref ::expr]]+ [:tuple key-schema [:ref ::comment-options] [:ref ::expr]]]+ ::item [:tuple [:= :item] [:ref ::comment-options] [:ref ::expr]]+ ::expr [:multi {:dispatch first}+ [:lit [:tuple [:= :lit] :any]]+ [:ref [:cat [:= :ref] [:+ key-schema]]]+ [:call [:cat [:= :call] :string [:* [:schema [:ref ::expr]]]]]+ [:arr [:cat [:= :arr] [:* [:or+ [:schema [:ref ::item]]+ [:schema [:ref ::expr]]]]]]+ [:obj [:cat [:= :obj] [:* [:schema [:ref ::entry]]]]]]+ ::statement [:multi {:dispatch first}+ [:const [:tuple [:= :const] key-schema [:ref ::expr]]]+ [:raw [:tuple [:= :raw] :string]]+ [:export-default [:tuple [:= :export-default] [:ref ::expr]]]]}}+ [:cat [:= :module] [:* [:schema [:ref ::statement]]]]])++(defn expression?+ "Returns true when `value` is an AST expression node."+ [value]+ (and (vector? value)+ (contains? expression-tags (first value))))++(defn call?+ "Returns true when `value` is a `[:call ...]` expression node."+ [value]+ (and (vector? value)+ (= :call (first value))))++(def ^:private js-identifier-pattern+ #"[A-Za-z_$][A-Za-z0-9_$]*")++(defn- spaces+ [indent]+ (apply str (repeat indent " ")))++(defn- javascript-key+ "Renders a value as a JavaScript object key.++ Valid identifiers render bare e.g. orders; everything else renders as a quoted+ property name e.g. \"orders-by-month\"."+ [entry-key]+ (let [key-name (u/qualified-name entry-key)]+ (if (re-matches js-identifier-pattern key-name)+ key-name+ (json/encode key-name))))++(defn- reference-path+ "Renders `[:ref ...]` segments as a JavaScript property path."+ [path-segments]+ (str (u/qualified-name (first path-segments))+ (apply str (for [path-segment (rest path-segments)]+ (let [segment-name (u/qualified-name path-segment)]+ (if (re-matches js-identifier-pattern segment-name)+ (str "." segment-name)+ (str "[" (json/encode segment-name) "]")))))))++(defn- entry-parts+ "Returns `[key comment-options expr]` for either entry form."+ [entry]+ (if (= 3 (count entry))+ entry+ [(first entry) nil (second entry)]))++(defn- item-parts+ "Returns `[comment-options expr]` for either array item form."+ [item]+ (if (and (vector? item) (= :item (first item)))+ [(second item) (nth item 2)]+ [nil item]))++(defn- comment-block+ "Returns `// ...` comment lines followed by a newline, or nil without comments."+ [indent comments]+ (when (seq comments)+ (str (str/join "\n" (map #(str (spaces indent) "// " %) comments)) "\n")))++(declare render-expression)++(defn- render-inline+ "Renders an expression on a single line, for `:call` arguments."+ [node]+ (case (first node)+ :lit (json/encode (second node))+ :ref (reference-path (rest node))+ :call (let [[_ function-name & args] node]+ (str function-name "(" (str/join ", " (map render-inline args)) ")"))+ :arr (str "[ " (str/join ", " (map render-inline (rest node))) " ]")+ :obj (str "{ "+ (str/join ", " (for [entry (rest node)+ :let [[entry-key _ expr] (entry-parts entry)]]+ (str (javascript-key entry-key) ": " (render-inline expr))))+ " }")))++(defn- literal-node?+ [node]+ (and (vector? node) (= :lit (first node))))++(defn- render-object+ [entries indent]+ (if (empty? entries)+ "{ }"+ (str "{\n"+ (str/join ",\n"+ (for [entry entries+ :let [[entry-key options expr] (entry-parts entry)+ entry-indent (+ indent 2)]]+ (str (comment-block entry-indent (:comments options))+ (spaces entry-indent)+ (javascript-key entry-key)+ ": "+ (render-expression expr entry-indent))))+ "\n" (spaces indent) "}")))++(defn- render-array+ [items indent]+ (cond+ (empty? items)+ "[ ]"++ (every? literal-node? items)+ (str "[ " (str/join ", " (map #(json/encode (second %)) items)) " ]")++ :else+ (str "[\n"+ (str/join ",\n"+ (for [item items+ :let [[options expr] (item-parts item)+ item-indent (+ indent 2)]]+ (str (comment-block item-indent (:comments options))+ (spaces item-indent)+ (render-expression expr item-indent))))+ "\n" (spaces indent) "]")))++(defn- render-expression+ [node indent]+ (case (first node)+ :lit (json/encode (second node))+ :ref (reference-path (rest node))+ :call (render-inline node)+ :obj (render-object (rest node) indent)+ :arr (render-array (rest node) indent)))++(defn- render-statement+ [statement]+ (case (first statement)+ :raw (second statement)+ :const (let [[_ const-name expr] statement]+ (str "const " (u/qualified-name const-name) " = "+ (render-expression expr 0) " as const;"))+ :export-default (str "export default " (render-expression (second statement) 0) ";")))++(defn render-js+ "Renders a `[:module ...]` AST as TypeScript source."+ [[_ & statements]]+ (str (str/join "\n\n" (map render-statement statements)) "\n"))diff --git a/src/metabase/typed_schemas/api/render.clj b/src/metabase/typed_schemas/render.cljsimilarity index 56%rename from src/metabase/typed_schemas/api/render.cljrename to src/metabase/typed_schemas/render.cljindex 8745821c564..e59c83ebfbd 100644--- a/src/metabase/typed_schemas/api/render.clj+++ b/src/metabase/typed_schemas/render.clj@@ -1,12 +1,23 @@-(ns metabase.typed-schemas.api.render- "TypeScript rendering for typed schemas."+(ns metabase.typed-schemas.render+ "Converts semantic schema values into the TypeScript AST.++ This is the pure policy stage of the typed-schema pipeline: it decides which+ schema keys become runtime data, which become `//` context comments, and how+ metric dimensions compact into `pickFields(...)` calls. The output is the+ tagged-vector AST described in [[metabase.typed-schemas.javascript]];+ [[schema->ast]] produces it, and the public+ [[metabase.typed-schemas.core/render-typescript]] composes it with the+ printer.++ Nothing here touches the database or builds strings: new rendering behavior+ should be expressed as schema -> AST transformation (extend+ [[schema-render-policy]] or emit different nodes), keeping it assertable+ structurally in tests."(:require[clojure.string :as str]- [metabase.typed-schemas.api.javascript :as javascript]- [metabase.util :as u]))+ [metabase.typed-schemas.javascript :as javascript]))-(def ^:private const-suffix- " as const")+(set! *warn-on-reflection* true);; The rendered schema serves two audiences:;; - `:runtime` keys are executable data consumed by the Lib.createTestQuery DSL.@@ -34,6 +45,16 @@:column {:runtime [:type :name :jsType]:comment [:displayName :description :baseType :effectiveType :semanticType :unit]}})+;; Nested entity kinds by parent kind and entry key. Map values hold keyed+;; entity maps; sequential values hold entity vectors. Metric `:dimensions` are+;; special-cased in [[entity->node]] because compaction mixes call expressions+;; with nested dimension maps.+(def ^:private entity-children+ {:question {:columns :column}+ :table {:fields :field, :segments :segment, :measures :measure}+ :measure {:columns :column}+ :metric {:columns :column}})+(def ^:private comment-labels{:baseType "Base type":databaseName "Database"@@ -48,53 +69,11 @@:unit "Unit":verified "Verified"})-(defn- spaces- [indent]- (apply str (repeat indent " ")))--(defn- entry-value- "Returns the requested entry without invoking map lookup.-- Some rendered schemas can contain sorted maps with string keys; direct keyword- lookup can make those maps compare keywords to strings and throw."- [entries-map entry-key-to-find]- (some (fn [[entry-key entry-value]]- (when (= entry-key entry-key-to-find)- entry-value))- entries-map))--(defn- node-kind- "Returns the schema node kind at `path`.-- Most nodes identify themselves with `:type`, but fields/columns are also- inferred from their position so the renderer can apply the right runtime and- comment policy."- [path value]- (when (map? value)- (let [kind (or (entry-value value :type)- (entry-value value :kind))]- (cond- (= kind "card") :question- (= kind "question") :question- (= kind "table") :table- (= kind "segment") :segment- (= kind "measure") :measure- (= kind "metric") :metric- (= (last (butlast path)) :fields) :field- (and (= kind "column")- (or (= (last (butlast path)) :dimensions)- (= (last (drop-last 2 path)) :dimensions))) :metric-dimension- (= (last (butlast path)) :columns) :column- :else nil))))-(defn- policy-runtime-keys"Returns the keys that should be emitted as runtime data for a schema node."[kind value]- (let [runtime-keys (get-in schema-render-policy [kind :runtime])]- (if runtime-keys- (->> runtime-keys- (filter #(contains? value %)))- (keys value))))+ (filter #(contains? value %)+ (get-in schema-render-policy [kind :runtime])))(defn- redundant-table-name-comment?"Returns true when a table-name comment would duplicate the object key."@@ -122,45 +101,90 @@(str/join ".")):else (str value)))-(defn- comment-lines- "Returns JavaScript comment lines for context-only schema fields.+(defn- entity-comments+ "Returns comment strings for context-only schema fields of an entity.Fields listed under `:comment` in [[schema-render-policy]] are useful contextfor humans and coding agents, but are intentionally omitted from runtimeobjects consumed by the Lib.createTestQuery DSL."- [indent kind value entry-key]- (let [prefix (spaces indent)]- (for [comment-key (policy-comment-keys kind value entry-key)- :let [comment-text (comment-value (get value comment-key))]- :when (not (str/blank? comment-text))]- (str prefix- "// "- (get comment-labels comment-key (name comment-key))- ": "- (str/replace comment-text #"\R+" " ")))))--(defn- schema-render-options- "Returns callbacks that adapt typed-schema render policy to [[javascript/render-value]].-- The JavaScript renderer knows how to print Clojure data as JavaScript syntax.- These callbacks teach it which schema keys are runtime fields and which fields- should be emitted as nearby context comments."- []- {:map-keys (fn [path value]- (policy-runtime-keys (node-kind path value) value))- :entry-comments (fn [indent path entry-key value]- (comment-lines indent (node-kind (conj path entry-key) value) value entry-key))- :item-comments (fn [indent path value]- (comment-lines indent (node-kind path value) value nil))})+ [kind value entry-key]+ (seq (for [comment-key (policy-comment-keys kind value entry-key)+ :let [comment-text (comment-value (get value comment-key))]+ :when (not (str/blank? comment-text))]+ (str (get comment-labels comment-key (name comment-key))+ ": "+ (str/replace comment-text #"\R+" " ")))))++(defn- generic->node+ "Converts a schema value without render policy into an AST expression.++ Values outside the policy table (action namespaces, parameter maps, instance+ metadata) emit every key and no comments."+ [value]+ (cond+ (javascript/expression? value) value+ (map? value) (into [:obj] (for [[entry-key entry-value] value]+ [entry-key (generic->node entry-value)]))+ (sequential? value) (into [:arr] (map generic->node) value)+ :else [:lit value]))++(declare entity->node)++(defn- keyed-entities->obj+ "Converts a map of entity key -> entity of `kind` into an object expression."+ [entities kind]+ (into [:obj]+ (for [[entry-key entity] entities]+ (if-let [comments (entity-comments kind entity entry-key)]+ [entry-key {:comments comments} (entity->node entity kind)]+ [entry-key (entity->node entity kind)]))))++(defn- entities->arr+ "Converts a sequence of entities of `kind` into an array expression."+ [entities kind]+ (into [:arr]+ (for [entity entities]+ (if-let [comments (entity-comments kind entity nil)]+ [:item {:comments comments} (entity->node entity kind)]+ (entity->node entity kind)))))++(defn- dimensions->obj+ "Converts compacted metric dimensions into an object expression.++ Compaction leaves a mix of `pickFields(...)` call expressions and grouped raw+ dimension maps under `:dimensions`."+ [dimensions]+ (into [:obj]+ (for [[entry-key group] dimensions]+ [entry-key (if (javascript/expression? group)+ group+ (keyed-entities->obj group :metric-dimension))])))++(defn- entity->node+ "Converts one schema entity of `kind` into an object expression, emitting the+ runtime keys chosen by [[schema-render-policy]]."+ [entity kind]+ (into [:obj]+ (for [entry-key (policy-runtime-keys kind entity)+ :let [value (get entity entry-key)+ child-kind (get-in entity-children [kind entry-key])]]+ [entry-key+ (cond+ (and (= kind :metric)+ (= entry-key :dimensions)) (dimensions->obj value)+ (and child-kind (map? value)) (keyed-entities->obj value child-kind)+ (and child-kind+ (sequential? value)) (entities->arr value child-kind)+ :else (generic->node value))])))(defn- table-fields-reference-lookup- "Returns table-id -> JavaScript field-map reference for tables in `schema`."+ "Returns table-id -> field-map reference expression for tables in `schema`."[schema](into {}(for [[table-key {:keys [id]}] (:tables schema):when (integer? id)][id {:key table-key- :reference (javascript/reference-path :tables table-key :fields)}])))+ :reference [:ref "tables" table-key "fields"]}])))(defn- field-lookup-keys"Returns the table-scoped lookup keys that can identify a field."@@ -187,14 +211,15 @@(:tables schema)))(defn- pick-fields-call- "Returns a TypeScript `pickFields(...)` call for compacted metric dimensions."+ "Returns a `pickFields(...)` call expression for compacted metric dimensions."([fields-reference field-keys](pick-fields-call fields-reference field-keys nil))([fields-reference field-keys source-field-id]- (str "pickFields(" fields-reference ", " (javascript/string-vector field-keys)- (when (integer? source-field-id)- (str ", { sourceFieldId: " source-field-id " }"))- ")")))+ (cond-> [:call "pickFields"+ fields-reference+ (into [:arr] (map (fn [field-key] [:lit field-key])) field-keys)]+ (integer? source-field-id)+ (conj [:obj ["sourceFieldId" [:lit source-field-id]]]))))(defn- dimension-group-output-key"Returns the output key for a compacted dimension group.@@ -229,9 +254,9 @@:table-key table-key})))(defn- compact-metric-dimension-fields- "Compacts table-backed metric dimensions into grouped JavaScript references.+ "Compacts table-backed metric dimensions into grouped call expressions.- Returns both the rendered `:fields` map and the original dimension keys that+ Returns both the compacted `:fields` map and the original dimension keys thatwere compacted, so raw dimensions can be preserved without another lookup pass."[fields-reference-by-table field-key-by-table-and-field dimensions](let [{:keys [fields-by-group dimension-keys]}@@ -252,9 +277,9 @@(reduce-kv (fn [acc _ {:keys [table-key reference field-keys source-field-id]}](assoc acc(dimension-group-output-key table-key source-field-id table-key-count)- (javascript/reference (pick-fields-call reference- (distinct field-keys)- source-field-id))))+ (pick-fields-call reference+ (distinct field-keys)+ source-field-id)))(array-map)fields-by-group))}))@@ -296,57 +321,64 @@(cond-> (dissoc metric :dimensions)compact-dimensions (assoc :dimensions compact-dimensions)))))))))-(defn- render-top-level-const- "Renders one top-level schema section as a TypeScript `const ... as const`."- [section-key value]- (str "const " (u/qualified-name section-key) " = " (javascript/render-value value (assoc (schema-render-options) :path [section-key]))- const-suffix ";\n\n"))--(defn- render-pick-fields-helper- "Renders the TypeScript helper used by compacted metric dimensions."- []- (javascript/line-block- "function pickFields<TFields extends object, TKey extends keyof TFields>("- " fields: TFields,"- " keys: readonly TKey[],"- " options?: { sourceFieldId?: number },"- "): Pick<TFields, TKey> {"- " return Object.fromEntries(keys.map((key) => {"- " const field = fields[key] as { tableId?: number };"- " if (options?.sourceFieldId == null) {"- " return [key, field];"- " }"- " const { tableId, ...joinedField } = field;"- ""- " return [key, { ...joinedField, sourceFieldId: options.sourceFieldId }];"- " })) as Pick<TFields, TKey>;"- "}"))+(def ^:private pick-fields-helper+ "The TypeScript helper used by compacted metric dimensions."+ [:raw (str/join "\n"+ ["function pickFields<TFields extends object, TKey extends keyof TFields>("+ " fields: TFields,"+ " keys: readonly TKey[],"+ " options?: { sourceFieldId?: number },"+ "): Pick<TFields, TKey> {"+ " return Object.fromEntries(keys.map((key) => {"+ " const field = fields[key] as { tableId?: number };"+ " if (options?.sourceFieldId == null) {"+ " return [key, field];"+ " }"+ " const { tableId, ...joinedField } = field;"+ ""+ " return [key, { ...joinedField, sourceFieldId: options.sourceFieldId }];"+ " })) as Pick<TFields, TKey>;"+ "}"])])(defn- uses-pick-fields-helper?- "Returns true when rendered metrics contain `pickFields(...)` references."+ "Returns true when compacted metrics contain `pickFields(...)` expressions."[schema](boolean(some (fn [metric]- (some javascript/reference? (vals (:dimensions metric))))+ (some javascript/call? (vals (:dimensions metric))))(vals (:metrics schema)))))-(defn render-typescript- "Render `schema` as an ES module containing `as const` TypeScript constants."+(def ^:private top-level-keys+ [:questions :models :tables :metrics])++(defn- section->node+ "Converts one top-level schema section into an object expression."+ [section-key section]+ (case section-key+ :questions (keyed-entities->obj section :question)+ :models (generic->node section)+ :tables (keyed-entities->obj section :table)+ :metrics (keyed-entities->obj section :metric)))++(defn schema->ast+ "Converts a semantic schema value into a TypeScript module AST.++ Each present top-level section becomes its own `const`; the `schema` const+ ties them together with the remaining schema metadata and is exported as the+ module default."[schema](let [schema (compact-metric-dimensions schema)- top-level-keys [:questions :models :tables :metrics]+ present-keys (filter #(contains? schema %) top-level-keys)schema-metadata (apply dissoc schema top-level-keys)- schema-value (reduce (fn [schema-value section-key]- (if (contains? schema section-key)- (assoc schema-value section-key (javascript/reference (name section-key)))- schema-value))- schema-metadata- top-level-keys)]- (str (when (uses-pick-fields-helper? schema)- (render-pick-fields-helper))- (apply str- (for [section-key top-level-keys- :when (contains? schema section-key)]- (render-top-level-const section-key (get schema section-key))))- "const schema = " (javascript/render-value schema-value (schema-render-options)) const-suffix ";\n\n"- "export default schema;\n")))+ schema-entries (concat (for [[entry-key value] schema-metadata]+ [entry-key (generic->node value)])+ (for [section-key present-keys]+ [section-key [:ref (name section-key)]]))]+ (into [:module]+ (concat+ (when (uses-pick-fields-helper? schema)+ [pick-fields-helper])+ (for [section-key present-keys]+ [:const (name section-key) (section->node section-key (get schema section-key))])+ [[:const "schema" (into [:obj] schema-entries)]+ [:export-default [:ref "schema"]]]))))diff --git a/src/metabase/typed_schemas/api/schema/common.clj b/src/metabase/typed_schemas/schema/common.cljsimilarity index 53%rename from src/metabase/typed_schemas/api/schema/common.cljrename to src/metabase/typed_schemas/schema/common.cljindex 655c9aac8dc..7bcc1380481 100644--- a/src/metabase/typed_schemas/api/schema/common.clj+++ b/src/metabase/typed_schemas/schema/common.clj@@ -1,4 +1,4 @@-(ns metabase.typed-schemas.api.schema.common+(ns metabase.typed-schemas.schema.common"Shared typed-schema source helpers."(:require[medley.core :as m]@@ -7,28 +7,42 @@[metabase.lib.core :as lib][metabase.metabot.core :as metabot][metabase.models.interface :as mi]- [metabase.typed-schemas.api.scope :as scope][toucan2.core :as t2]))(set! *warn-on-reflection* true)+(defn scope-filter-clause+ "Compiles a resolved scope (see [[metabase.typed-schemas.scope]]) into a+ Honey SQL where-clause conjunct: a nil scope means unscoped (no clause), an+ empty scope matches nothing.++ This is the one place the nil-vs-empty semantics of scopes are enforced when+ querying. Do not replace call sites with `(seq ...)` guards — that would+ turn \"resolved to nothing\" into \"unscoped\" and select everything.+ Compiling the empty scope to an always-false clause (rather than skipping+ the query) keeps queries with several scope filters composable."+ [scope-ids column]+ (when scope-ids+ (if (seq scope-ids)+ [:in column scope-ids]+ ;; no row has id -1: a resolved-but-empty scope matches no rows+ [:= column -1])))+(defn select-schema-cards"Returns readable, non-archived cards for schema generation.Metrics, models and saved questions are backed by cards. They needthe same visibility, archived, database and collection filters."- ([card-type database-ids]- (select-schema-cards card-type database-ids nil))- ([card-type database-ids collection-ids]- (->> (t2/select :model/Card- {:where (cond-> [:and- [:= :type (name card-type)]- [:= :archived false]- (collection/visible-collection-filter-clause :collection_id)]- database-ids (conj (scope/database-id-filter-clause database-ids :database_id))- collection-ids (conj (scope/id-filter-clause collection-ids :collection_id)))- :order-by [[:name :asc] [:id :asc]]})- (filter mi/can-read?))))+ [card-type database-ids collection-ids]+ (->> (t2/select :model/Card+ {:where (cond-> [:and+ [:= :type (name card-type)]+ [:= :archived false]+ (collection/visible-collection-filter-clause :collection_id)]+ database-ids (conj (scope-filter-clause database-ids :database_id))+ collection-ids (conj (scope-filter-clause collection-ids :collection_id)))+ :order-by [[:name :asc] [:id :asc]]})+ (filter mi/can-read?)))(defn aggregation-result-column-with-metadata-provider"Returns an aggregation result column using an existing metadata provider."diff --git a/src/metabase/typed_schemas/api/schema/metric.clj b/src/metabase/typed_schemas/schema/metric.cljsimilarity index 95%rename from src/metabase/typed_schemas/api/schema/metric.cljrename to src/metabase/typed_schemas/schema/metric.cljindex d31a8a17ef7..ddc4b7c3335 100644--- a/src/metabase/typed_schemas/api/schema/metric.clj+++ b/src/metabase/typed_schemas/schema/metric.clj@@ -1,4 +1,4 @@-(ns metabase.typed-schemas.api.schema.metric+(ns metabase.typed-schemas.schema.metric"Typed schema generation for metrics and metric dimensions."(:require[medley.core :as m]@@ -6,9 +6,9 @@[metabase.metrics.core :as metrics][metabase.models.interface :as mi][metabase.permissions.core :as perms]- [metabase.typed-schemas.api.common :as common]- [metabase.typed-schemas.api.schema.common :as schema.common]- [metabase.typed-schemas.api.schema.table :as schema.table]+ [metabase.typed-schemas.common :as common]+ [metabase.typed-schemas.schema.common :as schema.common]+ [metabase.typed-schemas.schema.table :as schema.table][metabase.util :as u][toucan2.core :as t2]))@@ -273,10 +273,8 @@(defn metric-schemas"Returns metric schemas, with optional database and collection scopes."- ([database-ids]- (metric-schemas database-ids nil))- ([database-ids collection-ids]- (for [card (schema.common/select-schema-cards :metric database-ids collection-ids)- :let [details (metric-details card)]- :when details]- (metric-schema details card))))+ [database-ids collection-ids]+ (for [card (schema.common/select-schema-cards :metric database-ids collection-ids)+ :let [details (metric-details card)]+ :when details]+ (metric-schema details card)))diff --git a/src/metabase/typed_schemas/api/schema/model.clj b/src/metabase/typed_schemas/schema/model.cljsimilarity index 90%rename from src/metabase/typed_schemas/api/schema/model.cljrename to src/metabase/typed_schemas/schema/model.cljindex 0c7691e3583..1e850fc6af0 100644--- a/src/metabase/typed_schemas/api/schema/model.clj+++ b/src/metabase/typed_schemas/schema/model.clj@@ -1,12 +1,12 @@-(ns metabase.typed-schemas.api.schema.model+(ns metabase.typed-schemas.schema.model"Typed schema generation for models and actions."(:require[clojure.string :as str][medley.core :as m][metabase.actions.core :as actions][metabase.lib.schema.common :as lib.schema.common]- [metabase.typed-schemas.api.common :as common]- [metabase.typed-schemas.api.schema.common :as schema.common]+ [metabase.typed-schemas.common :as common]+ [metabase.typed-schemas.schema.common :as schema.common][metabase.util :as u][toucan2.core :as t2]))@@ -251,19 +251,17 @@(defn model-schemas"Returns model schemas, with optional database and collection scopes."- ([database-ids]- (model-schemas database-ids nil))- ([database-ids collection-ids]- (let [models (schema.common/select-schema-cards :model database-ids collection-ids)]- (if (seq models)- (let [model-ids (set (map :id models))- action-rows-by-model-id (group-by :model_id (action-rows model-ids))- action-details-by-model-id (group-by :model_id (resolved-action-details-for-models models))]- (for [model models- :let [action-schemas (model-action-schemas model- (get action-rows-by-model-id (:id model))- (get action-details-by-model-id (:id model)))- schema (model-schema model action-schemas)]- :when schema]- schema))- []))))+ [database-ids collection-ids]+ (let [models (schema.common/select-schema-cards :model database-ids collection-ids)]+ (if (seq models)+ (let [model-ids (set (map :id models))+ action-rows-by-model-id (group-by :model_id (action-rows model-ids))+ action-details-by-model-id (group-by :model_id (resolved-action-details-for-models models))]+ (for [model models+ :let [action-schemas (model-action-schemas model+ (get action-rows-by-model-id (:id model))+ (get action-details-by-model-id (:id model)))+ schema (model-schema model action-schemas)]+ :when schema]+ schema))+ [])))diff --git a/src/metabase/typed_schemas/api/schema/question.clj b/src/metabase/typed_schemas/schema/question.cljsimilarity index 88%rename from src/metabase/typed_schemas/api/schema/question.cljrename to src/metabase/typed_schemas/schema/question.cljindex 20064724cec..09a995da7d2 100644--- a/src/metabase/typed_schemas/api/schema/question.clj+++ b/src/metabase/typed_schemas/schema/question.clj@@ -1,10 +1,10 @@-(ns metabase.typed-schemas.api.schema.question+(ns metabase.typed-schemas.schema.question"Typed schema generation for saved questions."(:require[medley.core :as m][metabase.metabot.tools.entity-details :as entity-details]- [metabase.typed-schemas.api.common :as common]- [metabase.typed-schemas.api.schema.common :as schema.common]))+ [metabase.typed-schemas.common :as common]+ [metabase.typed-schemas.schema.common :as schema.common]))(set! *warn-on-reflection* true)@@ -66,8 +66,6 @@(defn question-schemas"Returns schemas for saved questions, with optional database and collection scopes."- ([database-ids]- (question-schemas database-ids nil))- ([database-ids collection-ids]- (let [cards (schema.common/select-schema-cards :question database-ids collection-ids)]- (mapv question-schema (question-details cards)))))+ [database-ids collection-ids]+ (let [cards (schema.common/select-schema-cards :question database-ids collection-ids)]+ (mapv question-schema (question-details cards))))diff --git a/src/metabase/typed_schemas/api/schema/table.clj b/src/metabase/typed_schemas/schema/table.cljsimilarity index 90%rename from src/metabase/typed_schemas/api/schema/table.cljrename to src/metabase/typed_schemas/schema/table.cljindex 20fac8221bb..8a501e98878 100644--- a/src/metabase/typed_schemas/api/schema/table.clj+++ b/src/metabase/typed_schemas/schema/table.clj@@ -1,4 +1,4 @@-(ns metabase.typed-schemas.api.schema.table+(ns metabase.typed-schemas.schema.table"Typed schema generation for tables, fields, segments and measures."(:require[medley.core :as m]@@ -6,9 +6,8 @@[metabase.metabot.core :as metabot][metabase.models.interface :as mi][metabase.permissions.core :as perms]- [metabase.typed-schemas.api.common :as common]- [metabase.typed-schemas.api.schema.common :as schema.common]- [metabase.typed-schemas.api.scope :as scope]+ [metabase.typed-schemas.common :as common]+ [metabase.typed-schemas.schema.common :as schema.common][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -50,15 +49,13 @@Library and database endpoint paths both need the same active/readable tablerules; only their id filters differ."- ([database-ids]- (select-tables database-ids nil))- ([database-ids table-ids]- (->> (t2/select :model/Table- {:where (cond-> [:and [:= :active true]]- database-ids (conj (scope/database-id-filter-clause database-ids :db_id))- table-ids (conj (scope/id-filter-clause table-ids :id)))- :order-by [[:name :asc] [:id :asc]]})- (filter-readable-tables))))+ [database-ids table-ids]+ (->> (t2/select :model/Table+ {:where (cond-> [:and [:= :active true]]+ database-ids (conj (schema.common/scope-filter-clause database-ids :db_id))+ table-ids (conj (schema.common/scope-filter-clause table-ids :id)))+ :order-by [[:name :asc] [:id :asc]]})+ (filter-readable-tables)))(defn select-library-tables"Returns published tables from the library based on the given scope."@@ -67,7 +64,7 @@{:where [:and[:= :active true][:= :is_published true]- (scope/id-filter-clause data-collection-ids :collection_id)]+ (schema.common/scope-filter-clause data-collection-ids :collection_id)]:order-by [[:name :asc] [:id :asc]]})(filter-readable-tables)))diff --git a/src/metabase/typed_schemas/scope.clj b/src/metabase/typed_schemas/scope.cljnew file mode 100644index 00000000000..54a4ef52115--- /dev/null+++ b/src/metabase/typed_schemas/scope.clj@@ -0,0 +1,146 @@+(ns metabase.typed-schemas.scope+ "Resolves portable references into scopes.++ Callers hold *references*: portable ways to name content that survive+ moving between instances — a database by id or name, a collection by id or+ entity id, or a well-known library root by its fixed entity id. Remote-sync+ representations and coding agents pass names and entity ids precisely+ because numeric ids don't travel.++ A *scope* is what references resolve to on this instance: the concrete,+ permission-filtered sets of row ids that bound what fetching selects for the+ current user. Resolution:++ - filters to what the current user can read;+ - expands collection references to include their descendants;+ - distinguishes nil (no reference given: unscoped) from the empty set+ (references resolved to nothing: match nothing);+ - throws a 404 naming the collection references that don't resolve —+ missing and unreadable are deliberately indistinguishable — while+ database references that don't resolve yield an empty scope, and+ therefore an empty schema rather than an error.++ Database and collection scopes are bare id sets; the library scope is a+ [[LibraryScope]] map classifying the in-scope library tree by collection+ type. Resolution is batched: query count is constant regardless of how many+ references are passed."+ (:require+ [clojure.string :as str]+ [metabase.collections.models.collection :as collection]+ [metabase.models.interface :as mi]+ [metabase.util.malli :as mu]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^:private library-data-entity-id+ "Entity id of the root data library collection."+ "librarylibrarydatadat")++(def ^:private library-metrics-entity-id+ "Entity id of the root metrics library collection."+ "librarylibrarymetrics")++(defn database-ids-for-ref+ "Returns readable database ids matching a typed database reference.++ An id matches at most one database. Database names are not unique, so a+ name reference can genuinely match several databases — currently their+ content is unioned into one scope."+ [database-ref]+ (when database-ref+ (let [{:keys [id name]} database-ref]+ (if id+ (let [database (t2/select-one :model/Database :id id)]+ (if (and database (mi/can-read? database))+ #{id}+ #{}))+ (->> (t2/select :model/Database :name name)+ (filter mi/can-read?)+ (map :id)+ set)))))++(defn- not-found!+ [collection-refs]+ (throw (ex-info (str "Collections not found: "+ (str/join ", " (map pr-str collection-refs)))+ {:status-code 404+ :collection-refs (vec collection-refs)})))++(defn- resolve-collection-refs!+ "Batch-resolves collection references into readable collection rows, in ref+ order.++ Throws a 404 naming every reference that does not resolve to a collection+ satisfying `usable-collection?` that the current user can read; missing and+ unreadable references are deliberately indistinguishable."+ [collection-refs usable-collection?]+ (let [ids (into #{} (keep :id) collection-refs)+ entity-ids (into #{} (keep :entity-id) collection-refs)+ usable? (fn [collection]+ (and collection+ (usable-collection? collection)+ (mi/can-read? collection)))+ by-id (when (seq ids)+ (into {} (map (juxt :id identity))+ (t2/select :model/Collection :id [:in ids])))+ ;; entity_id is a fixed-width char column; some app dbs return it+ ;; space-padded, so key the lookup by the trimmed value.+ by-eid (when (seq entity-ids)+ (into {} (map (juxt (comp str/trimr :entity_id) identity))+ (t2/select :model/Collection :entity_id [:in entity-ids])))+ resolved (for [{:keys [id entity-id] :as collection-ref} collection-refs]+ [collection-ref+ (let [collection (if id (get by-id id) (get by-eid entity-id))]+ (when (usable? collection)+ collection))])+ missing (into [] (comp (remove second) (map first)) resolved)]+ (when (seq missing)+ (not-found! missing))+ (mapv second resolved)))++(defn collection-scope+ "Returns ids for the referenced collections and their descendants."+ [collection-refs]+ (when (seq collection-refs)+ (let [collections (resolve-collection-refs! collection-refs (constantly true))]+ (into (into #{} (map :id) collections)+ (map :id)+ (collection/descendants-flat-for collections)))))++(def LibraryScope+ "A resolved library scope: the readable library collection tree, expanded to+ descendants and classified by collection type. `:data-collection-ids` bound+ published table selection; `:metric-collection-ids` bound metric selection."+ [:map {:closed true}+ [:data-collection-ids [:set :int]]+ [:metric-collection-ids [:set :int]]])++(defn- library-collection?+ [collection]+ (contains? collection/library-collection-types (:type collection)))++(defn- library-refs+ "Returns collection references for explicit library refs plus any included+ well-known library roots."+ [{:keys [library-collection-refs include-data-library? include-metric-library?]}]+ (concat library-collection-refs+ (when include-data-library?+ [{:entity-id library-data-entity-id}])+ (when include-metric-library?+ [{:entity-id library-metrics-entity-id}])))++(mu/defn library-scope :- [:maybe LibraryScope]+ "Resolves library collection refs and include flags into a [[LibraryScope]],+ or nil when no library scope is requested."+ [scope-options]+ (when-let [refs (seq (library-refs scope-options))]+ (let [roots (resolve-collection-refs! refs library-collection?)+ collections (concat roots (collection/descendants-flat-for roots))+ ids-of-type (fn [collection-type]+ (into #{}+ (comp (filter #(= (:type %) collection-type))+ (map :id))+ collections))]+ {:data-collection-ids (ids-of-type collection/library-data-collection-type)+ :metric-collection-ids (ids-of-type collection/library-metrics-collection-type)})))diff --git a/src/metabase/typed_schemas/source.clj b/src/metabase/typed_schemas/source.cljnew file mode 100644index 00000000000..fe1c313cee7--- /dev/null+++ b/src/metabase/typed_schemas/source.clj@@ -0,0 +1,78 @@+(ns metabase.typed-schemas.source+ "Data access for typed schemas, reified as a protocol.++ [[SchemaSource]] names every read that [[metabase.typed-schemas.core/fetch-items]]+ performs, so the module's data-access surface is one definition instead of+ selects scattered across namespaces. [[app-db-source]] is the production+ implementation, backed by the application database and filtered by what the+ current user can read.++ Tests reify the protocol with literal values instead of redefining selection+ functions:++ (reify source/SchemaSource+ (library-scope [_ _] {:metric-collection-ids #{20}})+ (metrics [_ _ _] [{:type \"metric\" :key \"revenue\" :id 1}])+ ...)++ All methods return the shaped schema entities produced by the+ `metabase.typed-schemas.schema.*` builders, except [[library-tables]], which+ returns raw table rows (only `:id` is consumed).++ When the pipeline needs to read something new, add a protocol method and its+ [[app-db-source]] implementation here — do not call `t2`/`metabot` directly+ from `metabase.typed-schemas.core` or anything downstream of it. That keeps+ the module's data-access surface enumerable and every downstream stage+ testable with literal values."+ (:require+ [metabase.typed-schemas.schema.metric :as schema.metric]+ [metabase.typed-schemas.schema.model :as schema.model]+ [metabase.typed-schemas.schema.question :as schema.question]+ [metabase.typed-schemas.schema.table :as schema.table]+ [metabase.typed-schemas.scope :as scope]))++(set! *warn-on-reflection* true)++(defprotocol SchemaSource+ "The data-access patterns behind typed schema generation.++ Scope-resolution methods return ids; entity methods return shaped schema+ entities. A nil `database-ids`/`collection-ids` argument means unscoped;+ an empty set matches nothing."+ (database-ids [source database-ref]+ "Readable database ids matching a database reference, or nil without one.")+ (collection-ids [source collection-refs]+ "Ids of the referenced collections and their descendants, or nil without refs.")+ (library-scope [source scope-options]+ "Resolved [[metabase.typed-schemas.scope/LibraryScope]] for library+ collection refs and include flags, or nil when none are requested.")+ (questions [source database-ids collection-ids]+ "Question schema entities.")+ (models [source database-ids]+ "Model schema entities that have executable actions.")+ (metrics [source database-ids collection-ids]+ "Metric schema entities.")+ (tables [source database-ids table-ids]+ "Table schema entities.")+ (library-tables [source library-scope]+ "Published table rows in the library scope's data collections."))++(def app-db-source+ "The production [[SchemaSource]], backed by the application database."+ (reify SchemaSource+ (database-ids [_ database-ref]+ (scope/database-ids-for-ref database-ref))+ (collection-ids [_ collection-refs]+ (scope/collection-scope collection-refs))+ (library-scope [_ scope-options]+ (scope/library-scope scope-options))+ (questions [_ database-ids collection-ids]+ (vec (schema.question/question-schemas database-ids collection-ids)))+ (models [_ database-ids]+ (vec (schema.model/model-schemas database-ids nil)))+ (metrics [_ database-ids collection-ids]+ (vec (schema.metric/metric-schemas database-ids collection-ids)))+ (tables [_ database-ids table-ids]+ (vec (schema.table/table-schemas (schema.table/select-tables database-ids table-ids))))+ (library-tables [_ library-scope]+ (vec (schema.table/select-library-tables library-scope)))))diff --git a/src/metabase/typed_schemas_rest/api.clj b/src/metabase/typed_schemas_rest/api.cljnew file mode 100644index 00000000000..4561464612a--- /dev/null+++ b/src/metabase/typed_schemas_rest/api.clj@@ -0,0 +1,55 @@+(ns metabase.typed-schemas-rest.api+ "/api/typed-schemas endpoints.++ This is the only place REST-shaped strings exist: query parameters decode to+ typed options in [[metabase.typed-schemas-rest.api.query-params]] at this+ boundary, and everything in `metabase.typed-schemas.*` works on typed data."+ (:require+ [metabase.api.macros :as api.macros]+ [metabase.typed-schemas-rest.api.query-params :as query-params]+ [metabase.typed-schemas.core :as typed-schemas]+ [metabase.util.malli.schema :as ms]))++(set! *warn-on-reflection* true)++(def ^:private typescript-response-headers+ {"Content-Type" "text/typescript; charset=utf-8"+ "X-Content-Type-Options" "nosniff"+ "Cross-Origin-Resource-Policy" "same-origin"+ "Referrer-Policy" "no-referrer"+ "Cache-Control" "no-store"})++(def ^:private TypedSchemaQueryParams+ [:map+ [:database {:optional true}+ [:maybe {:description "Scopes the schema to a database by numeric id or name."}+ ms/NonBlankString]]+ [:library-collections {:optional true}+ [:maybe {:description (str "Comma-separated library collection ids or entity ids. "+ "Limits tables and metrics to those library collections.")}+ ms/NonBlankString]]+ [:question-collections {:optional true}+ [:maybe {:description (str "Comma-separated collection ids or entity ids. "+ "Includes saved questions from those collections.")}+ ms/NonBlankString]]+ [:include-data-library {:optional true}+ [:maybe {:description "Whether to include the entire data library."}+ :boolean]]+ [:include-metric-library {:optional true}+ [:maybe {:description "Whether to include the entire metric library."}+ :boolean]]+ [:include-models {:optional true}+ [:maybe {:description (str "Whether to include all readable models with executable actions. "+ "Database scope applies when provided.")}+ :boolean]]])++(api.macros/defendpoint :get "/v1/typescript" :- :any+ "Generate a TypeScript semantic schema module."+ [_route-params+ query-params :- TypedSchemaQueryParams]+ {:status 200+ :headers typescript-response-headers+ :body (-> query-params+ query-params/query-params->options+ typed-schemas/build-semantic-schema+ typed-schemas/render-typescript)})diff --git a/src/metabase/typed_schemas_rest/api/query_params.clj b/src/metabase/typed_schemas_rest/api/query_params.cljnew file mode 100644index 00000000000..6e871a04328--- /dev/null+++ b/src/metabase/typed_schemas_rest/api/query_params.clj@@ -0,0 +1,46 @@+(ns metabase.typed-schemas-rest.api.query-params+ "REST query parameter parsing for typed schemas."+ (:require+ [clojure.string :as str]))++(set! *warn-on-reflection* true)++(defn- non-blank-string+ [value]+ (some-> value str str/trim not-empty))++(defn- database-reference+ [value]+ (when-let [value (non-blank-string value)]+ (if-let [id (parse-long value)]+ {:id id}+ {:name value})))++(defn- comma-separated-references+ [value]+ (if-let [value (non-blank-string value)]+ (->> (str/split value #",")+ (keep (fn [value]+ (when-let [value (non-blank-string value)]+ (if-let [id (parse-long value)]+ {:id id}+ {:entity-id value}))))+ vec)+ []))++(defn- enabled?+ [value]+ (contains? #{true "true" "1"} value))++(defn query-params->options+ "Converts REST query parameters into semantic schema options.++ This is the sole typed-schema function that accepts REST-shaped query+ parameters."+ [query-params]+ {:database (database-reference (:database query-params))+ :library-collection-refs (comma-separated-references (:library-collections query-params))+ :question-collection-refs (comma-separated-references (:question-collections query-params))+ :include-data-library? (enabled? (:include-data-library query-params))+ :include-metric-library? (enabled? (:include-metric-library query-params))+ :include-models? (enabled? (:include-models query-params))})diff --git a/test/metabase/typed_schemas/api_test.clj b/test/metabase/typed_schemas/api_test.cljdeleted file mode 100644index 54a7fb32c87..00000000000--- a/test/metabase/typed_schemas/api_test.clj+++ /dev/null@@ -1,344 +0,0 @@-(ns metabase.typed-schemas.api-test- (:require- [clojure.string :as str]- [clojure.test :refer :all]- [metabase.test :as mt]- [metabase.test.fixtures :as fixtures]- [metabase.typed-schemas.api :as typed-schemas.api]- [metabase.typed-schemas.api.render :as typed-schemas.api.render]- [metabase.typed-schemas.api.schema.metric :as typed-schemas.api.schema.metric]- [metabase.typed-schemas.api.schema.model :as typed-schemas.api.schema.model]- [metabase.typed-schemas.api.schema.question :as typed-schemas.api.schema.question]- [metabase.typed-schemas.api.schema.table :as typed-schemas.api.schema.table]- [metabase.typed-schemas.api.scope :as typed-schemas.api.scope]))--(use-fixtures :once (fixtures/initialize :db :web-server :test-users))--(deftest typescript-endpoint-test- (let [response (mt/user-http-request-full-response :crowberto :get 200 "typed-schemas/v1/typescript")]- (is (= "text/typescript; charset=utf-8" (get-in response [:headers "Content-Type"])))- (is (str/starts-with? (:body response) "const questions = "))- (is (str/includes? (:body response) "\nconst schema = {"))- (is (str/includes? (:body response) "\n schemaVersion: 2"))- (is (str/includes? (:body response) "\n questions: questions"))- (is (str/includes? (:body response) "\n tables: tables"))- (is (str/includes? (:body response) "\n metrics: metrics"))- (is (str/ends-with? (:body response) "export default schema;\n"))- (is (not (str/includes? (:body response) "\"schemaVersion\"")))- (is (not (str/includes? (:body response) "operators: [ ]")))- (is (not (str/includes? (:body response) "parameters: [ ]")))- (is (not (str/includes? (:body response) "verified: false")))))--(deftest query-params-are-coerced-at-endpoint-test- (with-redefs [typed-schemas.api/typed-schema identity- typed-schemas.api.render/render-typescript pr-str]- (let [response (-> (mt/user-http-request-full-response- :crowberto- :get- 200- "typed-schemas/v1/typescript?include-models=true&questions=false&library-collections=1,2")- :body- read-string)]- (is (true? (:include-models response)))- (is (false? (:questions response)))- (is (= "1,2" (:library-collections response))))))--(deftest database-filter-test- (testing "a non-matching database name returns an empty semantic schema"- (let [response (mt/user-http-request-full-response- :crowberto- :get- 200- "typed-schemas/v1/typescript?database=__missing_database__")]- (is (str/includes? (:body response) "schemaVersion: 2"))- (is (str/includes? (:body response) "const questions = { }"))- (is (str/includes? (:body response) "const tables = { }"))- (is (str/includes? (:body response) "const metrics = { }"))))- (testing "questions=true returns only questions for a numeric database id"- (let [response (mt/user-http-request-full-response- :crowberto- :get- 200- (format "typed-schemas/v1/typescript?database=%s&questions=true" (mt/id)))]- (is (str/includes? (:body response) "schemaVersion: 2"))- (is (str/includes? (:body response) "const questions = {"))- (is (str/includes? (:body response) "const tables = { }"))- (is (str/includes? (:body response) "const metrics = { }")))))--(deftest database-filter-scopes-models-test- (let [model-database-ids (atom [])]- (with-redefs [typed-schemas.api.scope/database-ids-for-value (constantly #{42})- typed-schemas.api.schema.model/model-schemas (fn [database-ids]- (swap! model-database-ids conj database-ids)- [])- typed-schemas.api.schema.question/question-schemas (fn- ([_database-ids] [])- ([_database-ids _collection-ids] []))- typed-schemas.api.schema.metric/metric-schemas (fn- ([_database-ids] [])- ([_database-ids _collection-ids] []))- typed-schemas.api.schema.table/select-tables (fn- ([_database-ids] [])- ([_database-ids _table-ids] []))- typed-schemas.api.schema.table/table-schemas (constantly [])]- (#'typed-schemas.api/typed-schema {:database "Boba"})- (#'typed-schemas.api/typed-schema {:database "Boba" :questions "true"})- (is (= [#{42} #{42}] @model-database-ids)))))--(deftest library-and-database-are-mutually-exclusive-test- (mt/user-http-request-full-response- :crowberto- :get- 400- "typed-schemas/v1/typescript?library=1&database=1")- (mt/user-http-request-full-response- :crowberto- :get- 400- "typed-schemas/v1/typescript?library=1&questions=true")- (mt/user-http-request-full-response- :crowberto- :get- 400- "typed-schemas/v1/typescript?questions=true"))--(deftest collection-query-params-are-mutually-exclusive-test- (mt/user-http-request-full-response- :crowberto- :get- 400- "typed-schemas/v1/typescript?library-collections=1,2&database=1")- (mt/user-http-request-full-response- :crowberto- :get- 400- "typed-schemas/v1/typescript?question-collections=1,2&questions=true")- (mt/user-http-request-full-response- :crowberto- :get- 400- "typed-schemas/v1/typescript?library=1&library-collections=2"))--(deftest library-schema-includes-metric-mapped-tables-test- (let [selected-table-ids (atom nil)]- (with-redefs [typed-schemas.api.scope/library-collection-scope- (constantly {:metric-collection-ids #{20}- :data-collection-ids #{10}})- typed-schemas.api.schema.model/model-schemas- (fn- ([_database-ids]- (is false "library-only schemas should not load models"))- ([_database-ids _collection-ids]- (is false "library-only schemas should not load models")))- typed-schemas.api.schema.metric/metric-schemas- (fn [_database-ids collection-ids]- (is (= #{20} collection-ids))- [{:type "metric"- :key "revenue"- :id 1- :name "Revenue"- :columns [{:name "Revenue", :displayName "Revenue", :jsType "number"}]- :mappedTableIds [42]}])- typed-schemas.api.schema.table/select-library-tables- (constantly [{:id 10}])- typed-schemas.api.schema.table/select-tables- (fn [_database-ids table-ids]- (reset! selected-table-ids table-ids)- [{:id 10} {:id 42}])- typed-schemas.api.schema.table/table-schemas- (constantly [{:type "table", :key "publishedTable", :id 10}- {:type "table", :key "mappedTable", :id 42}])]- (let [schema (#'typed-schemas.api/typed-schema {:library "123"})]- (is (= #{10 42} @selected-table-ids))- (is (= {} (:questions schema)))- (is (= {} (:models schema)))- (is (= #{10 42} (->> (:tables schema) vals (map :id) set)))- (is (= #{1} (->> (:metrics schema) vals (map :id) set)))))))--(deftest collections-schema-includes-selected-data-and-metric-collections-test- (let [selected-table-ids (atom nil)]- (with-redefs [typed-schemas.api.scope/library-collections-scope- (fn [collection-values]- (is (= ["10" "20"] collection-values))- {:metric-collection-ids #{20}- :data-collection-ids #{10}})- typed-schemas.api.schema.model/model-schemas- (fn- ([_database-ids]- (is false "library-only schemas should not load models"))- ([_database-ids _collection-ids]- (is false "library-only schemas should not load models")))- typed-schemas.api.schema.metric/metric-schemas- (fn [_database-ids collection-ids]- (is (= #{20} collection-ids))- [{:type "metric"- :key "revenue"- :id 1- :name "Revenue"- :columns [{:name "Revenue", :displayName "Revenue", :jsType "number"}]- :mappedTableIds [42]}])- typed-schemas.api.schema.table/select-library-tables- (fn [library-scope]- (is (= #{10} (:data-collection-ids library-scope)))- [{:id 10}])- typed-schemas.api.schema.table/select-tables- (fn [_database-ids table-ids]- (reset! selected-table-ids table-ids)- [{:id 10} {:id 42}])- typed-schemas.api.schema.table/table-schemas- (constantly [{:type "table", :key "publishedTable", :id 10}- {:type "table", :key "mappedTable", :id 42}])]- (let [schema (#'typed-schemas.api/typed-schema {:library-collections "10, 20"})]- (is (= #{10 42} @selected-table-ids))- (is (= {} (:questions schema)))- (is (= {} (:models schema)))- (is (= #{10 42} (->> (:tables schema) vals (map :id) set)))- (is (= #{1} (->> (:metrics schema) vals (map :id) set)))))))--(deftest include-models-schema-includes-actionable-models-test- (with-redefs [typed-schemas.api.schema.model/model-schemas- (fn [database-ids]- (is (nil? database-ids))- [{:key "actionableModel"- :actions {"create" {:kind "action", :key "create", :id 1}}}])- typed-schemas.api.schema.question/question-schemas- (fn- ([_database-ids]- (is false "include-models-only schemas should not load questions"))- ([_database-ids _collection-ids]- (is false "include-models-only schemas should not load questions")))- typed-schemas.api.schema.metric/metric-schemas- (fn- ([_database-ids]- (is false "include-models-only schemas should not load metrics"))- ([_database-ids _collection-ids]- (is false "include-models-only schemas should not load metrics")))- typed-schemas.api.schema.table/select-tables- (fn- ([_database-ids]- (is false "include-models-only schemas should not load tables"))- ([_database-ids _table-ids]- (is false "include-models-only schemas should not load tables")))]- (let [schema (#'typed-schemas.api/typed-schema {:include-models "true"})]- (is (= {} (:questions schema)))- (is (= {"actionableModel" {:actions {"create" {:kind "action", :key "create", :id 1}}}}- (:models schema)))- (is (= {} (:tables schema)))- (is (= {} (:metrics schema))))))--(deftest include-models-with-database-scopes-models-test- (let [model-database-ids (atom [])]- (with-redefs [typed-schemas.api.scope/database-ids-for-value (constantly #{42})- typed-schemas.api.schema.model/model-schemas (fn [database-ids]- (swap! model-database-ids conj database-ids)- [{:key "databaseModel"- :actions {"create" {:kind "action", :key "create", :id 1}}}])- typed-schemas.api.schema.question/question-schemas (fn- ([_database-ids] [])- ([_database-ids _collection-ids] []))- typed-schemas.api.schema.metric/metric-schemas (fn- ([_database-ids] [])- ([_database-ids _collection-ids] []))- typed-schemas.api.schema.table/select-tables (fn- ([_database-ids] [])- ([_database-ids _table-ids] []))- typed-schemas.api.schema.table/table-schemas (constantly [])]- (let [schema (#'typed-schemas.api/typed-schema {:database "Boba" :include-models "true"})]- (is (= [#{42}] @model-database-ids))- (is (= {"databaseModel" {:actions {"create" {:kind "action", :key "create", :id 1}}}}- (:models schema)))))))--(deftest question-collections-schema-includes-selected-question-collections-test- (with-redefs [typed-schemas.api.scope/collection-scope- (fn [collection-values]- (is (= ["30" "40"] collection-values))- #{30 40})- typed-schemas.api.schema.question/question-schemas- (fn [database-ids collection-ids]- (is (nil? database-ids))- (is (= #{30 40} collection-ids))- [{:type "card", :key "ordersByMonth", :id 1}])- typed-schemas.api.schema.model/model-schemas- (fn- ([_database-ids]- (is false "question collection schemas should not load models"))- ([_database-ids _collection-ids]- (is false "question collection schemas should not load models")))]- (let [schema (#'typed-schemas.api/typed-schema {:question-collections "30, 40"})]- (is (= #{1} (->> (:questions schema) vals (map :id) set)))- (is (= {} (:models schema)))- (is (= {} (:tables schema)))- (is (= {} (:metrics schema))))))--(deftest library-and-question-collections-can-be-combined-test- (with-redefs [typed-schemas.api.scope/library-collections-scope- (fn [collection-values]- (is (= ["10"] collection-values))- {:metric-collection-ids #{10}- :data-collection-ids #{10}})- typed-schemas.api.scope/collection-scope- (fn [collection-values]- (is (= ["30"] collection-values))- #{30})- typed-schemas.api.schema.question/question-schemas- (fn [database-ids collection-ids]- (is (nil? database-ids))- (is (= #{30} collection-ids))- [{:type "card", :key "ordersByMonth", :id 1}])- typed-schemas.api.schema.model/model-schemas- (fn- ([_database-ids]- (is false "question collection schemas should not load models"))- ([_database-ids _collection-ids]- (is false "question collection schemas should not load models")))- typed-schemas.api.schema.metric/metric-schemas- (constantly [{:type "metric", :key "revenue", :id 2}])- typed-schemas.api.schema.table/select-library-tables- (constantly [{:id 3}])- typed-schemas.api.schema.table/select-tables- (constantly [{:id 3}])- typed-schemas.api.schema.table/table-schemas- (constantly [{:type "table", :key "orders", :id 3}])]- (let [schema (#'typed-schemas.api/typed-schema {:library-collections "10"- :question-collections "30"})]- (is (= #{1} (->> (:questions schema) vals (map :id) set)))- (is (= {} (:models schema)))- (is (= #{3} (->> (:tables schema) vals (map :id) set)))- (is (= #{2} (->> (:metrics schema) vals (map :id) set))))))--(deftest library-and-question-collections-can-be-combined-with-include-models-test- (with-redefs [typed-schemas.api.scope/library-collections-scope- (fn [collection-values]- (is (= ["10"] collection-values))- {:metric-collection-ids #{10}- :data-collection-ids #{10}})- typed-schemas.api.scope/collection-scope- (fn [collection-values]- (is (= ["30"] collection-values))- #{30})- typed-schemas.api.schema.question/question-schemas- (fn [database-ids collection-ids]- (is (nil? database-ids))- (is (= #{30} collection-ids))- [{:type "card", :key "ordersByMonth", :id 1}])- typed-schemas.api.schema.model/model-schemas- (fn [database-ids]- (is (nil? database-ids))- [{:key "selectedQuestionCollectionModel"- :actions {"create" {:kind "action", :key "create", :id 1}}}])- typed-schemas.api.schema.metric/metric-schemas- (constantly [{:type "metric", :key "revenue", :id 2}])- typed-schemas.api.schema.table/select-library-tables- (constantly [{:id 3}])- typed-schemas.api.schema.table/select-tables- (constantly [{:id 3}])- typed-schemas.api.schema.table/table-schemas- (constantly [{:type "table", :key "orders", :id 3}])]- (let [schema (#'typed-schemas.api/typed-schema {:library-collections "10"- :question-collections "30"- :include-models "true"})]- (is (= #{1} (->> (:questions schema) vals (map :id) set)))- (is (= {"selectedQuestionCollectionModel" {:actions {"create" {:kind "action", :key "create", :id 1}}}}- (:models schema)))- (is (= #{3} (->> (:tables schema) vals (map :id) set)))- (is (= #{2} (->> (:metrics schema) vals (map :id) set))))))diff --git a/test/metabase/typed_schemas/api/common_test.clj b/test/metabase/typed_schemas/common_test.cljsimilarity index 78%rename from test/metabase/typed_schemas/api/common_test.cljrename to test/metabase/typed_schemas/common_test.cljindex b97ce286bee..2e3750a9abf 100644--- a/test/metabase/typed_schemas/api/common_test.clj+++ b/test/metabase/typed_schemas/common_test.clj@@ -1,7 +1,7 @@-(ns metabase.typed-schemas.api.common-test+(ns metabase.typed-schemas.common-test(:require[clojure.test :refer :all]- [metabase.typed-schemas.api.common :as typed-schemas.api.common]))+ [metabase.typed-schemas.common :as typed-schemas.common]))(deftest column-schema-includes-description-test(is (= {:type "column"@@ -10,14 +10,14 @@:baseType "type/Text":jsType "string":description "Name of the customer"}- (#'typed-schemas.api.common/column-schema {:name "name"- :display_name "Name"- :base_type "type/Text"- :description "Name of the customer"}))))+ (typed-schemas.common/column-schema {:name "name"+ :display_name "Name"+ :base_type "type/Text"+ :description "Name of the customer"}))))(deftest column-schema-maps-metabase-types-test(are [column js-type] (= js-type- (:jsType (#'typed-schemas.api.common/column-schema column)))+ (:jsType (typed-schemas.common/column-schema column))){:name "bool", :base_type :type/Boolean} "boolean"{:name "int", :base_type :type/Integer} "number"{:name "date", :base_type :type/DateTime} "Date"@@ -34,7 +34,7 @@"channelOrders" {:key "channelOrders":id "ca9bef16-d484-4add-8245-ddbc78287e8f":tableId 167}}- (#'typed-schemas.api.common/keyed-map+ (typed-schemas.common/keyed-map[{:key "channel":id "ca9bef16-d484-4add-8245-ddbc78287e8f":tableId 167@@ -51,7 +51,7 @@"channelOrders2" {:key "channelOrders2":id 2:tableId 168}}- (#'typed-schemas.api.common/keyed-map+ (typed-schemas.common/keyed-map[{:key "channel":id 1:tableId 167diff --git a/test/metabase/typed_schemas/core_test.clj b/test/metabase/typed_schemas/core_test.cljnew file mode 100644index 00000000000..69861e4825f--- /dev/null+++ b/test/metabase/typed_schemas/core_test.clj@@ -0,0 +1,154 @@+(ns metabase.typed-schemas.core-test+ "Typed schema tests form a pyramid; put new tests at the lowest level that+ can express them:++ - In-memory (no db): printer goldens in `javascript-test` (the output format+ spec), structural =? AST tests in `render-test`, pure shaping in+ `common-test` and the `schema.*` tests, assembly from literal [[Items]]+ and scoping rules against a reified literal source here.+ - Db seam (with-temp): `source-test` for `app-db-source`, `scope-test` for+ library-tree classification.+ - One end-to-end test here on `mt/dataset test-data`, through the whole+ pipeline to rendered TypeScript."+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [metabase.typed-schemas.build :as build]+ [metabase.typed-schemas.core :as typed-schemas]+ [metabase.typed-schemas.source :as source]))++(use-fixtures :once (fixtures/initialize :db :test-users))++(def ^:private test-info+ {:generated-at "2026-01-01T00:00:00Z"+ :instance-url "https://metabase.example.com"})++(deftest options-reject-unknown-keys-test+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Invalid semantic schema options\."+ (build/fetch-items {:unknown-option true}))))++(deftest options-reject-collection-and-database-scopes-together-test+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"mutually exclusive"+ (build/fetch-items {:database {:id 1}+ :include-data-library? true}))))++(deftest create-schema-assembles-items-test+ (is (= {:schemaVersion 2+ :generatedAt "2026-01-01T00:00:00Z"+ :metabase {:instanceUrl "https://metabase.example.com"}+ :questions {"ordersByMonth" {:type "card", :key "ordersByMonth", :id 1}}+ :models {"orders" {:actions {"create" {:kind "action", :id 9}}}}+ :tables {"orders" {:type "table", :key "orders", :id 3}}+ :metrics {"revenue" {:type "metric", :key "revenue", :id 2}}}+ (build/create-schema+ {:questions [{:type "card", :key "ordersByMonth", :id 1}]+ :models [{:key "orders", :name "Orders", :actions {"create" {:kind "action", :id 9}}}]+ :tables [{:type "table", :key "orders", :id 3}]+ :metrics [{:type "metric", :key "revenue", :id 2}]}+ test-info))))++(deftest create-schema-disambiguates-duplicate-keys-test+ (let [schema (build/create-schema+ {:questions []+ :models []+ :tables [{:type "table", :key "orders", :id 3}+ {:type "table", :key "orders", :id 4}]+ :metrics []}+ test-info)]+ (is (= ["orders3" "orders4"] (keys (:tables schema))))+ (is (= ["orders3" "orders4"] (map :key (vals (:tables schema)))))))++(deftest create-schema-defaults-info-test+ (let [schema (build/create-schema {:questions [], :models [], :tables [], :metrics []})]+ (is (string? (:generatedAt schema)))+ (is (contains? (:metabase schema) :instanceUrl))))++(defn- literal-source+ "A [[source/SchemaSource]] over literal values. `tables` are filtered by the+ requested table ids so tests can see which tables fetching asked for."+ [{:keys [database-ids collection-ids library-scope library-tables+ questions models metrics tables]}]+ (reify source/SchemaSource+ (database-ids [_ _] database-ids)+ (collection-ids [_ _] collection-ids)+ (library-scope [_ _] library-scope)+ (questions [_ _ _] (vec questions))+ (models [_ _] (vec models))+ (metrics [_ _ _] (vec metrics))+ (tables [_ _ table-ids] (cond->> (vec tables)+ table-ids (filterv #(contains? table-ids (:id %)))))+ (library-tables [_ _] (vec library-tables))))++;; fetch-items decides *what* to fetch; the source supplies literal rows, so+;; the test pins the scoping rules, not the db.+(deftest fetch-items-includes-tables-mapped-by-library-metrics-test+ (let [source (literal-source+ {:library-scope {:metric-collection-ids #{20}+ :data-collection-ids #{10}}+ :library-tables [{:id 10}]+ :metrics [{:type "metric", :key "revenue", :id 1, :mappedTableIds [42]}]+ :tables [{:id 10, :type "table", :key "publishedTable"}+ {:id 42, :type "table", :key "mappedTable"}+ {:id 99, :type "table", :key "notInScope"}]})]+ (is (= {:questions []+ :models []+ :tables [{:id 10, :type "table", :key "publishedTable"}+ {:id 42, :type "table", :key "mappedTable"}]+ :metrics [{:type "metric", :key "revenue", :id 1, :mappedTableIds [42]}]}+ (build/fetch-items {:include-data-library? true} source)))))++;; One end-to-end test over the real test-data dataset: cards for every entity+;; kind on real synced tables, run through the whole pipeline to TypeScript.+;; Everything above tests the stages with cheap literal data; this proves the+;; composition works against the real thing. With real fields the question and+;; metric cards need no result_metadata — columns are computed from metadata.+(deftest full-pipeline-end-to-end-test+ (mt/dataset test-data+ (mt/with-actions-enabled+ (let [mp (mt/metadata-provider)+ orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ revenue-query (lib/aggregate orders-query+ (lib/sum (lib.metadata/field mp (mt/id :orders :total))))]+ (mt/with-temp [:model/Card _question {:name "Order totals", :database_id (mt/id), :table_id (mt/id :orders)+ :type :question, :display :table+ :dataset_query orders-query}+ :model/Card _metric {:name "Order revenue", :database_id (mt/id), :table_id (mt/id :orders)+ :type :metric, :display :scalar+ :dataset_query revenue-query}+ :model/Card model {:name "Order model", :database_id (mt/id), :table_id (mt/id :orders)+ :type :model+ :dataset_query orders-query+ :result_metadata [{:name "total", :display_name "Total"+ :base_type :type/Float+ :field_ref [:field (mt/id :orders :total) nil]+ :id (mt/id :orders :total)}]}+ :model/Action action {:name "Update order", :model_id (:id model), :type :implicit}+ :model/ImplicitAction _ {:action_id (:id action), :kind "row/update"}]+ (mt/with-current-user (mt/user->id :crowberto)+ (let [schema (typed-schemas/build-semantic-schema {:database {:id (mt/id)}} test-info)+ body (typed-schemas/render-typescript schema)]+ (testing "every entity kind lands in the schema with its real relationships"+ (is (=? {:generatedAt "2026-01-01T00:00:00Z"+ :metabase {:instanceUrl "https://metabase.example.com"}+ :questions {"orderTotals" {:type "card", :id int?}}+ :tables {"orders" {:fields {"total" {:jsType "number"}}}}+ :metrics {"orderRevenue" {:mappedTableIds [(mt/id :orders)]+ :columns [{:displayName "Sum of Total"+ :jsType "number"}]}}+ :models {"orderModel" {:actions {"updateOrder" {:kind "action"}}}}}+ schema)))+ (testing "only the temp cards are in scope for the dataset database"+ (is (= {:questions ["orderTotals"], :metrics ["orderRevenue"], :models ["orderModel"]}+ (update-vals (select-keys schema [:questions :metrics :models])+ (comp vec keys)))))+ (testing "the rendered module carries the real entities"+ (is (str/includes? body "orders: {"))+ (is (str/includes? body "name: \"Order revenue\""))+ (is (str/includes? body "updateOrder: {"))+ (is (str/ends-with? body "export default schema;\n"))))))))))diff --git a/test/metabase/typed_schemas/javascript_test.clj b/test/metabase/typed_schemas/javascript_test.cljnew file mode 100644index 00000000000..7e96a67823d--- /dev/null+++ b/test/metabase/typed_schemas/javascript_test.clj@@ -0,0 +1,105 @@+(ns metabase.typed-schemas.javascript-test+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase.typed-schemas.javascript :as javascript]+ [metabase.util.malli.registry :as mr]))++(defn- render-lines+ "Renders a one-statement module and returns its lines, without the trailing+ newline, so goldens read naturally as vectors of lines."+ [statement]+ (str/split-lines (javascript/render-js [:module statement])))++(deftest literal-expressions-test+ (are [expr rendered] (= [(str "const x = " rendered " as const;")]+ (render-lines [:const "x" expr]))+ [:lit 42] "42"+ [:lit "hi"] "\"hi\""+ [:lit true] "true"+ [:lit nil] "null"+ [:ref "tables"] "tables"+ [:arr] "[ ]"+ [:obj] "{ }"))++(deftest reference-paths-quote-non-identifier-segments-test+ (are [segments rendered] (= [(str "const x = " rendered " as const;")]+ (render-lines (into [:const "x"] [(into [:ref] segments)])))+ ["tables" "orders" "fields"] "tables.orders.fields"+ ["tables" "50thPercentile"] "tables[\"50thPercentile\"]"+ [:tables :orders] "tables.orders"))++(deftest arrays-of-literals-render-inline-test+ (is (= ["const x = [ 1, 2, 3 ] as const;"]+ (render-lines [:const "x" [:arr [:lit 1] [:lit 2] [:lit 3]]]))))++(deftest arrays-of-objects-render-multiline-with-comments-test+ (is (= ["const x = ["+ " // Display name: Total"+ " {"+ " name: \"total\""+ " },"+ " {"+ " name: \"tax\""+ " }"+ "] as const;"]+ (render-lines+ [:const "x" [:arr+ [:item {:comments ["Display name: Total"]}+ [:obj ["name" [:lit "total"]]]]+ [:obj ["name" [:lit "tax"]]]]]))))++(deftest objects-render-comments-and-quote-non-identifier-keys-test+ (is (= ["const x = {"+ " // Entity ID: abc123"+ " // Description: All orders"+ " orders: {"+ " type: \"table\","+ " \"has-totals\": true"+ " }"+ "} as const;"]+ (render-lines+ [:const "x" [:obj+ ["orders" {:comments ["Entity ID: abc123"+ "Description: All orders"]}+ [:obj ["type" [:lit "table"]]+ ["has-totals" [:lit true]]]]]]))))++(deftest call-expressions-render-inline-test+ (is (= ["const x = {"+ " orders: pickFields(tables.orders.fields, [ \"total\" ], { sourceFieldId: 42 })"+ "} as const;"]+ (render-lines+ [:const "x" [:obj+ ["orders" [:call "pickFields"+ [:ref "tables" "orders" "fields"]+ [:arr [:lit "total"]]+ [:obj ["sourceFieldId" [:lit 42]]]]]]]))))++(deftest modules-join-statements-with-blank-lines-test+ (is (= (str "function helper() {}\n"+ "\n"+ "const schema = { } as const;\n"+ "\n"+ "export default schema;\n")+ (javascript/render-js+ [:module+ [:raw "function helper() {}"]+ [:const "schema" [:obj]]+ [:export-default [:ref "schema"]]]))))++(deftest module-schema-accepts-the-grammar-test+ (is (mr/validate javascript/Module+ [:module+ [:raw "function helper() {}"]+ [:const "tables"+ [:obj ["orders" {:comments ["Entity ID: abc"]}+ [:obj ["ids" [:arr [:lit 1] [:item {:comments ["c"]} [:obj]]]]+ ["fields" [:call "pickFields" [:ref "tables" "orders"]]]]]]]+ [:export-default [:ref "schema"]]])))++(deftest module-schema-rejects-unknown-nodes-test+ (are [module] (not (mr/validate javascript/Module module))+ [:module [:const "x" [:string "not-a-node"]]]+ [:module [:const "x" [:obj ["key" {:commentz ["typo"]} [:lit 1]]]]]+ [:const "x" [:lit 1]]))diff --git a/test/metabase/typed_schemas/api/render_test.clj b/test/metabase/typed_schemas/render_test.cljsimilarity index 63%rename from test/metabase/typed_schemas/api/render_test.cljrename to test/metabase/typed_schemas/render_test.cljindex 46f6cc97814..7c9772ad629 100644--- a/test/metabase/typed_schemas/api/render_test.clj+++ b/test/metabase/typed_schemas/render_test.clj@@ -1,8 +1,11 @@-(ns metabase.typed-schemas.api.render-test+(ns metabase.typed-schemas.render-test(:require[clojure.string :as str][clojure.test :refer :all]- [metabase.typed-schemas.api.render :as typed-schemas.api.render]))+ [metabase.typed-schemas.core :as typed-schemas]+ [metabase.typed-schemas.javascript :as javascript]+ [metabase.typed-schemas.render :as render]+ [metabase.util.malli.registry :as mr]))(def ^:private orders-question{:type "card"@@ -71,7 +74,7 @@:jsType "Date"}}}}})(deftest typescript-renderer-emits-comments-and-runtime-metadata-test- (let [body (typed-schemas.api.render/render-typescript compacting-schema)]+ (let [body (typed-schemas/render-typescript compacting-schema)];; Emit comments to provide context for agents(is (str/includes? body "// Description: Saved orders"))(is (str/includes? body "// Description: Total order revenue"))@@ -87,7 +90,7 @@(is (not (str/includes? body "displayName: \"Payment Method\"")))))(deftest typescript-renderer-compacts-metric-dimensions-test- (let [body (typed-schemas.api.render/render-typescript compacting-schema)]+ (let [body (typed-schemas/render-typescript compacting-schema)];; Metric dimensions should compact into pickFields(...) references.(is (str/includes? body "function pickFields"))(is (str/includes? body "const field = fields[key] as { tableId?: number };"))@@ -100,8 +103,61 @@;; The generated TypeScript module should not contain metric id.(is (not (str/includes? body "metricId: 5")))))+(defn- module-const+ "Returns the expression bound to `const-name` in a module AST."+ [ast const-name]+ (some (fn [statement]+ (when (and (= :const (first statement))+ (= const-name (second statement)))+ (nth statement 2)))+ (rest ast)))++(defn- obj-entry+ "Returns the expression stored under `entry-key` in an `[:obj ...]` node."+ [obj-node entry-key]+ (some (fn [entry]+ (when (= entry-key (first entry))+ (last entry)))+ (rest obj-node)))++(deftest schema->ast-produces-valid-modules-test+ (are [schema] (mr/validate javascript/Module (render/schema->ast schema))+ compacting-schema+ raw-dimensions-schema))++(deftest schema->ast-compacts-metric-dimensions-test+ (let [ast (render/schema->ast compacting-schema)+ dimensions (-> (module-const ast "metrics")+ (obj-entry "revenue")+ (obj-entry :dimensions))]+ (is (= [:call "pickFields"+ [:ref "tables" "orders" "fields"]+ [:arr [:lit "paymentMethod"]]]+ (obj-entry dimensions "orders")))+ (is (= [:call "pickFields"+ [:ref "tables" "franchises" "fields"]+ [:arr [:lit "name"]]+ [:obj ["sourceFieldId" [:lit 42]]]]+ (obj-entry dimensions "franchises")))))++(deftest schema->ast-splits-runtime-keys-from-comments-test+ (let [ast (render/schema->ast compacting-schema)+ fields (-> (module-const ast "tables")+ (obj-entry "orders")+ (obj-entry :fields))+ [entry-key options field-node] (-> fields rest first)]+ (testing "comment-only policy keys become entry comments"+ (is (= "paymentMethod" entry-key))+ (is (= {:comments ["Display name: Payment Method"+ "Semantic type: type/Category"]}+ options)))+ (testing "runtime policy keys become object entries"+ (is (= [:lit "payment_method"] (obj-entry field-node :name)))+ (is (= [:lit "string"] (obj-entry field-node :jsType)))+ (is (nil? (obj-entry field-node :displayName))))))+(deftest typescript-renderer-omits-pick-fields-helper-for-raw-dimensions-test- (let [body (typed-schemas.api.render/render-typescript raw-dimensions-schema)]+ (let [body (typed-schemas/render-typescript raw-dimensions-schema)];; Dimensions that cannot be resolved to table fields stay as raw fields, so;; the rendered module should not include the pickFields helper.(is (not (str/includes? body "function pickFields")))diff --git a/test/metabase/typed_schemas/api/schema/metric_test.clj b/test/metabase/typed_schemas/schema/metric_test.cljsimilarity index 75%rename from test/metabase/typed_schemas/api/schema/metric_test.cljrename to test/metabase/typed_schemas/schema/metric_test.cljindex 71e8bbb5c5a..382b511acf6 100644--- a/test/metabase/typed_schemas/api/schema/metric_test.clj+++ b/test/metabase/typed_schemas/schema/metric_test.clj@@ -1,11 +1,11 @@-(ns metabase.typed-schemas.api.schema.metric-test+(ns metabase.typed-schemas.schema.metric-test(:require[clojure.test :refer :all][metabase.metabot.core :as metabot][metabase.models.interface :as mi][metabase.test :as mt]- [metabase.typed-schemas.api.schema.metric :as schema.metric]- [metabase.typed-schemas.api.schema.table :as schema.table]+ [metabase.typed-schemas.schema.metric :as schema.metric]+ [metabase.typed-schemas.schema.table :as schema.table][toucan2.core :as t2]))(deftest ^:parallel metric-dimension-schema-uses-dimension-id-test@@ -51,30 +51,30 @@{:dataset_query {:stages [{:source-card 42}]}})))))(deftest metric-details-skips-default-temporal-breakout-test- (mt/with-dynamic-fn-redefs [metabot/get-metric-details- (fn [options]- (is (false? (:with-default-temporal-breakout? options)))- {:structured-output {:id 247}})]- (is (= {:id 247}- (#'schema.metric/metric-details {:id 247})))))+ (let [requested (atom nil)]+ (mt/with-dynamic-fn-redefs [metabot/get-metric-details+ (fn [options]+ (reset! requested options)+ {:structured-output {:id 247}})]+ (is (= {:id 247}+ (#'schema.metric/metric-details {:id 247})))+ (is (=? {:with-default-temporal-breakout? false}+ @requested)))))(deftest metric-details-surfaces-error-responses-test(mt/with-dynamic-fn-redefs [metabot/get-metric-details(constantly {:output "Not found.":status-code 404})]- (let [exception (try- (#'schema.metric/metric-details {:id 247- :name "Customer Lifetime Value"- :type :metric})- (catch clojure.lang.ExceptionInfo exception- exception))]- (is (instance? clojure.lang.ExceptionInfo exception))- (is (= {:card-id 247- :card-name "Customer Lifetime Value"- :card-type :metric- :status-code 404- :error-message "Not found."}- (ex-data exception))))))+ (let [exception (is (thrown? clojure.lang.ExceptionInfo+ (#'schema.metric/metric-details {:id 247+ :name "Customer Lifetime Value"+ :type :metric})))]+ (is (=? {:card-id 247+ :card-name "Customer Lifetime Value"+ :card-type :metric+ :status-code 404+ :error-message "Not found."}+ (ex-data exception))))))(deftest table-source-names-filters-unreadable-tables-test(with-redefs [t2/select (constantly [{:id 10 :name "orders" :display_name "Orders"}@@ -86,16 +86,16 @@(#'schema.metric/table-key-disambiguators [10 20])))))(deftest metric-schema-keys-dimensions-test- (with-redefs [schema.metric/metric-result-column (constantly nil)- schema.metric/readable-table-source-rows- (constantly [{:id 10 :name "orders" :display_name "Orders"}])- schema.metric/sync-and-fetch-metric-dimensions!- (constantly [{:id "550e8400-e29b-41d4-a716-446655440001"- :name "orders"- :display-name "Orders"- :effective-type :type/Integer- :table-id 10- :sources [{:type :field, :field-id 42}]}])]+ (mt/with-dynamic-fn-redefs [schema.metric/metric-result-column (constantly nil)+ schema.metric/readable-table-source-rows+ (constantly [{:id 10 :name "orders" :display_name "Orders"}])+ schema.metric/sync-and-fetch-metric-dimensions!+ (constantly [{:id "550e8400-e29b-41d4-a716-446655440001"+ :name "orders"+ :display-name "Orders"+ :effective-type :type/Integer+ :table-id 10+ :sources [{:type :field, :field-id 42}]}])](is (= {:type "metric":key "customerLifetimeValue":id 247@@ -161,13 +161,13 @@(is (empty? @field-lookup-attempts)))))(deftest source-card-metric-schema-omits-mapped-table-dimensions-test- (with-redefs [schema.metric/metric-result-column (constantly nil)- schema.metric/sync-and-fetch-metric-dimensions!- (constantly [{:id "count-dimension-uuid"- :name "count"}- {:id "store-name-dimension-uuid"- :name "store_name"- :table-id 10}])]+ (mt/with-dynamic-fn-redefs [schema.metric/metric-result-column (constantly nil)+ schema.metric/sync-and-fetch-metric-dimensions!+ (constantly [{:id "count-dimension-uuid"+ :name "count"}+ {:id "store-name-dimension-uuid"+ :name "store_name"+ :table-id 10}])](is (= {:type "metric":key "storesWithOver5Employees":id 259diff --git a/test/metabase/typed_schemas/api/schema/model_test.clj b/test/metabase/typed_schemas/schema/model_test.cljsimilarity index 70%rename from test/metabase/typed_schemas/api/schema/model_test.cljrename to test/metabase/typed_schemas/schema/model_test.cljindex 856003c9903..34efbbc9efb 100644--- a/test/metabase/typed_schemas/api/schema/model_test.clj+++ b/test/metabase/typed_schemas/schema/model_test.clj@@ -1,15 +1,16 @@-(ns metabase.typed-schemas.api.schema.model-test+(ns metabase.typed-schemas.schema.model-test(:require[clojure.test :refer :all][metabase.actions.core :as actions]- [metabase.typed-schemas.api.schema.common :as schema.common]- [metabase.typed-schemas.api.schema.model :as schema.model]))+ [metabase.test :as mt]+ [metabase.typed-schemas.schema.common :as schema.common]+ [metabase.typed-schemas.schema.model :as schema.model]))++(comment mt/keep-me)(deftest model-schema-includes-actions-test- (with-redefs [schema.model/model-action-schemas- (fn [model]- (is (= 42 (:id model)))- [{:kind "action", :key "create", :id 5}])]+ (mt/with-dynamic-fn-redefs [schema.model/model-action-schemas+ (constantly [{:kind "action", :key "create", :id 5}])](is (= {:key "ordersModel":keyDisambiguator 42:actions {"create" {:kind "action", :key "create", :id 5}}}@@ -17,14 +18,10 @@{:id 42:name "Orders model"})))))-(deftest model-schemas-includes-actionable-models-test+(deftest model-schemas-includes-only-actionable-models-test(with-redefs [schema.common/select-schema-cards- (fn [card-type database-ids collection-ids]- (is (= :model card-type))- (is (= #{1} database-ids))- (is (nil? collection-ids))- [{:id 42 :name "Model 42"}- {:id 43 :name "Model 43"}])+ (constantly [{:id 42 :name "Model 42"}+ {:id 43 :name "Model 43"}])schema.model/action-rows(constantly [{:id 5 :model_id 42 :name "Create" :type :query}])actions/select-actions@@ -33,10 +30,9 @@:name "Create":type :query:parameters []}])]- (is (= #{"model42"}- (->> (schema.model/model-schemas #{1})- (map :key)- set)))))+ ;; only model 42 has an action, so model 43 is omitted+ (is (= ["model42"]+ (map :key (schema.model/model-schemas #{1} nil)))))); Ensures we are not doing N+1 queries for action rows and details(deftest model-schemas-bulk-loads-actions-test@@ -51,7 +47,7 @@actions/select-actions (fn [known-models & options](swap! action-details-calls conj [known-models options])[])]- (is (= [] (vec (schema.model/model-schemas #{1}))))+ (is (= [] (vec (schema.model/model-schemas #{1} nil))))(is (= [#{42 43}] @action-rows-calls))(is (= [[models [:model_id [:in #{42 43}]:archived false@@ -68,11 +64,11 @@:name "Broken model"})))](is (= "Failed to build action schemas for model \"Broken model\" (card 100): action lookup failed"(ex-message exception)))- (is (= {:model-id 100- :model-name "Broken model"- :status-code 500- :cause-message "action lookup failed"}- (select-keys (ex-data exception) [:model-id :model-name :status-code :cause-message]))))))+ (is (=? {:model-id 100+ :model-name "Broken model"+ :status-code 500+ :cause-message "action lookup failed"}+ (ex-data exception))))))(deftest model-schema-surfaces-action-rendering-errors-test(with-redefs [actions/select-actions (constantly [{:id 200@@ -89,15 +85,14 @@:name "Broken model"})))](is (= "Failed to build action schema for action \"Broken action\" (action 200, type query) on model \"Broken model\" (card 100): action parameters are invalid"(ex-message exception)))- (is (= {:model-id 100- :model-name "Broken model"- :action-id 200- :action-name "Broken action"- :action-type :query- :status-code 500- :cause-message "action parameters are invalid"}- (select-keys (ex-data exception)- [:model-id :model-name :action-id :action-name :action-type :status-code :cause-message]))))))+ (is (=? {:model-id 100+ :model-name "Broken model"+ :action-id 200+ :action-name "Broken action"+ :action-type :query+ :status-code 500+ :cause-message "action parameters are invalid"}+ (ex-data exception))))))(deftest model-schema-surfaces-unresolved-action-row-errors-test(with-redefs [schema.model/action-rows (constantly [{:id 200@@ -109,7 +104,7 @@:name "Broken model"})))](is (= "Failed to build action schemas for model \"Broken model\" (card 100): action rows could not be resolved: Broken action (action 200, type broken)"(ex-message exception)))- (is (= {:model-id 100- :model-name "Broken model"- :unresolved-action-rows [{:id 200, :name "Broken action", :type :broken}]}- (select-keys (ex-data exception) [:model-id :model-name :unresolved-action-rows]))))))+ (is (=? {:model-id 100+ :model-name "Broken model"+ :unresolved-action-rows [{:id 200, :name "Broken action", :type :broken}]}+ (ex-data exception))))))diff --git a/test/metabase/typed_schemas/api/schema/question_test.clj b/test/metabase/typed_schemas/schema/question_test.cljsimilarity index 88%rename from test/metabase/typed_schemas/api/schema/question_test.cljrename to test/metabase/typed_schemas/schema/question_test.cljindex 6f6c744d046..c6d9b87b50f 100644--- a/test/metabase/typed_schemas/api/schema/question_test.clj+++ b/test/metabase/typed_schemas/schema/question_test.clj@@ -1,10 +1,10 @@-(ns metabase.typed-schemas.api.schema.question-test+(ns metabase.typed-schemas.schema.question-test(:require[clojure.test :refer :all][metabase.metabot.tools.entity-details :as entity-details][metabase.test :as mt]- [metabase.typed-schemas.api.schema.common :as schema.common]- [metabase.typed-schemas.api.schema.question :as schema.question]))+ [metabase.typed-schemas.schema.common :as schema.common]+ [metabase.typed-schemas.schema.question :as schema.question]))(deftest question-schema-uses-card-source-discriminator-test(is (= {:type "card"@@ -33,7 +33,7 @@(swap! bulk-details-calls conj [card-type database-id selected-cards])(map #(assoc % :fields []) selected-cards))](is (= [41 42]- (mapv :id (schema.question/question-schemas nil))))+ (mapv :id (schema.question/question-schemas nil nil))))(is (= [[:question 1 cards]]@bulk-details-calls))(is (empty? @report-details-calls))))))diff --git a/test/metabase/typed_schemas/api/schema/table_test.clj b/test/metabase/typed_schemas/schema/table_test.cljsimilarity index 84%rename from test/metabase/typed_schemas/api/schema/table_test.cljrename to test/metabase/typed_schemas/schema/table_test.cljindex 6c322d41527..3c2c9fb8e2a 100644--- a/test/metabase/typed_schemas/api/schema/table_test.clj+++ b/test/metabase/typed_schemas/schema/table_test.clj@@ -1,11 +1,11 @@-(ns metabase.typed-schemas.api.schema.table-test+(ns metabase.typed-schemas.schema.table-test(:require[clojure.test :refer :all][metabase.lib-be.core :as lib-be][metabase.metabot.core :as metabot][metabase.test :as mt]- [metabase.typed-schemas.api.schema.common :as schema.common]- [metabase.typed-schemas.api.schema.table :as schema.table]+ [metabase.typed-schemas.schema.common :as schema.common]+ [metabase.typed-schemas.schema.table :as schema.table][toucan2.core :as t2]))(def ^:private created-at-field@@ -91,10 +91,10 @@(deftest measure-schema-uses-result-column-test(testing "measure result columns come from the measure definition when available"- (with-redefs [schema.table/measure-result-column- (constantly {:name "sum"- :display_name "Sum of Total"- :base_type "type/Decimal"})]+ (mt/with-dynamic-fn-redefs [schema.table/measure-result-column+ (constantly {:name "sum"+ :display_name "Sum of Total"+ :base_type "type/Decimal"})](is (= (assoc total-revenue-schema:columns [{:type "column":name "sum"@@ -103,7 +103,7 @@:jsType "number"}])(schema.table/measure-schema 10 2 total-revenue-measure)))))(testing "measure result columns fall back to the measure name"- (with-redefs [schema.table/measure-result-column (constantly nil)]+ (mt/with-dynamic-fn-redefs [schema.table/measure-result-column (constantly nil)](is (= (assoc total-revenue-schema:columns [{:type "column", :name "Total Revenue", :displayName "Total Revenue", :jsType "unknown"}])(schema.table/measure-schema 10 2 total-revenue-measure))))))@@ -112,16 +112,13 @@(mt/with-dynamic-fn-redefs [metabot/get-table-details(constantly {:output "Not found.":status-code 404})]- (let [exception (try- (doall (schema.table/table-schemas [{:id 10 :name "Orders"}]))- (catch clojure.lang.ExceptionInfo exception- exception))]- (is (instance? clojure.lang.ExceptionInfo exception))- (is (= {:table-id 10- :table-name "Orders"- :status-code 404- :error-message "Not found."}- (ex-data exception))))))+ (let [exception (is (thrown? clojure.lang.ExceptionInfo+ (doall (schema.table/table-schemas [{:id 10 :name "Orders"}]))))]+ (is (=? {:table-id 10+ :table-name "Orders"+ :status-code 404+ :error-message "Not found."}+ (ex-data exception))))));; Batch measure definitions to avoid N+1 queries.(deftest table-schema-bulk-loads-measure-definitions-testdiff --git a/test/metabase/typed_schemas/api/scope_test.clj b/test/metabase/typed_schemas/scope_test.cljsimilarity index 54%rename from test/metabase/typed_schemas/api/scope_test.cljrename to test/metabase/typed_schemas/scope_test.cljindex e22e32d3fbb..434d8699799 100644--- a/test/metabase/typed_schemas/api/scope_test.clj+++ b/test/metabase/typed_schemas/scope_test.clj@@ -1,31 +1,14 @@-(ns metabase.typed-schemas.api.scope-test+(ns metabase.typed-schemas.scope-test(:require[clojure.test :refer :all][metabase.collections.models.collection :as collection][metabase.test :as mt][metabase.test.fixtures :as fixtures]- [metabase.typed-schemas.api.scope :as typed-schemas.api.scope]))+ [metabase.typed-schemas.scope :as typed-schemas.scope]))(use-fixtures :once (fixtures/initialize :db :test-users))-(deftest library-collection-scope-accepts-subcollection-id-test- (mt/with-temp [:model/Collection root {:name "Library"- :type "library"- :location "/"}- :model/Collection data {:name "Data"- :type "library-data"- :location (collection/children-location root)}- :model/Collection child {:name "Boba Data"- :type "library-data"- :location (collection/children-location data)}]- (mt/with-test-user :crowberto- (is (= {:collection-ids #{(:id child)}- :data-collection-ids #{(:id child)}- :metric-collection-ids #{}}- (select-keys (#'typed-schemas.api.scope/library-collection-scope (str (:id child)))- [:collection-ids :data-collection-ids :metric-collection-ids]))))))--(deftest library-collections-scope-accepts-comma-separated-subcollection-ids-test+(deftest library-scope-accepts-comma-separated-subcollection-ids-test(mt/with-temp [:model/Collection root {:name "Library":type "library":location "/"}@@ -45,17 +28,12 @@:type "library-metrics":location (collection/children-location metrics)}](mt/with-test-user :crowberto- (is (= {:collection-ids #{(:id data-child) (:id data-grandkid) (:id metric-child)}- :data-collection-ids #{(:id data-child) (:id data-grandkid)}- :metric-collection-ids #{(:id metric-child)}}- (select-keys (#'typed-schemas.api.scope/library-collections-scope- (#'typed-schemas.api.scope/query-library-collection-values- {:library-collections (format "%d, %d"- (:id data-child)- (:id metric-child))}))- [:collection-ids :data-collection-ids :metric-collection-ids]))))))+ (is (=? {:data-collection-ids #{(:id data-child) (:id data-grandkid)}+ :metric-collection-ids #{(:id metric-child)}}+ (typed-schemas.scope/library-scope+ {:library-collection-refs [{:id (:id data-child)} {:id (:id metric-child)}]}))))))-(deftest library-collections-scope-accepts-representation-entity-ids-test+(deftest library-scope-accepts-representation-entity-ids-test(mt/with-temp [:model/Collection root {:name "Library":type "library":location "/"}@@ -70,15 +48,14 @@:type "library-data":location (collection/children-location website)}](mt/with-test-user :crowberto- (is (= {:collection-ids #{(:id website) (:id website-page)}- :data-collection-ids #{(:id website) (:id website-page)}- :metric-collection-ids #{}}- (select-keys (#'typed-schemas.api.scope/library-collections-scope ["g-jLnamuHKdezZMthJ-z7"])- [:collection-ids :data-collection-ids :metric-collection-ids]))))))+ (is (=? {:data-collection-ids #{(:id website) (:id website-page)}+ :metric-collection-ids #{}}+ (typed-schemas.scope/library-scope+ {:library-collection-refs [{:entity-id "g-jLnamuHKdezZMthJ-z7"}]}))))))(deftest library-scope-includes-canonical-data-and-metrics-libraries-test- (with-redefs [typed-schemas.api.scope/library-data-entity-id "test-library-data"- typed-schemas.api.scope/library-metrics-entity-id "test-library-metrics"]+ (with-redefs [typed-schemas.scope/library-data-entity-id "test-library-data"+ typed-schemas.scope/library-metrics-entity-id "test-library-metrics"](mt/with-temp [:model/Collection root {:name "Library":type "library":location "/"}@@ -97,27 +74,33 @@:type "library-metrics":location (collection/children-location metrics)}](mt/with-test-user :crowberto- (is (= {:collection-ids #{(:id data) (:id data-child) (:id metrics) (:id metric-child)}- :data-collection-ids #{(:id data) (:id data-child)}- :metric-collection-ids #{(:id metrics) (:id metric-child)}}- (select-keys (#'typed-schemas.api.scope/library-scope- {:include-data-library "true"- :include-metric-library "true"})- [:collection-ids :data-collection-ids :metric-collection-ids])))))))+ (is (=? {:data-collection-ids #{(:id data) (:id data-child)}+ :metric-collection-ids #{(:id metrics) (:id metric-child)}}+ (typed-schemas.scope/library-scope+ {:include-data-library? true+ :include-metric-library? true})))))))-(deftest ^:parallel query-collection-values-use-kebab-case-params-test- (is (= ["1" "2"]- (#'typed-schemas.api.scope/query-library-collection-values {:library-collections "1, 2"})))- (is (= ["3" "4"]- (#'typed-schemas.api.scope/query-question-collection-values {:question-collections "3, 4"})))- (is (true?- (#'typed-schemas.api.scope/query-include-models? {:include-models "true"})))- (is (nil?- (#'typed-schemas.api.scope/query-library-collection-values {:libraryCollections "1, 2"})))- (is (nil?- (#'typed-schemas.api.scope/query-question-collection-values {:questionCollections "3, 4"})))- (is (false?- (#'typed-schemas.api.scope/query-include-models? {:includeModels "true"}))))+(deftest library-scope-combines-explicit-and-root-collections-test+ (with-redefs [typed-schemas.scope/library-metrics-entity-id "test-library-metrics"]+ (mt/with-temp [:model/Collection root {:name "Library"+ :type "library"+ :location "/"}+ :model/Collection data {:name "Data"+ :type "library-data"+ :location (collection/children-location root)}+ :model/Collection metrics {:name "Metrics"+ :type "library-metrics"+ :entity_id "test-library-metrics"+ :location (collection/children-location root)}+ :model/Collection metric-child {:name "Boba Metrics"+ :type "library-metrics"+ :location (collection/children-location metrics)}]+ (mt/with-test-user :crowberto+ (is (=? {:data-collection-ids #{(:id data)}+ :metric-collection-ids #{(:id metrics) (:id metric-child)}}+ (typed-schemas.scope/library-scope+ {:library-collection-refs [{:id (:id data)}]+ :include-metric-library? true})))))))(deftest question-collection-scope-accepts-comma-separated-collection-ids-test(mt/with-temp [:model/Collection parent {:name "Question Parent"@@ -126,9 +109,7 @@:location (collection/children-location parent)}](mt/with-test-user :crowberto(is (= #{(:id parent) (:id child)}- (#'typed-schemas.api.scope/collection-scope- (#'typed-schemas.api.scope/query-question-collection-values- {:question-collections (str (:id parent))})))))))+ (typed-schemas.scope/collection-scope [{:id (:id parent)}]))))))(deftest question-collection-scope-accepts-representation-entity-ids-test(mt/with-temp [:model/Collection parent {:name "Question Parent"@@ -138,10 +119,13 @@:location (collection/children-location parent)}](mt/with-test-user :crowberto(is (= #{(:id parent) (:id child)}- (#'typed-schemas.api.scope/collection-scope ["question-entity-id-1"]))))))+ (typed-schemas.scope/collection-scope [{:entity-id "question-entity-id-1"}]))))))(deftest question-collection-scope-rejects-missing-collection-ref-test(mt/with-test-user :crowberto- (let [e (is (thrown? clojure.lang.ExceptionInfo- (#'typed-schemas.api.scope/collection-scope ["missing-entity-id-1"])))]- (is (= 404 (:status-code (ex-data e)))))))+ (let [e (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Collections not found: \{:entity-id \"missing-entity-id-1\"\}"+ (typed-schemas.scope/collection-scope [{:entity-id "missing-entity-id-1"}])))]+ (is (=? {:status-code 404+ :collection-refs [{:entity-id "missing-entity-id-1"}]}+ (ex-data e))))))diff --git a/test/metabase/typed_schemas/source_test.clj b/test/metabase/typed_schemas/source_test.cljnew file mode 100644index 00000000000..b9a772f6073--- /dev/null+++ b/test/metabase/typed_schemas/source_test.clj@@ -0,0 +1,78 @@+(ns metabase.typed-schemas.source-test+ "Tests [[source/app-db-source]] against real application-database rows.++ Everything downstream of the source works on plain data and is covered by+ cheap in-memory tests; these tests are the seam where typed schemas meet the+ database."+ (:require+ [clojure.test :refer :all]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [metabase.typed-schemas.scope :as scope]+ [metabase.typed-schemas.source :as source]+ [metabase.util.malli.registry :as mr]))++(use-fixtures :once (fixtures/initialize :db :test-users))++(deftest database-ids-test+ (mt/with-temp [:model/Database db {:name "Typed Schema Source DB"}]+ (mt/with-current-user (mt/user->id :crowberto)+ (testing "resolves a database by id and by name"+ (is (= #{(:id db)} (source/database-ids source/app-db-source {:id (:id db)})))+ (is (= #{(:id db)} (source/database-ids source/app-db-source {:name "Typed Schema Source DB"}))))+ (testing "an unmatched reference resolves to an empty scope, not nil"+ (is (= #{} (source/database-ids source/app-db-source {:name "__no_such_database__"}))))+ (testing "no reference means unscoped"+ (is (nil? (source/database-ids source/app-db-source nil)))))))++(deftest collection-ids-include-descendants-test+ (mt/with-temp [:model/Collection parent {:name "Parent"}+ :model/Collection child {:name "Child", :location (format "/%d/" (:id parent))}+ :model/Collection grandchild {:name "Grandchild"+ :location (format "/%d/%d/" (:id parent) (:id child))}]+ (mt/with-current-user (mt/user->id :crowberto)+ (testing "resolves numeric and entity-id references with their descendants"+ (is (= #{(:id parent) (:id child) (:id grandchild)}+ (source/collection-ids source/app-db-source [{:id (:id parent)}])))+ (is (= #{(:id child) (:id grandchild)}+ (source/collection-ids source/app-db-source [{:entity-id (:entity_id child)}]))))+ (testing "no references means unscoped"+ (is (nil? (source/collection-ids source/app-db-source [])))))))++(deftest library-scope-and-library-tables-test+ (mt/with-temp [:model/Collection data-collection {:name "Data Library", :type "library-data"}+ :model/Collection plain-collection {:name "Not A Library"}+ :model/Database db {}+ :model/Table published {:db_id (:id db), :name "published_table", :active true+ :is_published true, :collection_id (:id data-collection)}+ :model/Table _unpublished {:db_id (:id db), :name "unpublished_table", :active true+ :is_published false, :collection_id (:id data-collection)}]+ (mt/with-current-user (mt/user->id :crowberto)+ (let [scope (source/library-scope source/app-db-source+ {:library-collection-refs [{:id (:id data-collection)}]})]+ (testing "library scope classifies collection ids by type"+ (is (mr/validate scope/LibraryScope scope))+ (is (= {:data-collection-ids #{(:id data-collection)}+ :metric-collection-ids #{}}+ scope)))+ (testing "library tables are the published tables in the data collections"+ (is (= [(:id published)]+ (map :id (source/library-tables source/app-db-source scope))))))+ (testing "a non-library collection reference is not found"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Collections not found"+ (source/library-scope source/app-db-source+ {:library-collection-refs [{:id (:id plain-collection)}]})))))))++(deftest tables-test+ (mt/with-temp [:model/Database db {}+ :model/Table table {:db_id (:id db), :name "widgets", :display_name "Widgets", :active true}+ :model/Field _ {:table_id (:id table), :name "price", :base_type :type/Float}]+ (mt/with-current-user (mt/user->id :crowberto)+ (testing "returns shaped table entities for a database scope"+ (is (=? [{:type "table"+ :key "widgets"+ :fields {"price" {:jsType "number"}}}]+ (source/tables source/app-db-source #{(:id db)} nil))))+ (testing "an empty table-id scope matches nothing"+ (is (= [] (source/tables source/app-db-source nil #{})))))))diff --git a/test/metabase/typed_schemas_rest/api/query_params_test.clj b/test/metabase/typed_schemas_rest/api/query_params_test.cljnew file mode 100644index 00000000000..a8157e2275a--- /dev/null+++ b/test/metabase/typed_schemas_rest/api/query_params_test.clj@@ -0,0 +1,30 @@+(ns metabase.typed-schemas-rest.api.query-params-test+ (:require+ [clojure.test :refer :all]+ [metabase.typed-schemas-rest.api.query-params :as query-params]))++(deftest ^:parallel query-params->options-test+ (is (= {:database {:name "Boba"}+ :library-collection-refs [{:id 10} {:id 20}]+ :question-collection-refs [{:entity-id "question-entity-id-1"}]+ :include-data-library? false+ :include-metric-library? false+ :include-models? true}+ (query-params/query-params->options+ {:database " Boba "+ :library-collections " 10, 20 "+ :question-collections " question-entity-id-1 "+ :include-models true}))))++(deftest ^:parallel query-params->options-coerces-values-and-applies-defaults-test+ (is (= {:database {:id 1}+ :library-collection-refs [{:id 3}]+ :question-collection-refs []+ :include-data-library? true+ :include-metric-library? true+ :include-models? false}+ (query-params/query-params->options+ {:database "1"+ :library-collections "3"+ :include-data-library "1"+ :include-metric-library "true"}))))diff --git a/test/metabase/typed_schemas_rest/api_test.clj b/test/metabase/typed_schemas_rest/api_test.cljnew file mode 100644index 00000000000..1f47d4a4045--- /dev/null+++ b/test/metabase/typed_schemas_rest/api_test.clj@@ -0,0 +1,72 @@+(ns metabase.typed-schemas-rest.api-test+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [metabase.typed-schemas.core :as typed-schemas]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db :web-server :test-users))++(defn- without-generated-at+ [typescript]+ (str/replace typescript #"generatedAt: \"[^\"]+\"" "generatedAt: \"<generated-at>\""))++(deftest typescript-endpoint-test+ (let [response (mt/user-http-request-full-response :crowberto :get 200 "typed-schemas/v1/typescript")]+ (is (= "text/typescript; charset=utf-8" (get-in response [:headers "Content-Type"])))+ ;; `includes?` rather than `starts-with?`: instances with compactable+ ;; metrics render the pickFields helper before the first const.+ (is (str/includes? (:body response) "const questions = "))+ (is (str/includes? (:body response) "\nconst schema = {"))+ (is (str/includes? (:body response) "\n schemaVersion: 2"))+ (is (str/includes? (:body response) "\n questions: questions"))+ (is (str/includes? (:body response) "\n tables: tables"))+ (is (str/includes? (:body response) "\n metrics: metrics"))+ (is (str/ends-with? (:body response) "export default schema;\n"))+ (is (not (str/includes? (:body response) "\"schemaVersion\"")))+ (is (not (str/includes? (:body response) "operators: [ ]")))+ (is (not (str/includes? (:body response) "parameters: [ ]")))+ (is (not (str/includes? (:body response) "verified: false")))))++(deftest query-params-are-decoded-at-endpoint-test+ (with-redefs [typed-schemas/build-semantic-schema identity+ typed-schemas/render-typescript pr-str]+ (let [response (-> (mt/user-http-request-full-response+ :crowberto+ :get+ 200+ "typed-schemas/v1/typescript?include-models=true&library-collections=1,2")+ :body+ read-string)]+ (is (true? (:include-models? response)))+ (is (= [{:id 1} {:id 2}] (:library-collection-refs response))))))++(deftest database-filter-test+ (let [database-id (mt/id)+ database-name (t2/select-one-fn :name :model/Database :id database-id)+ schema-by-id (:body (mt/user-http-request-full-response+ :crowberto :get 200 "typed-schemas/v1/typescript" :database database-id))+ schema-by-name (:body (mt/user-http-request-full-response+ :crowberto :get 200 "typed-schemas/v1/typescript" :database database-name))+ missing-schema (:body (mt/user-http-request-full-response+ :crowberto :get 200 "typed-schemas/v1/typescript" :database "__missing_database__"))]+ (testing "a database id and name select the same generated schema"+ (is (= (without-generated-at schema-by-id)+ (without-generated-at schema-by-name))))+ (testing "a matching database includes its real tables"+ (is (str/includes? schema-by-id " venues: {"))+ (is (str/includes? schema-by-id "name: \"Venues\"")))+ (testing "a non-matching database name returns an empty semantic schema"+ (is (str/includes? missing-schema "schemaVersion: 2"))+ (is (str/includes? missing-schema "const questions = { }"))+ (is (str/includes? missing-schema "const tables = { }"))+ (is (str/includes? missing-schema "const metrics = { }")))))++(deftest collection-and-database-query-params-are-mutually-exclusive-test+ (mt/user-http-request-full-response+ :crowberto+ :get+ 400+ "typed-schemas/v1/typescript?library-collections=1,2&database=1"))