The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Concise schema source (#72667)
dpsutton74f715fa7280
diff --git a/enterprise/backend/src/metabase_enterprise/checker/cli.clj b/enterprise/backend/src/metabase_enterprise/checker/cli.cljindex e1fc45d62e5..eb70a204dd1 100644--- a/enterprise/backend/src/metabase_enterprise/checker/cli.clj+++ b/enterprise/backend/src/metabase_enterprise/checker/cli.clj@@ -36,14 +36,38 @@(not (.isDirectory f))(fail! (str "Not a directory: " path)))))+(defn- validate-schema-path!+ "Validate that --schema-dir matches the expected shape for `fmt`:+ :serdes → must be a directory (containing database subdirectories)+ :concise → must be a regular file (the JSON metadata file)"+ [path fmt]+ (let [f (io/file path)]+ (cond+ (not (.exists f))+ (fail! (str "--schema-dir does not exist: " path))++ (and (= fmt :serdes) (not (.isDirectory f)))+ (fail! (str "--schema-format serdes expects a directory of database folders, got a file: " path)+ "Did you mean --schema-format concise?")++ (and (= fmt :concise) (not (.isFile f)))+ (fail! (str "--schema-format concise expects a single JSON file, got a directory: " path)+ "Did you mean --schema-format serdes?"))))+(defn- run-checker"Run the semantic checker."- [export-dir {:keys [output errors-only schema-dir]}]+ [export-dir {:keys [output errors-only schema-dir schema-format]}](when-not schema-dir(fail! "Missing --schema-dir option"))- (validate-directory! schema-dir)- (let [{:keys [results]} (try- (checker/check export-dir schema-dir)+ (let [fmt (case schema-format+ "concise" :concise+ "serdes" :serdes+ nil :serdes+ (fail! (str "Unknown --schema-format: " schema-format+ " (must be 'serdes' or 'concise')")))+ _ (validate-schema-path! schema-dir fmt)+ {:keys [results]} (try+ (checker/check export-dir schema-dir nil {:schema-format fmt})(catch clojure.lang.ExceptionInfo e(fail! (.getMessage e))))summary (checker/summarize-results results)@@ -88,7 +112,8 @@"CLI options for checker mode."[[nil "--mode MODE" "Mode (handled by bootstrap, included here for completeness)"][nil "--export PATH" "Path to serdes export directory"]- [nil "--schema-dir PATH" "Path to database schema directory"]+ [nil "--schema-dir PATH" "Path to schema source (directory for serdes, JSON file for concise)"]+ [nil "--schema-format FMT" "Schema format: 'serdes' (default) or 'concise'"][nil "--output PATH" "Path to output file for results"][nil "--errors-only" "Output only errors to stdout (concise format for LLM consumption)"]["-h" "--help" "Show this help"]])diff --git a/enterprise/backend/src/metabase_enterprise/checker/format/concise_schema.clj b/enterprise/backend/src/metabase_enterprise/checker/format/concise_schema.cljnew file mode 100644index 00000000000..b553e8b9ec1--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/checker/format/concise_schema.clj@@ -0,0 +1,118 @@+(ns metabase-enterprise.checker.format.concise-schema+ "Concise JSON schema format — single file with three flat lists, fully+ in-memory.++ The file is a single JSON document with three flat lists keyed by integer+ IDs. Tables reference their database via `:db_id`; fields reference their+ table via `:table_id`:++ {\"databases\": [{\"id\": 1, \"name\": \"Sample Database\", \"engine\": \"h2\"}, ...],+ \"tables\": [{\"id\": 3, \"db_id\": 1, \"name\": \"PRODUCTS\", \"schema\": \"PUBLIC\"}, ...],+ \"fields\": [{\"id\": 12, \"table_id\": 3, \"name\": \"ID\",+ \"base_type\": \"type/BigInteger\",+ \"semantic_type\": \"type/PK\",+ \"database_type\": \"BIGINT\"}, ...]}++ At load time we build path-keyed indexes by joining `tables.db_id →+ databases.id` and `fields.table_id → tables.id`, so the rest of the+ checker (which speaks portable paths like `[db-name schema table]`) sees+ the same shape it gets from any other source.++ This replaces the 500k-file serdes directory tree."+ (:require+ [clojure.java.io :as io]+ [metabase-enterprise.checker.source :as source]+ [metabase.util.json :as json])+ (:import+ (java.io File)))++(set! *warn-on-reflection* true)++(defn- build-model+ "Parse the concise metadata JSON file and build path-keyed indexes.++ The on-disk shape uses integer IDs to link entities. Here we resolve those+ IDs to their containing entity's name/path so callers can use portable+ references throughout.++ Returns:+ {:databases {db-name db-data}+ :tables {[db-name schema table-name] table-data}+ :fields {[db-name schema table-name field-name] field-data}}"+ [^File f]+ (let [parsed (json/decode (io/input-stream f) true)+ databases (:databases parsed)+ tables (:tables parsed)+ fields (:fields parsed)+ ;; id-keyed lookup tables+ db-by-id (into {} (map (juxt :id identity)) databases)+ tbl-by-id (into {} (map (juxt :id identity)) tables)+ ;; path-keyed result maps+ db-map (into {} (map (juxt :name identity)) databases)+ tbl-map (into {}+ (keep (fn [t]+ (when-let [db (db-by-id (:db_id t))]+ (let [db-name (:name db)+ schema (:schema t)+ table-name (:name t)+ table-path [db-name schema table-name]]+ [table-path (assoc t :db_id db-name)]))))+ tables)+ field-map (into {}+ (keep (fn [field]+ (when-let [t (tbl-by-id (:table_id field))]+ (when-let [db (db-by-id (:db_id t))]+ (let [db-name (:name db)+ schema (:schema t)+ table-name (:name t)+ field-name (:name field)+ table-path [db-name schema table-name]+ field-path [db-name schema table-name field-name]]+ [field-path (assoc field :table_id table-path)])))))+ fields)]+ {:databases db-map+ :tables tbl-map+ :fields field-map}))++(defn- fields-by-table+ "Build a map of table-path → set of field-paths from the fields map."+ [fields-map]+ (reduce-kv (fn [m field-path _]+ (let [table-path (subvec field-path 0 3)]+ (update m table-path (fnil conj #{}) field-path)))+ {}+ fields-map))++(deftype ConciseSchemaSource [databases tables fields fields-index]+ source/SchemaSource+ (resolve-database [_ db-name]+ (get databases db-name))++ (resolve-table [_ table-path]+ (get tables table-path))++ (resolve-field [_ field-path]+ (get fields field-path))++ (fields-for-table [_ table-path]+ (get fields-index table-path))++ (all-field-paths [_]+ (set (keys fields)))++ (all-database-names [_]+ (keys databases))++ (all-table-paths [_]+ (keys tables))++ (tables-for-database [_ db-name]+ (filterv #(= (first %) db-name) (keys tables))))++(defn make-source+ "Create a SchemaSource from a single concise metadata JSON file.+ Loads everything into memory at construction time."+ [schema-file]+ (let [{:keys [databases tables fields]} (build-model (io/file schema-file))+ fi (fields-by-table fields)]+ (->ConciseSchemaSource databases tables fields fi)))diff --git a/enterprise/backend/src/metabase_enterprise/checker/semantic.clj b/enterprise/backend/src/metabase_enterprise/checker/semantic.cljindex 1d905ae2e5c..8ddc8ce0c3f 100644--- a/enterprise/backend/src/metabase_enterprise/checker/semantic.clj+++ b/enterprise/backend/src/metabase_enterprise/checker/semantic.clj@@ -11,6 +11,7 @@- `(setup export-dir schema-dir)` + `(check-one ctx entity-id)` — REPL workflow"(:require[clojure.string :as str]+ [metabase-enterprise.checker.format.concise-schema :as concise-schema][metabase-enterprise.checker.format.serdes-assets :as serdes-assets][metabase-enterprise.checker.format.serdes-schema :as serdes.schema][metabase-enterprise.checker.provider :as provider]@@ -714,11 +715,24 @@(swap! lines conj (str " error: " (:error result))))(str/join "\n" @lines)))))+(defn- make-schema-source+ "Construct a schema source. `schema-format` is one of :serdes or :concise.+ For :serdes, `schema-path` is a directory containing database subdirectories.+ For :concise, `schema-path` is a single JSON file with the concise metadata."+ [schema-format schema-path]+ (case schema-format+ :concise (concise-schema/make-source schema-path)+ :serdes (serdes.schema/make-database-source schema-path)+ (throw (ex-info (str "Unknown schema format: " (pr-str schema-format)+ ". Must be :serdes or :concise.")+ {:schema-format schema-format}))))+(defn- make-sources-and-index- "Build schema and assets sources from `schema-dir` and `export-dir`,- and a merged assets index (cards, dashboards, segments, etc.)."- [export-dir schema-dir]- (let [schema-source (serdes.schema/make-database-source schema-dir)+ "Build schema and assets sources, plus the merged assets index (cards,+ dashboards, segments, etc.). `opts` may contain `:schema-format` (defaults+ to :serdes)."+ [export-dir schema-path & {:keys [schema-format] :or {schema-format :serdes}}]+ (let [schema-source (make-schema-source schema-format schema-path)assets-source (serdes-assets/make-source export-dir)index (serdes-assets/source-index assets-source)]{:schema-source schema-source :assets-source assets-source :index index}))@@ -727,32 +741,45 @@"Check entities in an export directory against database schemas.`export-dir` — directory containing serdes-exported entities (collections/)- `schema-dir` — directory containing serdes-exported database schemas+ `schema-path` — path to schema source (directory for :serdes, file for :concise)`entity-ids` — optional seq of entity-ids to check (defaults to all entities)+ `opts` — optional map; `:schema-format` is :serdes (default) or :conciseReturns `{:results results-map}` where results-map is `{entity-id result-map}`.See the Results processing section above for the result-map and summary-map shapes."- ([export-dir schema-dir]- (let [{:keys [schema-source assets-source index]} (make-sources-and-index export-dir schema-dir)]- {:results (check-entities schema-source assets-source index)}))- ([export-dir schema-dir entity-ids]- (let [{:keys [schema-source assets-source index]} (make-sources-and-index export-dir schema-dir)]- {:results (check-entities schema-source assets-source index entity-ids)})))+ ([export-dir schema-path]+ (check export-dir schema-path nil nil))+ ([export-dir schema-path entity-ids]+ (check export-dir schema-path entity-ids nil))+ ([export-dir schema-path entity-ids opts]+ (let [{:keys [schema-source assets-source index]}+ (make-sources-and-index export-dir schema-path+ :schema-format (:schema-format opts :serdes))]+ {:results (if entity-ids+ (check-entities schema-source assets-source index entity-ids)+ (check-entities schema-source assets-source index))})))(defn setup"Create a store and provider for REPL iteration.Returns {:store store :provider provider :index index}.+ `opts` may contain `:schema-format` (:serdes default, or :concise).+Usage:(def ctx (setup \"/path/to/export\" \"/path/to/schemas\"))+ (def ctx (setup \"/path/to/export\" \"/path/to/metadata.json\" {:schema-format :concise}))(check-one ctx \"entity-id\")(check-one ctx \"entity-id\" :verbose true)"- [export-dir schema-dir]- (let [{:keys [schema-source assets-source index]} (make-sources-and-index export-dir schema-dir)- store (binding [mu.fn/*enforce* false]- (store/make-store schema-source assets-source index))- provider (provider/make-provider store)]- {:store store :provider provider :index index}))+ ([export-dir schema-path]+ (setup export-dir schema-path nil))+ ([export-dir schema-path opts]+ (let [{:keys [schema-source assets-source index]}+ (make-sources-and-index export-dir schema-path+ :schema-format (:schema-format opts :serdes))+ store (binding [mu.fn/*enforce* false]+ (store/make-store schema-source assets-source index))+ provider (provider/make-provider store)]+ {:store store :provider provider :index index})))(defn check-one"Check a single entity by entity-id using a pre-built context from [[setup]].diff --git a/enterprise/backend/src/metabase_enterprise/checker/source.clj b/enterprise/backend/src/metabase_enterprise/checker/source.cljindex a6ffee95911..ddcb2d397b6 100644--- a/enterprise/backend/src/metabase_enterprise/checker/source.clj+++ b/enterprise/backend/src/metabase_enterprise/checker/source.clj@@ -6,16 +6,45 @@- **SchemaSource** resolves database schema references (databases, tables, fields).Backed by the `--schema-dir` directory.- **AssetsSource** resolves serialized entity references (cards, snippets, transforms,- segments). Backed by the `--export` directory.+ segments, dashboards, collections, documents, measures).+ Backed by the `--export` directory.- Portable references:- - Database: string name- - Table: [db-name schema-name table-name] where schema can be nil- - Field: [db-name schema-name table-name field-name]- - Card/snippet/transform/segment: entity-id string+ The checker assigns integer IDs; sources just say 'yes this exists, here's the+ data' or 'no, this reference is unresolved'.- The checker assigns integer IDs; sources just say 'yes this exists, here's the data'- or 'no, this reference is unresolved'.")+ ## The contract++ A source returns plain Clojure maps with **snake_case keys** matching the+ serialized form. The provider (`checker.provider`) is responsible for all+ conversion to lib metadata format:++ - snake_case → kebab-case (`:base_type` → `:base-type`)+ - strings → keywords (`:engine \"h2\"` → `:engine :h2`)+ - adding `:lib/type` discriminators+ - filling in defaults (`:active` defaults to `true` if absent)+ - resolving portable refs (path vectors) to integer IDs via the store+ - MBQL normalization++ This means a source author only has to answer 'given this portable reference,+ give me the raw map' — no need to know about lib metadata, type keywords, or+ the integer ID space. New sources (REST API, git repo, LLM, app DB via+ toucan2) can return the snake_case shape directly.++ ## Portable references++ - Database: string `db-name`+ - Table: `[db-name schema-name table-name]` — `schema-name` may be `nil`+ for schema-less databases (e.g. SQLite)+ - Field: `[db-name schema-name table-name field-name]`+ - Card / snippet / transform / segment / dashboard / collection / document /+ measure: string `entity-id`++ ## Required vs optional keys++ The docstrings below describe each `resolve-*` method's return shape. Keys+ marked **required** must be present for the checker to function. Keys marked+ **optional** are read defensively (the provider treats them as nil/default+ when absent).")(set! *warn-on-reflection* true)@@ -23,52 +52,153 @@"Resolve database schema references to entity data."(resolve-database [this db-name]- "Resolve database by name. Returns map with :name, :engine, :settings or nil.")+ "Resolve a database by its real name. Returns a map or nil.++ Returned map keys (snake_case):+ :id any — required; opaque to checker, used as synthetic ID+ :name string — required+ :engine string — required; provider keywordizes (e.g. \"h2\" → :h2)+ :settings map — optional+ :dbms_version map — optional")(resolve-table [this table-path]- "Resolve table by [db schema table]. Returns map with :name, :schema, :display-name, etc. or nil.")+ "Resolve a table by `[db-name schema-name table-name]`. `schema-name` is nil+ for schema-less databases. Returns a map or nil.++ Returned map keys (snake_case):+ :id any — required; opaque to checker+ :name string — required+ :display_name string — optional+ :schema string — required (may be nil for schema-less DBs)+ :db_id any — required; matches the database :id (or db-name)+ :active bool — optional, defaults to true+ :visibility_type string — optional; provider keywordizes")(resolve-field [this field-path]- "Resolve field by [db schema table field]. Returns map with :name, :base-type, :semantic-type, etc. or nil.")+ "Resolve a field by `[db-name schema-name table-name field-name]`.+ Returns a map or nil.++ Returned map keys (snake_case):+ :id any — required; opaque to checker+ :table_id any — required; matches the table :id (or table-path)+ :name string — required+ :display_name string — optional+ :base_type string — required; provider keywordizes+ :effective_type string — optional; defaults to :base_type+ :semantic_type string — optional; provider keywordizes+ :database_type string — optional+ :active bool — optional, defaults to true+ :visibility_type string — optional; provider keywordizes+ :position int — optional+ :fk_target_field_id vector — optional; must be a field-path vector+ to participate in FK resolution")(fields-for-table [this table-path]- "Return a set of field paths belonging to the given table path [db schema table].")+ "Return a collection of field-paths belonging to the given table-path+ `[db-name schema-name table-name]`. Returns nil if the table is unknown.")(all-field-paths [this]- "Return all known field paths.")+ "Return a collection of all known field-paths.")(all-database-names [this]- "Return all known database names.")+ "Return a collection of all known database names (strings).")(all-table-paths [this]- "Return all known table paths.")+ "Return a collection of all known table-paths.")(tables-for-database [this db-name]- "Return table paths belonging to the given database name."))+ "Return a collection of table-paths belonging to the given database name."))(defprotocol AssetsSource- "Resolve serialized entity references to entity data."+ "Resolve serialized entity references to entity data.++ Each `resolve-*` method takes an `entity-id` string and returns the raw+ entity map (or nil if not found). Maps use snake_case keys matching the+ serialized YAML/JSON form."(resolve-card [this entity-id]- "Resolve card by entity-id. Returns map with :name, :dataset-query, :result-metadata, etc. or nil.")+ "Resolve a card by entity-id. Returns a map or nil.++ Returned map keys (snake_case):+ :id any — required; opaque to checker+ :name string — required+ :type string — required; provider keywordizes (e.g. \"question\")+ :dataset_query map — required; MBQL query, normalized by provider+ Has :database (db-name string or db-id),+ :stages (or :type+:query for legacy form)+ :table_id vector|int — optional; portable path or integer ID+ :result_metadata seq — optional; column maps, may contain portable refs+ in :id, :table_id, :fk_target_field_id, :field_ref+ :archived bool — optional+ :collection_id string — optional; entity-id of containing collection+ :dashboard_id string — optional; entity-id of containing dashboard+ :document_id string — optional; entity-id of containing document")(resolve-snippet [this entity-id]- "Resolve native query snippet by entity-id. Returns map with :name, :content, etc. or nil.")+ "Resolve a native query snippet by entity-id. Returns a map or nil.++ Returned map keys (snake_case):+ :id any — required; opaque to checker+ :name string — required+ :content string — required; the snippet's SQL fragment")(resolve-transform [this entity-id]- "Resolve transform by entity-id. Returns map with :name, :source, etc. or nil.")+ "Resolve a transform by entity-id. Returns a map or nil.++ Returned map keys (snake_case):+ :id any — required; opaque to checker+ :name string — required+ :source map — required+ Has :type (\"query\" for query-based transforms),+ and :query when type=\"query\"+ :source_database_id any — optional; fallback when :source.query has no :database")(resolve-segment [this entity-id]- "Resolve segment by entity-id. Returns map with :name, :definition, etc. or nil.")+ "Resolve a segment by entity-id. Returns a map or nil.++ Returned map keys (snake_case):+ :id any — required; opaque to checker+ :name string — required+ :definition map — required; MBQL definition (a dataset_query)")(resolve-dashboard [this entity-id]- "Resolve dashboard by entity-id. Returns map with :name, :dashcards, :tabs, etc. or nil.")+ "Resolve a dashboard by entity-id. Returns a map or nil.++ Returned map keys (snake_case):+ :id any — required; opaque to checker+ :name string — required+ :tabs seq — required; each tab has :entity_id (string)+ :dashcards seq — required; each dashcard has:+ :entity_id string — required+ :card_id string — optional (nil for virtual cards)+ :dashboard_tab_id vector|string — optional+ :col, :row,+ :size_x, :size_y int — optional, default 0/0/1/1+ :collection_id string — optional; entity-id of containing collection")(resolve-collection [this entity-id]- "Resolve collection by entity-id. Returns map with :name, :parent_id, etc. or nil.")+ "Resolve a collection by entity-id. Returns a map or nil.++ Returned map keys (snake_case):+ :id any — required; opaque to checker+ :name string — required+ :parent_id string — optional; entity-id of parent collection+ :collection_id string — optional; same as :parent_id in some contexts")(resolve-document [this entity-id]- "Resolve document by entity-id. Returns map with :name, :document, etc. or nil.")+ "Resolve a document by entity-id. Returns a map or nil.++ Returned map keys (snake_case):+ :id any — required; opaque to checker+ :name string — required+ :document map — required; ProseMirror content tree+ cardEmbed nodes have :attrs.id with serdes refs+ text nodes may have :marks with link hrefs+ :collection_id string — optional; entity-id of containing collection")(resolve-measure [this entity-id]- "Resolve measure by entity-id. Returns map with :name, :definition, etc. or nil."))+ "Resolve a measure by entity-id. Returns a map or nil.++ Returned map keys (snake_case):+ :id any — required; opaque to checker+ :name string — required"))diff --git a/enterprise/backend/test/metabase_enterprise/checker/format/concise_schema_test.clj b/enterprise/backend/test/metabase_enterprise/checker/format/concise_schema_test.cljnew file mode 100644index 00000000000..4767123fbb6--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/checker/format/concise_schema_test.clj@@ -0,0 +1,119 @@+(ns metabase-enterprise.checker.format.concise-schema-test+ "Tests for the concise JSON schema format — single file with three flat lists."+ (:require+ [clojure.java.io :as io]+ [clojure.test :refer :all]+ [metabase-enterprise.checker.format.concise-schema :as concise-schema]+ [metabase-enterprise.checker.source :as source]))++(set! *warn-on-reflection* true)++(def ^:private fixture-file "test_resources/yaml_checks/concise_metadata.json")++(defn- fixture-path []+ (let [f (io/file fixture-file)]+ (if (.isAbsolute f)+ fixture-file+ (.getPath (io/file (System/getProperty "user.dir") fixture-file)))))++;;; ===========================================================================+;;; Database resolution+;;; ===========================================================================++(deftest resolve-database-test+ (let [source (concise-schema/make-source (fixture-path))]+ (testing "resolves known database"+ (let [db (source/resolve-database source "Test Database")]+ (is (some? db))+ (is (= "Test Database" (:name db)))+ (is (= "h2" (:engine db)))))+ (testing "returns nil for unknown database"+ (is (nil? (source/resolve-database source "Nonexistent"))))))++(deftest all-database-names-test+ (let [source (concise-schema/make-source (fixture-path))]+ (is (= #{"Test Database" "SQLite DB"}+ (set (source/all-database-names source))))))++;;; ===========================================================================+;;; Table resolution+;;; ===========================================================================++(deftest resolve-table-test+ (let [source (concise-schema/make-source (fixture-path))]+ (testing "resolves table with schema"+ (let [table (source/resolve-table source ["Test Database" "public" "orders"])]+ (is (some? table))+ (is (= "orders" (:name table)))+ (is (= "public" (:schema table)))))+ (testing "resolves schema-less table"+ (let [table (source/resolve-table source ["SQLite DB" nil "orders"])]+ (is (some? table))+ (is (= "orders" (:name table)))))+ (testing "returns nil for unknown table"+ (is (nil? (source/resolve-table source ["Test Database" "public" "nope"]))))))++(deftest all-table-paths-test+ (let [source (concise-schema/make-source (fixture-path))+ tables (source/all-table-paths source)]+ (is (= 4 (count tables)))+ (is (some #(= ["Test Database" "public" "orders"] %) tables))+ (is (some #(= ["Test Database" "public" "products"] %) tables))+ (is (some #(= ["SQLite DB" nil "orders"] %) tables))+ (is (some #(= ["SQLite DB" nil "users"] %) tables))))++(deftest tables-for-database-test+ (let [source (concise-schema/make-source (fixture-path))]+ (is (= 2 (count (source/tables-for-database source "Test Database"))))+ (is (= 2 (count (source/tables-for-database source "SQLite DB"))))))++;;; ===========================================================================+;;; Field resolution+;;; ===========================================================================++(deftest resolve-field-test+ (let [source (concise-schema/make-source (fixture-path))]+ (testing "resolves field with schema"+ (let [field (source/resolve-field source ["Test Database" "public" "orders" "total"])]+ (is (some? field))+ (is (= "total" (:name field)))+ (is (= "type/Float" (:base_type field)))+ (is (= "DOUBLE PRECISION" (:database_type field)))))+ (testing "resolves schema-less field"+ (let [field (source/resolve-field source ["SQLite DB" nil "orders" "id"])]+ (is (some? field))+ (is (= "id" (:name field)))))+ (testing "returns nil for unknown field"+ (is (nil? (source/resolve-field source ["Test Database" "public" "orders" "nope"]))))))++(deftest fields-for-table-test+ (let [source (concise-schema/make-source (fixture-path))]+ (testing "returns field paths for table"+ (let [fields (source/fields-for-table source ["Test Database" "public" "orders"])]+ (is (set? fields))+ (is (= 4 (count fields)))+ (is (contains? fields ["Test Database" "public" "orders" "id"]))+ (is (contains? fields ["Test Database" "public" "orders" "total"]))))+ (testing "returns nil for unknown table"+ (is (nil? (source/fields-for-table source ["Nope" "x" "y"]))))))++(deftest all-field-paths-test+ (let [source (concise-schema/make-source (fixture-path))]+ (is (= 11 (count (source/all-field-paths source))))))++;;; ===========================================================================+;;; Data shape — returned maps have keys the provider expects+;;; ===========================================================================++(deftest table-has-schema-and-db-id-test+ (testing "table data includes :schema and :db_id for provider"+ (let [source (concise-schema/make-source (fixture-path))+ table (source/resolve-table source ["Test Database" "public" "orders"])]+ (is (= "public" (:schema table)))+ (is (= "Test Database" (:db_id table))))))++(deftest field-has-table-id-test+ (testing "field data includes :table_id as path vector for provider"+ (let [source (concise-schema/make-source (fixture-path))+ field (source/resolve-field source ["Test Database" "public" "orders" "total"])]+ (is (= ["Test Database" "public" "orders"] (:table_id field))))))diff --git a/test_resources/yaml_checks/concise_metadata.json b/test_resources/yaml_checks/concise_metadata.jsonnew file mode 100644index 00000000000..92a59ab21d0--- /dev/null+++ b/test_resources/yaml_checks/concise_metadata.json@@ -0,0 +1,28 @@+{+ "databases": [+ {"id": 1, "name": "Test Database", "engine": "h2"},+ {"id": 2, "name": "SQLite DB", "engine": "sqlite"}+ ],+ "tables": [+ {"id": 10, "db_id": 1, "name": "orders", "schema": "public"},+ {"id": 11, "db_id": 1, "name": "products", "schema": "public"},+ {"id": 20, "db_id": 2, "name": "orders"},+ {"id": 21, "db_id": 2, "name": "users"}+ ],+ "fields": [+ {"id": 100, "table_id": 10, "name": "id", "base_type": "type/BigInteger", "semantic_type": "type/PK", "database_type": "BIGINT"},+ {"id": 101, "table_id": 10, "name": "total", "base_type": "type/Float", "database_type": "DOUBLE PRECISION"},+ {"id": 102, "table_id": 10, "name": "product_id", "base_type": "type/Integer", "semantic_type": "type/FK", "database_type": "INTEGER"},+ {"id": 103, "table_id": 10, "name": "created_at", "base_type": "type/DateTime", "database_type": "TIMESTAMP"},++ {"id": 110, "table_id": 11, "name": "id", "base_type": "type/BigInteger", "semantic_type": "type/PK", "database_type": "BIGINT"},+ {"id": 111, "table_id": 11, "name": "title", "base_type": "type/Text", "database_type": "CHARACTER VARYING"},+ {"id": 112, "table_id": 11, "name": "price", "base_type": "type/Float", "database_type": "DOUBLE PRECISION"},++ {"id": 200, "table_id": 20, "name": "id", "base_type": "type/BigInteger", "semantic_type": "type/PK", "database_type": "INTEGER"},+ {"id": 201, "table_id": 20, "name": "user_id", "base_type": "type/Integer", "semantic_type": "type/FK", "database_type": "INTEGER"},++ {"id": 210, "table_id": 21, "name": "id", "base_type": "type/BigInteger", "semantic_type": "type/PK", "database_type": "INTEGER"},+ {"id": 211, "table_id": 21, "name": "name", "base_type": "type/Text", "database_type": "TEXT"}+ ]+}