Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Split source (#72666)

dpsutton113ee31093db
diff --git a/enterprise/backend/src/metabase_enterprise/checker/format/serdes_assets.clj b/enterprise/backend/src/metabase_enterprise/checker/format/serdes_assets.clj
new file mode 100644
index 00000000000..75401d6d07a
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/checker/format/serdes_assets.clj
@@ -0,0 +1,304 @@
+(ns metabase-enterprise.checker.format.serdes-assets
+ "Serdes export format — directory structure, path parsing, entity loading.
+
+ Directory structure (on disk, names may be slugified):
+ databases/<db-dir>/<db-dir>.yaml
+ databases/<db-dir>/schemas/<schema-dir>/tables/<table-dir>/<table-dir>.yaml
+ databases/<db-dir>/schemas/<schema-dir>/tables/<table-dir>/fields/<field>.yaml
+ databases/<db-dir>/tables/<table-dir>/<table-dir>.yaml # schema-less
+ databases/<db-dir>/tables/<table-dir>/fields/<field>.yaml # schema-less
+ collections/<entity-id>_<slug>/<entity-id>_<slug>.yaml
+ collections/.../cards/<entity-id>_<slug>.yaml
+ collections/.../dashboards/<entity-id>_<slug>.yaml
+
+ Directory names may differ from entity names (e.g. `analytics_data_warehouse`
+ for \"Analytics Data Warehouse\", or `orders` for \"ORDERS\" in H2).
+
+ ## Lazy resolution cache
+
+ The `SerdesSource` maintains a single lazily-populated cache atom that maps
+ real entity names (from YAML) to file locations:
+
+ {db-name ; real name, e.g. \"Analytics Data Warehouse\"
+ {schema ; real name or nil for schema-less
+ ::not-indexed ; schema exists but tables not yet walked
+ | {table-name ; real name from YAML
+ {:table-file \"path/to/table.yaml\" ; path string to the table YAML
+ :fields ::not-indexed ; fields not yet walked for this table
+ | {field-name \"path\" ...}}}}} ; real name → field YAML path
+
+ Two levels of laziness:
+ 1. Schema → tables: realized when any table in that schema is resolved
+ (via `resolve-table`, `resolve-field`, `fields-for-table`) or when
+ `tables-for-database` is called. Walks the schema's `tables/` directory
+ and reads each table's `name:` from YAML via fast regex.
+ 2. Table → fields: realized when `resolve-field` or `fields-for-table` is
+ called for that specific table. Walks the table's `fields/` directory
+ and reads each field's `name:` via fast regex. Other tables in the same
+ schema are unaffected.
+
+ The `db-name->dir` map (built at startup from ~40 database YAMLs) handles the
+ database name → directory name translation. Schema and table name → directory
+ name translation is handled by case-insensitive directory matching during
+ schema indexing."
+ (:require
+ [clojure.java.io :as io]
+ [clojure.string :as str]
+ [metabase-enterprise.checker.source :as source]
+ [metabase.util.yaml :as yaml])
+ (:import
+ (java.io File)))
+
+(set! *warn-on-reflection* true)
+
+;;; ===========================================================================
+;;; YAML Loading
+;;; ===========================================================================
+
+(defn load-yaml
+ "Load and parse a YAML file."
+ [path]
+ (yaml/parse-string (slurp path)))
+
+(defn- db-dir
+ "Resolve a real database name to its directory on disk."
+ ^File [^File databases-dir db-name->dir db-name]
+ (let [dir-name (get db-name->dir db-name db-name)]
+ (io/file databases-dir dir-name)))
+
+(defn quick-extract
+ "Extract a field from YAML using regex (fast) with full parse fallback."
+ [file-path field-name pattern]
+ (try
+ (let [content (slurp file-path)]
+ (if-let [[_ value] (re-find pattern content)]
+ (str/trim value)
+ (get (yaml/parse-string content) (keyword field-name))))
+ (catch Exception _ nil)))
+
+(defn- read-yaml-name
+ "Read the `name:` field from a YAML file using regex (fast).
+ Falls back to `fallback` if the file doesn't exist or can't be read."
+ [^File yaml-file ^String fallback]
+ (if (.exists yaml-file)
+ (or (quick-extract (.getPath yaml-file) "name" #"(?m)^name:\s*(.+)")
+ fallback)
+ fallback))
+
+(defn extract-entity-id
+ "Extract the entity id from a yaml file without parsing it. Looking for a top level entity_id at start of line."
+ [file-path]
+ (quick-extract file-path "entity_id" #"(?m)^entity_id:\s*(\S+)"))
+
+(defn extract-model
+ "Extract the serdes model from a YAML file (e.g., \"Collection\", \"Card\", \"Dashboard\").
+ Looks for the model field inside serdes/meta to avoid matching embedded refs."
+ [file-path]
+ (try
+ (let [content (slurp file-path)]
+ (when-let [[_ model] (re-find #"(?m)^serdes/meta:\s*\n\s*- id:.*\n\s+(?:label:.*\n\s+)?model:\s*(\S+)" content)]
+ (str/trim model)))
+ (catch Exception _ nil)))
+
+(defn- index-segments-and-measures
+ "Walk a table directory's segments/ and measures/ subdirectories.
+ Reads entity_id from each YAML (only a handful exist).
+ Returns a seq of {:kind :ref :file} entries."
+ [^File table-dir]
+ (into []
+ (mapcat
+ (fn [[subdir kind]]
+ (let [^File d (io/file table-dir subdir)]
+ (when (.isDirectory d)
+ (keep (fn [^File f]
+ (when (and (.isFile f) (str/ends-with? (.getName f) ".yaml"))
+ (when-let [eid (extract-entity-id (.getPath f))]
+ {:kind kind :ref eid :file (.getPath f)})))
+ (.listFiles d))))))
+ [["segments" :segment] ["measures" :measure]]))
+
+(defn- list-table-dirs
+ "Return a seq of table directory Files for a database.
+ Walks schemas/<schema>/tables/<table>/ and tables/<table>/ directories."
+ [^File databases-dir db-name->dir-map db-name]
+ (let [^File db (db-dir databases-dir db-name->dir-map db-name)
+ result (volatile! [])]
+ (let [^File schemas-dir (io/file db "schemas")]
+ (when (.isDirectory schemas-dir)
+ (doseq [^File schema-d (.listFiles schemas-dir)
+ :when (.isDirectory schema-d)]
+ (let [^File tables-dir (io/file schema-d "tables")]
+ (when (.isDirectory tables-dir)
+ (doseq [^File td (.listFiles tables-dir)
+ :when (.isDirectory td)]
+ (vswap! result conj td)))))))
+ (let [^File tables-dir (io/file db "tables")]
+ (when (.isDirectory tables-dir)
+ (doseq [^File td (.listFiles tables-dir)
+ :when (.isDirectory td)]
+ (vswap! result conj td))))
+ @result))
+
+(defn- index-schema-dir
+ "Walk a databases directory and index databases, segments, and measures.
+ Reads ~40 database YAMLs for real names, plus a handful of segment/measure YAMLs
+ for entity_ids. Tables and fields are resolved on demand from the filesystem.
+
+ When `include-segments-measures?` is true, also walks table directories
+ to find segment and measure YAMLs (for export dirs). Set to false for
+ schema-only dirs to avoid walking thousands of table directories.
+
+ Returns {:entries [...]
+ :db-name->dir {real-db-name dir-name}}."
+ [schema-dir & {:keys [include-segments-measures?]}]
+ (let [schema-dir (io/file schema-dir)
+ db-name->dir (volatile! {})
+ entries (volatile! [])]
+ (doseq [^File db-dir (.listFiles schema-dir)
+ :when (.isDirectory db-dir)]
+ (let [dir-name (.getName db-dir)
+ db-yaml (io/file db-dir (str dir-name ".yaml"))
+ db-name (read-yaml-name db-yaml dir-name)]
+ (vswap! db-name->dir assoc db-name dir-name)
+ (vswap! entries conj {:kind :database :ref db-name
+ :file (if (.exists db-yaml) (.getPath db-yaml) (.getPath db-dir))})
+ ;; Walk table directories for segments and measures (export dirs only)
+ (when include-segments-measures?
+ (doseq [^File table-dir (list-table-dirs schema-dir @db-name->dir db-name)]
+ (vswap! entries into (index-segments-and-measures table-dir))))))
+ {:entries @entries
+ :db-name->dir @db-name->dir}))
+
+;;; ===========================================================================
+;;; File Walking - Build index of all entities
+;;; ===========================================================================
+
+(def ^:private model->kind
+ "Map serdes model names to index kinds."
+ {"Card" :card
+ "Dashboard" :dashboard
+ "Collection" :collection
+ "Document" :document
+ "Measure" :measure
+ "Segment" :segment
+ "NativeQuerySnippet" :snippet
+ "Transform" :transform})
+
+(defn- index-collections-tree
+ "Walk the collections/ directory and index all entities by their serdes model.
+ Returns entries for cards, dashboards, and collections."
+ [export-dir]
+ (let [collections-dir (io/file export-dir "collections")]
+ (when (.isDirectory collections-dir)
+ (for [^File file (file-seq collections-dir)
+ :when (.isFile file)
+ :let [path (.getPath file)]
+ :when (str/ends-with? (.getName file) ".yaml")
+ :let [model (extract-model path)
+ kind (get model->kind model)]
+ :when kind
+ :let [entity-id (extract-entity-id path)]
+ :when entity-id]
+ {:kind kind :ref entity-id :file path}))))
+
+(defn build-file-index
+ "Build index of all entity files in an export directory.
+ Returns `{:index {kind {ref file-path}} :db-name->dir {real-name dir-name}}`
+ where index also has `:duplicates` for refs appearing in multiple files."
+ [export-dir]
+ (let [databases-dir (io/file export-dir "databases")
+ schema-result (when (.isDirectory databases-dir)
+ (index-schema-dir databases-dir :include-segments-measures? true))
+ entries (concat
+ (:entries schema-result)
+ (index-collections-tree export-dir))
+ ;; Group by [kind ref] to find duplicates
+ by-key (reduce (fn [m {:keys [kind ref file]}]
+ (update m [kind ref] (fnil conj []) file))
+ {}
+ entries)
+ ;; Build the index (first file wins) and collect duplicates
+ index (reduce-kv (fn [idx [kind ref] files]
+ (assoc-in idx [kind ref] (first files)))
+ {}
+ by-key)
+ dupes (into []
+ (keep (fn [[[kind ref] files]]
+ (when (> (count files) 1)
+ {:kind kind :ref ref :files files})))
+ by-key)]
+ (cond-> index
+ (seq dupes) (assoc :duplicates dupes))))
+
+(defn index-stats
+ "Get statistics about a file index.
+ Note: tables and fields are resolved on demand and won't appear in the index
+ for schema-dir sources. They will be 0 for `build-database-dir-index` results."
+ [index]
+ {:databases (count (:database index))
+ :tables (count (:table index))
+ :fields (count (:field index))
+ :measures (count (:measure index))
+ :segments (count (:segment index))
+ :cards (count (:card index))
+ :dashboards (count (:dashboard index))
+ :collections (count (:collection index))
+ :database-names (vec (keys (:database index)))})
+
+(deftype SerdesSource [databases-dir assets-index]
+ source/AssetsSource
+ (resolve-card [_ entity-id]
+ (when-let [file (get-in assets-index [:card entity-id])]
+ (load-yaml file)))
+
+ (resolve-snippet [_ entity-id]
+ (when-let [file (get-in assets-index [:snippet entity-id])]
+ (load-yaml file)))
+
+ (resolve-transform [_ entity-id]
+ (when-let [file (get-in assets-index [:transform entity-id])]
+ (load-yaml file)))
+
+ (resolve-segment [_ entity-id]
+ (when-let [file (get-in assets-index [:segment entity-id])]
+ (load-yaml file)))
+
+ (resolve-dashboard [_ entity-id]
+ (when-let [file (get-in assets-index [:dashboard entity-id])]
+ (load-yaml file)))
+
+ (resolve-collection [_ entity-id]
+ (when-let [file (get-in assets-index [:collection entity-id])]
+ (load-yaml file)))
+
+ (resolve-document [_ entity-id]
+ (when-let [file (get-in assets-index [:document entity-id])]
+ (load-yaml file)))
+
+ (resolve-measure [_ entity-id]
+ (when-let [file (get-in assets-index [:measure entity-id])]
+ (load-yaml file))))
+
+(defn make-source
+ "Create a MetadataSource for a serdes export directory.
+ The databases-dir for field resolution is export-dir/databases."
+ [export-dir]
+ (let [databases-dir (io/file export-dir "databases")
+ index (build-file-index export-dir)
+ assets-index (dissoc index :database)]
+ (->SerdesSource databases-dir assets-index)))
+
+(defn source-index
+ "Get the assets index from a SerdesSource. Returns a flat map of
+ {[kind ref] file-path} for cards, dashboards, segments, etc."
+ [^SerdesSource source]
+ (.-assets-index source))
+
+;;; ===========================================================================
+;;; Enumeration
+;;; ===========================================================================
+
+(defn all-card-ids
+ "Get all card entity-ids from source."
+ [^SerdesSource source]
+ (keys (:card (.-assets-index source))))
diff --git a/enterprise/backend/src/metabase_enterprise/checker/format/serdes.clj b/enterprise/backend/src/metabase_enterprise/checker/format/serdes_schema.clj
similarity index 65%
rename from enterprise/backend/src/metabase_enterprise/checker/format/serdes.clj
rename to enterprise/backend/src/metabase_enterprise/checker/format/serdes_schema.clj
index 0845a829640..d3d7d5652d2 100644
--- a/enterprise/backend/src/metabase_enterprise/checker/format/serdes.clj
+++ b/enterprise/backend/src/metabase_enterprise/checker/format/serdes_schema.clj
@@ -1,4 +1,4 @@
-(ns metabase-enterprise.checker.format.serdes
+(ns metabase-enterprise.checker.format.serdes-schema
"Serdes export format — directory structure, path parsing, entity loading.
Directory structure (on disk, names may be slugified):
@@ -53,15 +53,47 @@
(set! *warn-on-reflection* true)
-;;; ===========================================================================
-;;; YAML Loading
-;;; ===========================================================================
-
(defn load-yaml
"Load and parse a YAML file."
[path]
(yaml/parse-string (slurp path)))
+(defn- db-dir
+ "Resolve a real database name to its directory on disk."
+ ^File [^File databases-dir db-name->dir db-name]
+ (let [dir-name (get db-name->dir db-name db-name)]
+ (io/file databases-dir dir-name)))
+
+;;; ===========================================================================
+;;; MetadataSource Implementation
+;;;
+;;; Returns raw YAML data - the checker handles conversion to lib format.
+;;; This keeps format knowledge here, lib knowledge in checker.
+;;; ===========================================================================
+
+(defn- resolve-schema-name
+ "Resolve a real schema name to the key used in the model.
+
+ Schema names are the one entity type with no canonical YAML file — the directory
+ name is the filesystem key, and the real name comes from the `schema:` field in
+ table YAMLs. These may differ (e.g. dir=`public`, real=`PUBLIC` for H2).
+
+ Strategy:
+ 1. Exact match in the model's :schemas
+ 2. Case-insensitive match (handles H2-style uppercasing)
+ 3. nil for schema-less databases
+
+ To change schema resolution in the future, modify this function."
+ [model db-name schema]
+ (let [schemas (:schemas (get model db-name))
+ lower (fn [s] (.toLowerCase (str s) Locale/US))]
+ (cond
+ (contains? schemas schema) schema
+ schema (let [lowered (lower schema)]
+ (some (fn [k] (when (and (string? k) (= (lower k) lowered)) k))
+ (keys schemas)))
+ :else (when (contains? schemas nil) nil))))
+
(defn quick-extract
"Extract a field from YAML using regex (fast) with full parse fallback."
[file-path field-name pattern]
@@ -72,20 +104,14 @@
(get (yaml/parse-string content) (keyword field-name))))
(catch Exception _ nil)))
-(defn extract-entity-id
- "Extract the entity id from a yaml file without parsing it. Looking for a top level entity_id at start of line."
- [file-path]
- (quick-extract file-path "entity_id" #"(?m)^entity_id:\s*(\S+)"))
-
-(defn extract-model
- "Extract the serdes model from a YAML file (e.g., \"Collection\", \"Card\", \"Dashboard\").
- Looks for the model field inside serdes/meta to avoid matching embedded refs."
- [file-path]
- (try
- (let [content (slurp file-path)]
- (when-let [[_ model] (re-find #"(?m)^serdes/meta:\s*\n\s*- id:.*\n\s+(?:label:.*\n\s+)?model:\s*(\S+)" content)]
- (str/trim model)))
- (catch Exception _ nil)))
+(defn- read-yaml-schema
+ "Read the `schema:` field from a YAML file using regex (fast).
+ Returns nil if not found or if the value is `null`."
+ [^File yaml-file]
+ (when (.exists yaml-file)
+ (let [v (quick-extract (.getPath yaml-file) "schema" #"(?m)^schema:\s*(\S+)")]
+ (when (and v (not= v "null"))
+ v))))
(defn- read-yaml-name
"Read the `name:` field from a YAML file using regex (fast).
@@ -96,178 +122,6 @@
fallback)
fallback))
-(defn- db-dir
- "Resolve a real database name to its directory on disk."
- ^File [^File databases-dir db-name->dir db-name]
- (let [dir-name (get db-name->dir db-name db-name)]
- (io/file databases-dir dir-name)))
-
-(defn- list-table-dirs
- "Return a seq of table directory Files for a database.
- Walks schemas/<schema>/tables/<table>/ and tables/<table>/ directories."
- [^File databases-dir db-name->dir-map db-name]
- (let [^File db (db-dir databases-dir db-name->dir-map db-name)
- result (volatile! [])]
- (let [^File schemas-dir (io/file db "schemas")]
- (when (.isDirectory schemas-dir)
- (doseq [^File schema-d (.listFiles schemas-dir)
- :when (.isDirectory schema-d)]
- (let [^File tables-dir (io/file schema-d "tables")]
- (when (.isDirectory tables-dir)
- (doseq [^File td (.listFiles tables-dir)
- :when (.isDirectory td)]
- (vswap! result conj td)))))))
- (let [^File tables-dir (io/file db "tables")]
- (when (.isDirectory tables-dir)
- (doseq [^File td (.listFiles tables-dir)
- :when (.isDirectory td)]
- (vswap! result conj td))))
- @result))
-
-(defn- index-segments-and-measures
- "Walk a table directory's segments/ and measures/ subdirectories.
- Reads entity_id from each YAML (only a handful exist).
- Returns a seq of {:kind :ref :file} entries."
- [^File table-dir]
- (into []
- (mapcat
- (fn [[subdir kind]]
- (let [^File d (io/file table-dir subdir)]
- (when (.isDirectory d)
- (keep (fn [^File f]
- (when (and (.isFile f) (str/ends-with? (.getName f) ".yaml"))
- (when-let [eid (extract-entity-id (.getPath f))]
- {:kind kind :ref eid :file (.getPath f)})))
- (.listFiles d))))))
- [["segments" :segment] ["measures" :measure]]))
-
-(defn- index-schema-dir
- "Walk a databases directory and index databases, segments, and measures.
- Reads ~40 database YAMLs for real names, plus a handful of segment/measure YAMLs
- for entity_ids. Tables and fields are resolved on demand from the filesystem.
-
- When `include-segments-measures?` is true, also walks table directories
- to find segment and measure YAMLs (for export dirs). Set to false for
- schema-only dirs to avoid walking thousands of table directories.
-
- Returns {:entries [...]
- :db-name->dir {real-db-name dir-name}}."
- [schema-dir & {:keys [include-segments-measures?]}]
- (let [schema-dir (io/file schema-dir)
- db-name->dir (volatile! {})
- entries (volatile! [])]
- (doseq [^File db-dir (.listFiles schema-dir)
- :when (.isDirectory db-dir)]
- (let [dir-name (.getName db-dir)
- db-yaml (io/file db-dir (str dir-name ".yaml"))
- db-name (read-yaml-name db-yaml dir-name)]
- (vswap! db-name->dir assoc db-name dir-name)
- (vswap! entries conj {:kind :database :ref db-name
- :file (if (.exists db-yaml) (.getPath db-yaml) (.getPath db-dir))})
- ;; Walk table directories for segments and measures (export dirs only)
- (when include-segments-measures?
- (doseq [^File table-dir (list-table-dirs schema-dir @db-name->dir db-name)]
- (vswap! entries into (index-segments-and-measures table-dir))))))
- {:entries @entries
- :db-name->dir @db-name->dir}))
-
-;;; ===========================================================================
-;;; File Walking - Build index of all entities
-;;; ===========================================================================
-
-(def ^:private model->kind
- "Map serdes model names to index kinds."
- {"Card" :card
- "Dashboard" :dashboard
- "Collection" :collection
- "Document" :document
- "Measure" :measure
- "Segment" :segment
- "NativeQuerySnippet" :snippet
- "Transform" :transform})
-
-(defn- index-collections-tree
- "Walk the collections/ directory and index all entities by their serdes model.
- Returns entries for cards, dashboards, and collections."
- [export-dir]
- (let [collections-dir (io/file export-dir "collections")]
- (when (.isDirectory collections-dir)
- (for [^File file (file-seq collections-dir)
- :when (.isFile file)
- :let [path (.getPath file)]
- :when (str/ends-with? (.getName file) ".yaml")
- :let [model (extract-model path)
- kind (get model->kind model)]
- :when kind
- :let [entity-id (extract-entity-id path)]
- :when entity-id]
- {:kind kind :ref entity-id :file path}))))
-
-(defn build-file-index
- "Build index of all entity files in an export directory.
- Returns `{:index {kind {ref file-path}} :db-name->dir {real-name dir-name}}`
- where index also has `:duplicates` for refs appearing in multiple files."
- [export-dir]
- (let [databases-dir (io/file export-dir "databases")
- schema-result (when (.isDirectory databases-dir)
- (index-schema-dir databases-dir :include-segments-measures? true))
- entries (concat
- (:entries schema-result)
- (index-collections-tree export-dir))
- ;; Group by [kind ref] to find duplicates
- by-key (reduce (fn [m {:keys [kind ref file]}]
- (update m [kind ref] (fnil conj []) file))
- {}
- entries)
- ;; Build the index (first file wins) and collect duplicates
- index (reduce-kv (fn [idx [kind ref] files]
- (assoc-in idx [kind ref] (first files)))
- {}
- by-key)
- dupes (into []
- (keep (fn [[[kind ref] files]]
- (when (> (count files) 1)
- {:kind kind :ref ref :files files})))
- by-key)]
- {:index (cond-> index
- (seq dupes) (assoc :duplicates dupes))
- :db-name->dir (:db-name->dir schema-result)}))
-
-(defn build-database-dir-index
- "Build index of databases from a schema directory.
- Tables and fields are resolved on demand.
-
- Returns `{:index {:database {name file-path}} :db-name->dir {real-name dir-name}}`."
- [databases-dir]
- (let [{:keys [entries db-name->dir]} (index-schema-dir databases-dir)]
- {:index (reduce (fn [idx {:keys [kind ref file]}]
- (assoc-in idx [kind ref] file))
- {}
- entries)
- :db-name->dir db-name->dir}))
-
-(defn index-stats
- "Get statistics about a file index.
- Note: tables and fields are resolved on demand and won't appear in the index
- for schema-dir sources. They will be 0 for `build-database-dir-index` results."
- [index]
- {:databases (count (:database index))
- :tables (count (:table index))
- :fields (count (:field index))
- :measures (count (:measure index))
- :segments (count (:segment index))
- :cards (count (:card index))
- :dashboards (count (:dashboard index))
- :collections (count (:collection index))
- :database-names (vec (keys (:database index)))})
-
-;;; ===========================================================================
-;;; MetadataSource Implementation
-;;;
-;;; Returns raw YAML data - the checker handles conversion to lib format.
-;;; This keeps format knowledge here, lib knowledge in checker.
-;;; ===========================================================================
-
(defn- table-real-name
"Read the real table name from its YAML file via regex. Falls back to dir name."
[^File table-dir]
@@ -275,22 +129,6 @@
^File yaml (io/file table-dir (str dir-name ".yaml"))]
(read-yaml-name yaml dir-name)))
-(defn- field-real-name
- "Read the real field name from its YAML file via regex. Falls back to filename without extension."
- [^File field-yaml]
- (let [fname (.getName field-yaml)
- fallback (str/replace fname #"\.yaml$" "")]
- (read-yaml-name field-yaml fallback)))
-
-(defn- read-yaml-schema
- "Read the `schema:` field from a YAML file using regex (fast).
- Returns nil if not found or if the value is `null`."
- [^File yaml-file]
- (when (.exists yaml-file)
- (let [v (quick-extract (.getPath yaml-file) "schema" #"(?m)^schema:\s*(\S+)")]
- (when (and v (not= v "null"))
- v))))
-
(defn- index-schema-tables
"Walk a single schema's tables directory and return:
{:real-schema-name string-or-nil ; the real schema name from YAML (may differ from dir name)
@@ -314,48 +152,6 @@
{:real-schema-name @real-schema
:tables tables})))
-(defn- index-table-fields
- "Walk a table's fields/ directory and return a map of:
- {real-field-name File ...}
- Reads real field names from YAMLs via fast regex.
- Skips FieldValues and FieldUserSettings files."
- [table-file-path]
- (let [^File table-dir (.getParentFile (io/file table-file-path))
- ^File fields-dir (io/file table-dir "fields")]
- (if (.isDirectory fields-dir)
- (into {}
- (keep (fn [^File f]
- (let [fname (.getName f)]
- (when (and (.isFile f)
- (str/ends-with? fname ".yaml")
- (not (str/includes? fname "___")))
- [(field-real-name f) (.getPath f)]))))
- (.listFiles fields-dir))
- {})))
-
-(defn- resolve-schema-name
- "Resolve a real schema name to the key used in the model.
-
- Schema names are the one entity type with no canonical YAML file — the directory
- name is the filesystem key, and the real name comes from the `schema:` field in
- table YAMLs. These may differ (e.g. dir=`public`, real=`PUBLIC` for H2).
-
- Strategy:
- 1. Exact match in the model's :schemas
- 2. Case-insensitive match (handles H2-style uppercasing)
- 3. nil for schema-less databases
-
- To change schema resolution in the future, modify this function."
- [model db-name schema]
- (let [schemas (:schemas (get model db-name))
- lower (fn [s] (.toLowerCase (str s) Locale/US))]
- (cond
- (contains? schemas schema) schema
- schema (let [lowered (lower schema)]
- (some (fn [k] (when (and (string? k) (= (lower k) lowered)) k))
- (keys schemas)))
- :else (when (contains? schemas nil) nil))))
-
(defn- ensure-schema-indexed!
"Ensure a schema's tables are indexed in the model. No-op if already done.
@@ -384,6 +180,32 @@
true
(assoc-in [db-name :schemas final-key] tables))))))))
+(defn- field-real-name
+ "Read the real field name from its YAML file via regex. Falls back to filename without extension."
+ [^File field-yaml]
+ (let [fname (.getName field-yaml)
+ fallback (str/replace fname #"\.yaml$" "")]
+ (read-yaml-name field-yaml fallback)))
+
+(defn- index-table-fields
+ "Walk a table's fields/ directory and return a map of:
+ {real-field-name File ...}
+ Reads real field names from YAMLs via fast regex.
+ Skips FieldValues and FieldUserSettings files."
+ [table-file-path]
+ (let [^File table-dir (.getParentFile (io/file table-file-path))
+ ^File fields-dir (io/file table-dir "fields")]
+ (if (.isDirectory fields-dir)
+ (into {}
+ (keep (fn [^File f]
+ (let [fname (.getName f)]
+ (when (and (.isFile f)
+ (str/ends-with? fname ".yaml")
+ (not (str/includes? fname "___")))
+ [(field-real-name f) (.getPath f)]))))
+ (.listFiles fields-dir))
+ {})))
+
(defn- ensure-fields-indexed!
"Ensure a table's fields are indexed in the model. No-op if already done."
[^File databases-dir db-name->dir db-name schema table-name model]
@@ -393,47 +215,53 @@
(let [field-map (index-table-fields table-file)]
(swap! model assoc-in [db-name :schemas schema table-name :fields] field-map)))))
-(defn- list-schemas
- "List schema directory names for a database. Returns seq of strings, or [nil] for schema-less."
- [^File databases-dir db-name->dir db-name]
- (let [^File db (db-dir databases-dir db-name->dir db-name)
- ^File schemas-dir (io/file db "schemas")
- ^File tables-dir (io/file db "tables")]
- (cond-> []
- (.isDirectory schemas-dir)
- (into (map #(.getName ^File %))
- (filter #(.isDirectory ^File %) (.listFiles schemas-dir)))
- (.isDirectory tables-dir)
- (conj nil))))
+(defn- index-schema-dir
+ "Walk a databases directory and index databases, segments, and measures.
+ Reads ~40 database YAMLs for real names, plus a handful of segment/measure YAMLs
+ for entity_ids. Tables and fields are resolved on demand from the filesystem.
-(defn- init-schema-model
- "Build the initial schema model with db-file paths and ::not-indexed for each schema.
+ When `include-segments-measures?` is true, also walks table directories
+ to find segment and measure YAMLs (for export dirs). Set to false for
+ schema-only dirs to avoid walking thousands of table directories.
- Schema model shape:
- {db-name {:db-file \"path/to/db.yaml\"
- :schemas {schema ::not-indexed
- | {table {:table-file \"path\" :fields ::not-indexed | {field \"path\"}}}}}}
+ Returns {:entries [...]
+ :db-name->dir {real-db-name dir-name}}."
+ [schema-dir]
+ (let [schema-dir (io/file schema-dir)
+ db-name->dir (volatile! {})
+ entries (volatile! [])]
+ (doseq [^File db-dir (.listFiles schema-dir)
+ :when (.isDirectory db-dir)]
+ (let [dir-name (.getName db-dir)
+ db-yaml (io/file db-dir (str dir-name ".yaml"))
+ db-name (read-yaml-name db-yaml dir-name)]
+ (vswap! db-name->dir assoc db-name dir-name)
+ (vswap! entries conj {:kind :database :ref db-name
+ :file (if (.exists db-yaml) (.getPath db-yaml) (.getPath db-dir))})))
+ {:entries @entries
+ :db-name->dir @db-name->dir}))
- Two levels of laziness:
- 1. Schema → tables: populated when the schema is first accessed
- 2. Table → fields: populated when that table's fields are first accessed"
- [^File databases-dir db-name->dir db-file-index]
- (atom
- (into {}
- (for [[db-name _dir-name] db-name->dir]
- [db-name {:db-file (get db-file-index db-name)
- :schemas (into {}
- (for [schema (list-schemas databases-dir db-name->dir db-name)]
- [schema ::not-indexed]))}]))))
+(defn build-database-dir-index
+ "Build index of databases from a schema directory.
+ Tables and fields are resolved on demand.
+
+ Returns `{:index {:database {name file-path}} :db-name->dir {real-name dir-name}}`."
+ [databases-dir]
+ (let [{:keys [entries db-name->dir]} (index-schema-dir databases-dir)]
+ {:index (reduce (fn [idx {:keys [kind ref file]}]
+ (assoc-in idx [kind ref] file))
+ {}
+ entries)
+ :db-name->dir db-name->dir}))
(p.types/defprotocol+ ISchemaModelDebug
"Non exported protocol to get the schema of the SerdesSource. Useful for debugging and asserting on the schema model
built up during serdes."
(schema-model [_] "Deref and return the schema model. See [[init-schema-model]] for details of this structure."))
-(deftype SerdesSource [databases-dir assets-index db-name->dir
- ;; See init-schema-model for shape documentation
- schema-model]
+(deftype SerdesSchemaSource [databases-dir db-name->dir
+ ;; See init-schema-model for shape documentation
+ schema-model]
source/SchemaSource
(resolve-database [_ db-name]
(when-let [file (get-in @schema-model [db-name :db-file])]
@@ -499,51 +327,41 @@
table-name (keys tables)]
[db-name schema table-name])))
- source/AssetsSource
- (resolve-card [_ entity-id]
- (when-let [file (get-in assets-index [:card entity-id])]
- (load-yaml file)))
-
- (resolve-snippet [_ entity-id]
- (when-let [file (get-in assets-index [:snippet entity-id])]
- (load-yaml file)))
-
- (resolve-transform [_ entity-id]
- (when-let [file (get-in assets-index [:transform entity-id])]
- (load-yaml file)))
-
- (resolve-segment [_ entity-id]
- (when-let [file (get-in assets-index [:segment entity-id])]
- (load-yaml file)))
-
- (resolve-dashboard [_ entity-id]
- (when-let [file (get-in assets-index [:dashboard entity-id])]
- (load-yaml file)))
+ ISchemaModelDebug
+ (schema-model [_] (deref schema-model)))
- (resolve-collection [_ entity-id]
- (when-let [file (get-in assets-index [:collection entity-id])]
- (load-yaml file)))
+(defn- list-schemas
+ "List schema directory names for a database. Returns seq of strings, or [nil] for schema-less."
+ [^File databases-dir db-name->dir db-name]
+ (let [^File db (db-dir databases-dir db-name->dir db-name)
+ ^File schemas-dir (io/file db "schemas")
+ ^File tables-dir (io/file db "tables")]
+ (cond-> []
+ (.isDirectory schemas-dir)
+ (into (map #(.getName ^File %))
+ (filter #(.isDirectory ^File %) (.listFiles schemas-dir)))
+ (.isDirectory tables-dir)
+ (conj nil))))
- (resolve-document [_ entity-id]
- (when-let [file (get-in assets-index [:document entity-id])]
- (load-yaml file)))
+(defn- init-schema-model
+ "Build the initial schema model with db-file paths and ::not-indexed for each schema.
- (resolve-measure [_ entity-id]
- (when-let [file (get-in assets-index [:measure entity-id])]
- (load-yaml file)))
- ISchemaModelDebug
- (schema-model [_] (deref schema-model)))
+ Schema model shape:
+ {db-name {:db-file \"path/to/db.yaml\"
+ :schemas {schema ::not-indexed
+ | {table {:table-file \"path\" :fields ::not-indexed | {field \"path\"}}}}}}
-(defn make-source
- "Create a MetadataSource for a serdes export directory.
- The databases-dir for field resolution is export-dir/databases."
- [export-dir]
- (let [databases-dir (io/file export-dir "databases")
- {:keys [index db-name->dir]} (build-file-index export-dir)
- db-files (:database index)
- assets-index (dissoc index :database)]
- (->SerdesSource databases-dir assets-index db-name->dir
- (init-schema-model databases-dir db-name->dir db-files))))
+ Two levels of laziness:
+ 1. Schema → tables: populated when the schema is first accessed
+ 2. Table → fields: populated when that table's fields are first accessed"
+ [^File databases-dir db-name->dir db-file-index]
+ (atom
+ (into {}
+ (for [[db-name _dir-name] db-name->dir]
+ [db-name {:db-file (get db-file-index db-name)
+ :schemas (into {}
+ (for [schema (list-schemas databases-dir db-name->dir db-name)]
+ [schema ::not-indexed]))}]))))
(defn make-database-source
"Create a MetadataSource for a directory that IS the databases directory.
@@ -553,20 +371,5 @@
(let [{:keys [index db-name->dir]} (build-database-dir-index databases-dir)
databases-dir (io/file databases-dir)
db-files (:database index)]
- (->SerdesSource databases-dir {} db-name->dir
- (init-schema-model databases-dir db-name->dir db-files))))
-
-(defn source-index
- "Get the assets index from a SerdesSource. Returns a flat map of
- {[kind ref] file-path} for cards, dashboards, segments, etc."
- [^SerdesSource source]
- (.-assets-index source))
-
-;;; ===========================================================================
-;;; Enumeration
-;;; ===========================================================================
-
-(defn all-card-ids
- "Get all card entity-ids from source."
- [^SerdesSource source]
- (keys (:card (.-assets-index source))))
+ (->SerdesSchemaSource databases-dir db-name->dir
+ (init-schema-model databases-dir db-name->dir db-files))))
diff --git a/enterprise/backend/src/metabase_enterprise/checker/semantic.clj b/enterprise/backend/src/metabase_enterprise/checker/semantic.clj
index 8a9c656aac9..1d905ae2e5c 100644
--- a/enterprise/backend/src/metabase_enterprise/checker/semantic.clj
+++ b/enterprise/backend/src/metabase_enterprise/checker/semantic.clj
@@ -11,7 +11,8 @@
- `(setup export-dir schema-dir)` + `(check-one ctx entity-id)` — REPL workflow"
(:require
[clojure.string :as str]
- [metabase-enterprise.checker.format.serdes :as serdes]
+ [metabase-enterprise.checker.format.serdes-assets :as serdes-assets]
+ [metabase-enterprise.checker.format.serdes-schema :as serdes.schema]
[metabase-enterprise.checker.provider :as provider]
[metabase-enterprise.checker.store :as store]
[metabase-enterprise.dependencies.analysis :as deps.analysis]
@@ -717,9 +718,9 @@
"Build schema and assets sources from `schema-dir` and `export-dir`,
and a merged assets index (cards, dashboards, segments, etc.)."
[export-dir schema-dir]
- (let [schema-source (serdes/make-database-source schema-dir)
- assets-source (serdes/make-source export-dir)
- index (serdes/source-index assets-source)]
+ (let [schema-source (serdes.schema/make-database-source schema-dir)
+ assets-source (serdes-assets/make-source export-dir)
+ index (serdes-assets/source-index assets-source)]
{:schema-source schema-source :assets-source assets-source :index index}))
(defn check
@@ -770,7 +771,13 @@
(comment
;; REPL workflow:
(check "/Users/dan/projects/work/stats-remote-sync"
- "/tmp/metadata/metadata/databases")
+ "/Users/dan/projects/work/exports-root/metadata/metadata/databases")
+
+ (->> (check "/Users/dan/projects/work/representations/examples/v1"
+ "/Users/dan/projects/work/yaml-checked-files-v1/exports/sqlite-based/databases")
+ (into {} (filter (fn [[_id stuff]]
+ (letfn [(bad [x] ((some-fn :bad-refs :unresolved :native-errors) x))]
+ (bad stuff))))))
(setup
"/Users/dan/projects/work/stats-remote-sync"
"/Users/dan/projects/work/stats-remote-sync/databases")
diff --git a/enterprise/backend/test/metabase_enterprise/checker/format/serdes_assets_test.clj b/enterprise/backend/test/metabase_enterprise/checker/format/serdes_assets_test.clj
new file mode 100644
index 00000000000..55fc8b3e21c
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/checker/format/serdes_assets_test.clj
@@ -0,0 +1,178 @@
+(ns metabase-enterprise.checker.format.serdes-assets-test
+ "Tests for the serdes assets format module — YAML extraction, file indexing, and assets source resolution."
+ (:require
+ [clojure.java.io :as io]
+ [clojure.test :refer [deftest is testing]]
+ [metabase-enterprise.checker.format.serdes-assets :as serdes]
+ [metabase-enterprise.checker.source :as source]
+ [metabase.util.yaml :as yaml]))
+
+(set! *warn-on-reflection* true)
+
+;;; ===========================================================================
+;;; Helpers
+;;; ===========================================================================
+
+(def ^:private fixtures-dir "test_resources/yaml_checks")
+
+(defn- fixtures-path []
+ (let [f (io/file fixtures-dir)]
+ (if (.isAbsolute f)
+ fixtures-dir
+ (.getPath (io/file (System/getProperty "user.dir") fixtures-dir)))))
+
+(defn- with-temp-dir
+ "Create a temp directory, call (f dir-path), then clean up."
+ [f]
+ (let [dir (java.io.File/createTempFile "serdes-test" "")]
+ (.delete dir)
+ (.mkdirs dir)
+ (try
+ (f (.getPath dir))
+ (finally
+ (doseq [fl (reverse (file-seq dir))]
+ (.delete ^java.io.File fl))))))
+
+(defn- write-yaml!
+ "Write a YAML file, creating parent directories as needed."
+ [dir & path-and-data]
+ (let [data (last path-and-data)
+ path-segs (butlast path-and-data)
+ ^java.io.File file (apply io/file dir (map str path-segs))]
+ (.mkdirs (.getParentFile file))
+ (spit file (yaml/generate-string data))
+ (.getPath file)))
+
+;;; ===========================================================================
+;;; YAML extraction tests (pure, no files needed)
+;;; ===========================================================================
+
+(deftest extract-entity-id-from-fixture-test
+ (testing "extract-entity-id reads entity_id from a card YAML"
+ (let [card-file (io/file (fixtures-path) "collections" "cards" "simple-orders_simple_orders.yaml")]
+ (is (= "simple-orders" (serdes/extract-entity-id (.getPath card-file)))))))
+
+(deftest extract-model-from-fixture-test
+ (testing "extract-model reads the serdes model from a card YAML"
+ (let [card-file (io/file (fixtures-path) "collections" "cards" "simple-orders_simple_orders.yaml")]
+ (is (= "Card" (serdes/extract-model (.getPath card-file)))))))
+
+(deftest extract-entity-id-from-temp-file-test
+ (testing "extract-entity-id with various formats"
+ (with-temp-dir
+ (fn [dir]
+ (let [f (write-yaml! dir "test.yaml" {:name "X" :entity_id "xYz123AbC"})]
+ (is (= "xYz123AbC" (serdes/extract-entity-id f))))))))
+
+;;; ===========================================================================
+;;; Index tests — databases are indexed, tables/fields are on-demand
+;;; ===========================================================================
+
+(deftest build-file-index-databases-test
+ (testing "build-file-index indexes databases from fixtures"
+ (let [index (serdes/build-file-index (fixtures-path))]
+ (is (contains? (:database index) "Test Database"))
+ (is (contains? (:database index) "SQLite DB")))))
+
+(deftest build-file-index-cards-test
+ (testing "build-file-index indexes cards by entity_id"
+ (let [index (serdes/build-file-index (fixtures-path))]
+ (is (contains? (:card index) "simple-orders"))
+ (is (contains? (:card index) "native-orders"))
+ (is (contains? (:card index) "orders-with-products")))))
+
+(deftest build-file-index-no-duplicates-test
+ (testing "fixture index has no duplicates"
+ (let [index (serdes/build-file-index (fixtures-path))]
+ (is (nil? (:duplicates index))))))
+
+;;; ===========================================================================
+;;; AssetsSource protocol — resolution tests
+;;; ===========================================================================
+
+(deftest source-resolves-card-test
+ (testing "SerdesSource resolves cards by entity-id"
+ (let [source (serdes/make-source (fixtures-path))
+ card (source/resolve-card source "simple-orders")]
+ (is (some? card))
+ (is (= "Simple Orders" (:name card))))))
+
+(deftest source-returns-nil-for-unknown-test
+ (testing "SerdesSource returns nil for unknown entities"
+ (let [source (serdes/make-source (fixtures-path))]
+ (is (nil? (source/resolve-card source "no-such-card"))))))
+
+;;; ===========================================================================
+;;; Segments and measures — indexed from export dir
+;;; ===========================================================================
+
+(deftest export-dir-indexes-segments-and-measures-test
+ (testing "build-file-index (export dir) finds segments and measures under databases/"
+ (let [index (serdes/build-file-index (fixtures-path))]
+ (is (contains? (:segment index) "big-orders-segment")
+ "segment from with-schema table should be indexed")
+ (is (contains? (:segment index) "recent-orders-segment")
+ "segment from schema-less table should be indexed")
+ (is (contains? (:measure index) "total-revenue-measure")
+ "measure should be indexed"))))
+
+(deftest segments-resolve-from-export-source-test
+ (testing "segments can be resolved from the export source"
+ (let [source (serdes/make-source (fixtures-path))]
+ (let [seg (source/resolve-segment source "big-orders-segment")]
+ (is (some? seg))
+ (is (= "Big Orders" (:name seg))))
+ (let [seg (source/resolve-segment source "recent-orders-segment")]
+ (is (some? seg))
+ (is (= "Recent Orders" (:name seg)))))))
+
+(deftest measures-resolve-from-export-source-test
+ (testing "measures can be resolved from the export source"
+ (let [source (serdes/make-source (fixtures-path))
+ m (source/resolve-measure source "total-revenue-measure")]
+ (is (some? m))
+ (is (= "Total Revenue" (:name m))))))
+
+;;; ===========================================================================
+;;; Temp file edge cases
+;;; ===========================================================================
+
+(deftest build-file-index-empty-dir-test
+ (testing "build-file-index on empty directory returns empty index"
+ (with-temp-dir
+ (fn [dir]
+ (let [index (serdes/build-file-index dir)]
+ (is (nil? (:database index)))
+ (is (nil? (:card index))))))))
+
+(def ^:private card-yaml-template
+ "name: %s\nentity_id: %s\nserdes/meta:\n- id: %s\n model: Card\n")
+
+(deftest build-file-index-detects-duplicates-test
+ (testing "build-file-index reports duplicate entity_ids"
+ (with-temp-dir
+ (fn [dir]
+ ;; Create two cards with the same entity_id in different directories
+ (let [^java.io.File f1 (io/file dir "collections" "a" "cards" "dup.yaml")
+ ^java.io.File f2 (io/file dir "collections" "b" "cards" "dup.yaml")]
+ (.mkdirs (.getParentFile f1))
+ (.mkdirs (.getParentFile f2))
+ (spit f1 (format card-yaml-template "Card A" "dup-id" "dup-id"))
+ (spit f2 (format card-yaml-template "Card B" "dup-id" "dup-id"))
+ (let [index (serdes/build-file-index dir)]
+ (is (seq (:duplicates index)) "Should detect duplicate entity_ids")
+ (is (= "dup-id" (:ref (first (:duplicates index)))))))))))
+
+(deftest source-index-accessor-test
+ (testing "source-index returns the assets index (no databases — those are in the schema source)"
+ (let [source (serdes/make-source (fixtures-path))
+ index (serdes/source-index source)]
+ (is (map? index))
+ (is (not (contains? index :database)) "databases are in the schema source, not the assets index")
+ (is (contains? index :card)))))
+
+(deftest all-card-ids-test
+ (testing "all-card-ids returns all card entity-ids"
+ (let [source (serdes/make-source (fixtures-path))]
+ (is (>= (count (serdes/all-card-ids source)) 5))
+ (is (contains? (set (serdes/all-card-ids source)) "simple-orders")))))
diff --git a/enterprise/backend/test/metabase_enterprise/checker/format/serdes_test.clj b/enterprise/backend/test/metabase_enterprise/checker/format/serdes_schema_test.clj
similarity index 57%
rename from enterprise/backend/test/metabase_enterprise/checker/format/serdes_test.clj
rename to enterprise/backend/test/metabase_enterprise/checker/format/serdes_schema_test.clj
index 9ff088efa38..a8ec4eb9033 100644
--- a/enterprise/backend/test/metabase_enterprise/checker/format/serdes_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/checker/format/serdes_schema_test.clj
@@ -1,9 +1,9 @@
-(ns metabase-enterprise.checker.format.serdes-test
- "Tests for the serdes format module — YAML extraction, file indexing, and source resolution."
+(ns metabase-enterprise.checker.format.serdes-schema-test
+ "Tests for the serdes schema format module — database/table/field resolution and lazy indexing."
(:require
[clojure.java.io :as io]
- [clojure.test :refer [deftest is testing]]
- [metabase-enterprise.checker.format.serdes :as serdes]
+ [clojure.test :refer :all]
+ [metabase-enterprise.checker.format.serdes-schema :as serdes-schema]
[metabase-enterprise.checker.source :as source]
[metabase.util.yaml :as yaml]))
@@ -43,53 +43,10 @@
(spit file (yaml/generate-string data))
(.getPath file)))
-;;; ===========================================================================
-;;; YAML extraction tests (pure, no files needed)
-;;; ===========================================================================
-
-(deftest extract-entity-id-from-fixture-test
- (testing "extract-entity-id reads entity_id from a card YAML"
- (let [card-file (io/file (fixtures-path) "collections" "cards" "simple-orders_simple_orders.yaml")]
- (is (= "simple-orders" (serdes/extract-entity-id (.getPath card-file)))))))
-
-(deftest extract-model-from-fixture-test
- (testing "extract-model reads the serdes model from a card YAML"
- (let [card-file (io/file (fixtures-path) "collections" "cards" "simple-orders_simple_orders.yaml")]
- (is (= "Card" (serdes/extract-model (.getPath card-file)))))))
-
-(deftest extract-entity-id-from-temp-file-test
- (testing "extract-entity-id with various formats"
- (with-temp-dir
- (fn [dir]
- (let [f (write-yaml! dir "test.yaml" {:name "X" :entity_id "xYz123AbC"})]
- (is (= "xYz123AbC" (serdes/extract-entity-id f))))))))
-
-;;; ===========================================================================
-;;; Index tests — databases are indexed, tables/fields are on-demand
-;;; ===========================================================================
-
-(deftest build-file-index-databases-test
- (testing "build-file-index indexes databases from fixtures"
- (let [{:keys [index]} (serdes/build-file-index (fixtures-path))]
- (is (contains? (:database index) "Test Database"))
- (is (contains? (:database index) "SQLite DB")))))
-
-(deftest build-file-index-cards-test
- (testing "build-file-index indexes cards by entity_id"
- (let [{:keys [index]} (serdes/build-file-index (fixtures-path))]
- (is (contains? (:card index) "simple-orders"))
- (is (contains? (:card index) "native-orders"))
- (is (contains? (:card index) "orders-with-products")))))
-
-(deftest build-file-index-no-duplicates-test
- (testing "fixture index has no duplicates"
- (let [{:keys [index]} (serdes/build-file-index (fixtures-path))]
- (is (nil? (:duplicates index))))))
-
(deftest build-database-dir-index-test
(testing "build-database-dir-index indexes databases and provides db-name->dir mapping"
(let [db-dir (str (fixtures-path) "/databases")
- {:keys [index db-name->dir]} (serdes/build-database-dir-index db-dir)]
+ {:keys [index db-name->dir]} (serdes-schema/build-database-dir-index db-dir)]
(is (contains? (:database index) "Test Database"))
(is (contains? (:database index) "SQLite DB"))
(is (contains? db-name->dir "Test Database"))
@@ -110,7 +67,7 @@
;; Create a database with a slugified directory name but real name in YAML
(write-yaml! dir "databases" "my_fancy_db" "my_fancy_db.yaml"
{:name "My Fancy DB" :engine "h2"})
- (let [{:keys [db-name->dir]} (serdes/build-database-dir-index (str dir "/databases"))]
+ (let [{:keys [db-name->dir]} (serdes-schema/build-database-dir-index (str dir "/databases"))]
(is (= "my_fancy_db" (get db-name->dir "My Fancy DB")))))))
(testing "db-name->dir includes ALL databases, not just some"
@@ -120,7 +77,7 @@
(let [slug (str "db_" i)]
(write-yaml! dir "databases" slug (str slug ".yaml")
{:name (str "Database " i) :engine "h2"})))
- (let [{:keys [db-name->dir]} (serdes/build-database-dir-index (str dir "/databases"))]
+ (let [{:keys [db-name->dir]} (serdes-schema/build-database-dir-index (str dir "/databases"))]
(is (= 10 (count db-name->dir)))
(doseq [i (range 10)]
(is (= (str "db_" i) (get db-name->dir (str "Database " i))))))))))
@@ -130,50 +87,42 @@
;;; ===========================================================================
(deftest source-resolves-database-test
- (testing "SerdesSource resolves databases by name"
- (let [source (serdes/make-source (fixtures-path))
+ (testing "SerdesSchemaSource resolves databases by name"
+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))
db (source/resolve-database source "Test Database")]
(is (some? db))
(is (= "Test Database" (:name db))))))
(deftest source-resolves-table-on-demand-test
- (testing "SerdesSource resolves tables by path on demand (no pre-indexing)"
- (let [source (serdes/make-source (fixtures-path))
+ (testing "SerdesSchemaSource resolves tables by path on demand (no pre-indexing)"
+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))
table (source/resolve-table source ["Test Database" "public" "orders"])]
(is (some? table))
(is (= "orders" (:name table))))
(testing "schema-less tables"
- (let [source (serdes/make-source (fixtures-path))
+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))
table (source/resolve-table source ["SQLite DB" nil "orders"])]
(is (some? table))
(is (= "orders" (:name table)))))))
(deftest source-resolves-field-on-demand-test
- (testing "SerdesSource resolves fields by path on demand"
- (let [source (serdes/make-source (fixtures-path))
+ (testing "SerdesSchemaSource resolves fields by path on demand"
+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))
field (source/resolve-field source ["Test Database" "public" "orders" "id"])]
(is (some? field))
(is (= "id" (:name field))))
(testing "schema-less fields"
- (let [source (serdes/make-source (fixtures-path))
+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))
field (source/resolve-field source ["SQLite DB" nil "orders" "id"])]
(is (some? field))
(is (= "id" (:name field)))))))
-(deftest source-resolves-card-test
- (testing "SerdesSource resolves cards by entity-id"
- (let [source (serdes/make-source (fixtures-path))
- card (source/resolve-card source "simple-orders")]
- (is (some? card))
- (is (= "Simple Orders" (:name card))))))
-
(deftest source-returns-nil-for-unknown-test
- (testing "SerdesSource returns nil for unknown entities"
- (let [source (serdes/make-source (fixtures-path))]
+ (testing "SerdesSchemaSource returns nil for unknown entities"
+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))]
(is (nil? (source/resolve-database source "Nonexistent")))
(is (nil? (source/resolve-table source ["DB" "x" "y"])))
- (is (nil? (source/resolve-field source ["DB" "x" "y" "z"])))
- (is (nil? (source/resolve-card source "no-such-card"))))))
+ (is (nil? (source/resolve-field source ["DB" "x" "y" "z"]))))))
;;; ===========================================================================
;;; fields-for-table and all-table-paths — on-demand enumeration
@@ -181,23 +130,23 @@
(deftest fields-for-table-test
(testing "fields-for-table returns field paths for a table"
- (let [source (serdes/make-source (fixtures-path))
+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))
fields (source/fields-for-table source ["Test Database" "public" "orders"])]
(is (set? fields))
(is (contains? fields ["Test Database" "public" "orders" "id"]))
(is (contains? fields ["Test Database" "public" "orders" "total"]))))
(testing "fields-for-table works for schema-less tables"
- (let [source (serdes/make-source (fixtures-path))
+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))
fields (source/fields-for-table source ["SQLite DB" nil "orders"])]
(is (set? fields))
(is (contains? fields ["SQLite DB" nil "orders" "id"]))))
(testing "fields-for-table returns nil for unknown table"
- (let [source (serdes/make-source (fixtures-path))]
+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))]
(is (nil? (source/fields-for-table source ["Nope" "x" "y"]))))))
(deftest all-table-paths-test
(testing "all-table-paths enumerates all tables across databases"
- (let [source (serdes/make-source (fixtures-path))
+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))
tables (source/all-table-paths source)]
(is (seq tables))
(is (some #(= ["Test Database" "public" "orders"] %) tables))
@@ -205,7 +154,7 @@
(deftest all-database-names-test
(testing "all-database-names returns database names"
- (let [source (serdes/make-source (fixtures-path))]
+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))]
(is (= #{"Test Database" "SQLite DB"} (set (source/all-database-names source)))))))
;;; ===========================================================================
@@ -233,7 +182,7 @@
{:id "public" :model "Schema"}
{:id "orders" :model "Table"}
{:id "id" :model "Field"}]})
- (let [source (serdes/make-database-source db-dir)]
+ (let [source (serdes-schema/make-database-source db-dir)]
;; Database resolves by real name
(is (some? (source/resolve-database source "Analytics Data Warehouse")))
;; Table resolves using real db name
@@ -253,7 +202,7 @@
(deftest index-does-not-contain-tables-or-fields-test
(testing "the file index only contains databases — tables and fields are resolved on demand"
(let [db-dir (str (fixtures-path) "/databases")
- {:keys [index]} (serdes/build-database-dir-index db-dir)]
+ {:keys [index]} (serdes-schema/build-database-dir-index db-dir)]
(is (seq (:database index)) "databases should be in the index")
(is (nil? (:table index)) "tables should NOT be in the index")
(is (nil? (:field index)) "fields should NOT be in the index"))))
@@ -262,7 +211,7 @@
;;; Cache shape and lazy resolution
;;; ===========================================================================
-(def ^:private not-indexed :metabase-enterprise.checker.format.serdes/not-indexed)
+(def ^:private not-indexed ::serdes-schema/not-indexed)
(deftest schema-model-shape-test
(testing "schema model starts with ::not-indexed for each schema, populates lazily"
@@ -277,24 +226,24 @@
{:name "T2" :schema "s2"})
(write-yaml! dir "databases" "db" "schemas" "s2" "tables" "t2" "fields" "f2.yaml"
{:name "F2" :base_type "type/Text"})
- (let [source (serdes/make-database-source (str dir "/databases"))]
+ (let [source (serdes-schema/make-database-source (str dir "/databases"))]
;; Initial: db-file set, all schemas ::not-indexed
(is (=? {"DB" {:db-file string?
:schemas {"s1" not-indexed
"s2" not-indexed}}}
- (serdes/schema-model source)))
+ (serdes-schema/schema-model source)))
;; After resolving a table in s1: s1 indexed with tables, s2 untouched
(source/resolve-table source ["DB" "s1" "T1"])
(is (=? {"DB" {:schemas {"s1" {"T1" {:table-file string?
:fields not-indexed}}
"s2" not-indexed}}}
- (serdes/schema-model source)))
+ (serdes-schema/schema-model source)))
;; After resolving fields for T1: T1 fields indexed, s2 still untouched
(source/fields-for-table source ["DB" "s1" "T1"])
(is (=? {"DB" {:schemas {"s1" {"T1" {:table-file string?
:fields {"F1" string?}}}
"s2" not-indexed}}}
- (serdes/schema-model source))))))))
+ (serdes-schema/schema-model source))))))))
(deftest case-insensitive-schema-resolution-test
(testing "schemas with different casing on disk vs YAML resolve correctly"
@@ -306,91 +255,9 @@
{:name "ORDERS" :schema "PUBLIC"})
(write-yaml! dir "databases" "db" "schemas" "public" "tables" "orders" "fields" "id.yaml"
{:name "ID" :base_type "type/Integer"})
- (let [source (serdes/make-database-source (str dir "/databases"))]
+ (let [source (serdes-schema/make-database-source (str dir "/databases"))]
;; Resolve by real names (uppercase) — schema re-keys from "public" to "PUBLIC"
(is (some? (source/resolve-table source ["DB" "PUBLIC" "ORDERS"])))
(is (some? (source/resolve-field source ["DB" "PUBLIC" "ORDERS" "ID"])))
(is (=? {"DB" {:schemas {"PUBLIC" {"ORDERS" {:fields {"ID" string?}}}}}}
- (serdes/schema-model source))))))))
-
-;;; ===========================================================================
-;;; Segments and measures — indexed from export dir, NOT schema dir
-;;; ===========================================================================
-
-(deftest export-dir-indexes-segments-and-measures-test
- (testing "build-file-index (export dir) finds segments and measures under databases/"
- (let [{:keys [index]} (serdes/build-file-index (fixtures-path))]
- (is (contains? (:segment index) "big-orders-segment")
- "segment from with-schema table should be indexed")
- (is (contains? (:segment index) "recent-orders-segment")
- "segment from schema-less table should be indexed")
- (is (contains? (:measure index) "total-revenue-measure")
- "measure should be indexed"))))
-
-(deftest schema-dir-does-not-index-segments-test
- (testing "build-database-dir-index (schema dir) does NOT index segments or measures"
- (let [db-dir (str (fixtures-path) "/databases")
- {:keys [index]} (serdes/build-database-dir-index db-dir)]
- (is (nil? (:segment index)) "schema dir should not contain segments")
- (is (nil? (:measure index)) "schema dir should not contain measures"))))
-
-(deftest segments-resolve-from-export-source-test
- (testing "segments can be resolved from the export source"
- (let [source (serdes/make-source (fixtures-path))]
- (let [seg (source/resolve-segment source "big-orders-segment")]
- (is (some? seg))
- (is (= "Big Orders" (:name seg))))
- (let [seg (source/resolve-segment source "recent-orders-segment")]
- (is (some? seg))
- (is (= "Recent Orders" (:name seg)))))))
-
-(deftest measures-resolve-from-export-source-test
- (testing "measures can be resolved from the export source"
- (let [source (serdes/make-source (fixtures-path))
- m (source/resolve-measure source "total-revenue-measure")]
- (is (some? m))
- (is (= "Total Revenue" (:name m))))))
-
-;;; ===========================================================================
-;;; Temp file edge cases
-;;; ===========================================================================
-
-(deftest build-file-index-empty-dir-test
- (testing "build-file-index on empty directory returns empty index"
- (with-temp-dir
- (fn [dir]
- (let [{:keys [index]} (serdes/build-file-index dir)]
- (is (nil? (:database index)))
- (is (nil? (:card index))))))))
-
-(def ^:private card-yaml-template
- "name: %s\nentity_id: %s\nserdes/meta:\n- id: %s\n model: Card\n")
-
-(deftest build-file-index-detects-duplicates-test
- (testing "build-file-index reports duplicate entity_ids"
- (with-temp-dir
- (fn [dir]
- ;; Create two cards with the same entity_id in different directories
- (let [^java.io.File f1 (io/file dir "collections" "a" "cards" "dup.yaml")
- ^java.io.File f2 (io/file dir "collections" "b" "cards" "dup.yaml")]
- (.mkdirs (.getParentFile f1))
- (.mkdirs (.getParentFile f2))
- (spit f1 (format card-yaml-template "Card A" "dup-id" "dup-id"))
- (spit f2 (format card-yaml-template "Card B" "dup-id" "dup-id"))
- (let [{:keys [index]} (serdes/build-file-index dir)]
- (is (seq (:duplicates index)) "Should detect duplicate entity_ids")
- (is (= "dup-id" (:ref (first (:duplicates index)))))))))))
-
-(deftest source-index-accessor-test
- (testing "source-index returns the assets index (no databases — those are in the schema model)"
- (let [source (serdes/make-source (fixtures-path))
- index (serdes/source-index source)]
- (is (map? index))
- (is (not (contains? index :database)) "databases are in the schema model, not the assets index")
- (is (contains? index :card)))))
-
-(deftest all-card-ids-test
- (testing "all-card-ids returns all card entity-ids"
- (let [source (serdes/make-source (fixtures-path))]
- (is (>= (count (serdes/all-card-ids source)) 5))
- (is (contains? (set (serdes/all-card-ids source)) "simple-orders")))))
+ (serdes-schema/schema-model source))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/checker/semantic_test.clj b/enterprise/backend/test/metabase_enterprise/checker/semantic_test.clj
index 68064b532a1..7f3c7775c1e 100644
--- a/enterprise/backend/test/metabase_enterprise/checker/semantic_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/checker/semantic_test.clj
@@ -6,7 +6,8 @@
(:require
[clojure.java.io :as io]
[clojure.test :refer [deftest is testing]]
- [metabase-enterprise.checker.format.serdes :as serdes-format]
+ [metabase-enterprise.checker.format.serdes-assets :as serdes-format]
+ [metabase-enterprise.checker.format.serdes-schema :as serdes-schema]
[metabase-enterprise.checker.provider :as provider]
[metabase-enterprise.checker.semantic :as checker]
[metabase-enterprise.checker.source :as source]
@@ -30,14 +31,18 @@
test-fixtures-dir
(.getPath (io/file (System/getProperty "user.dir") test-fixtures-dir)))))
-(defn- make-test-source []
- (serdes-format/make-source (fixtures-path)))
+(defn- make-test-sources
+ "Create both schema and assets sources from fixtures.
+ Returns [schema-source assets-source]."
+ []
+ [(serdes-schema/make-database-source (str (fixtures-path) "/databases"))
+ (serdes-format/make-source (fixtures-path))])
-(defn- check-all [source]
- (checker/check-entities source source (serdes-format/source-index source)))
+(defn- check-all [[schema-source assets-source]]
+ (checker/check-entities schema-source assets-source (serdes-format/source-index assets-source)))
-(defn- check-specific [source entity-ids]
- (checker/check-entities source source (serdes-format/source-index source) entity-ids))
+(defn- check-specific [[schema-source assets-source] entity-ids]
+ (checker/check-entities schema-source assets-source (serdes-format/source-index assets-source) entity-ids))
;;; ===========================================================================
;;; Tests Using Real YAML Files
@@ -45,20 +50,20 @@
(deftest source-index-test
(testing "Source correctly indexes databases and cards; tables/fields are on-demand"
- (let [source (make-test-source)
- index (serdes-format/source-index source)]
- ;; Databases are in the schema model, not the assets index
- (is (= #{"Test Database" "SQLite DB"} (set (source/all-database-names source))))
+ (let [[schema-source assets-source] (make-test-sources)
+ index (serdes-format/source-index assets-source)]
+ ;; Databases are in the schema source
+ (is (= #{"Test Database" "SQLite DB"} (set (source/all-database-names schema-source))))
;; Cards are in the assets index
(is (= 5 (count (:card index))) "Should have 5 cards")
- ;; Tables and fields are resolved on demand
- (is (= 4 (count (source/all-table-paths source))) "Should have 4 tables via source")
- (is (= 11 (count (source/all-field-paths source))) "Should have 11 fields via source"))))
+ ;; Tables and fields are resolved on demand via schema source
+ (is (= 4 (count (source/all-table-paths schema-source))) "Should have 4 tables via source")
+ (is (= 11 (count (source/all-field-paths schema-source))) "Should have 11 fields via source"))))
(deftest simple-mbql-query-test
(testing "Simple MBQL query on orders table validates successfully"
- (let [source (make-test-source)
- results (check-all source)
+ (let [sources (make-test-sources)
+ results (check-all sources)
result (get results "simple-orders")]
(is (some? result) "Card should be found")
(is (= "Simple Orders" (:name result)) "Card name should match")
@@ -70,8 +75,8 @@
(deftest native-query-test
(testing "Native SQL query validates successfully"
- (let [source (make-test-source)
- results (check-all source)
+ (let [sources (make-test-sources)
+ results (check-all sources)
result (get results "native-orders")]
(is (some? result) "Card should be found")
(is (= "Native Orders" (:name result)) "Card name should match")
@@ -80,8 +85,8 @@
(deftest mbql-with-joins-test
(testing "MBQL query with joins validates successfully"
- (let [source (make-test-source)
- results (check-all source)
+ (let [sources (make-test-sources)
+ results (check-all sources)
result (get results "orders-with-products")]
(is (some? result) "Card should be found")
(is (= "Orders With Products" (:name result)) "Card name should match")
@@ -94,8 +99,8 @@
(deftest all-cards-checked-test
(testing "All cards in fixtures are checked"
- (let [source (make-test-source)
- results (check-all source)]
+ (let [sources (make-test-sources)
+ results (check-all sources)]
(is (>= (count results) 5) "Should check at least 5 cards")
(is (contains? results "simple-orders"))
(is (contains? results "native-orders"))
@@ -112,14 +117,14 @@
(deftest schemaless-database-index-test
(testing "Schema-less databases are indexed correctly with nil schema"
- (let [source (make-test-source)]
- (is (= #{"Test Database" "SQLite DB"} (set (source/all-database-names source))))
- (is (= 4 (count (source/all-table-paths source))) "Should have 4 tables via source"))))
+ (let [[schema-source _] (make-test-sources)]
+ (is (= #{"Test Database" "SQLite DB"} (set (source/all-database-names schema-source))))
+ (is (= 4 (count (source/all-table-paths schema-source))) "Should have 4 tables via source"))))
(deftest schemaless-query-test
(testing "Query on schema-less database validates successfully"
- (let [source (make-test-source)
- results (check-all source)
+ (let [sources (make-test-sources)
+ results (check-all sources)
result (get results "schemaless-query")]
(is (some? result) "Card should be found")
(is (= "Schemaless Query" (:name result)) "Card name should match")
@@ -136,9 +141,9 @@
(deftest fk-in-field-metadata-test
(testing "FK target in field metadata is resolved to integer ID"
- (let [source (make-test-source)
- index (serdes-format/source-index source)
- provider (provider/make-provider (store/make-store source source index))
+ (let [[schema-source assets-source] (make-test-sources)
+ index (serdes-format/source-index assets-source)
+ provider (provider/make-provider (store/make-store schema-source assets-source index))
;; Get the product_id field which has FK to products.id
fields (lib.metadata.protocols/metadatas
provider {:lib/type :metadata/column})
@@ -150,8 +155,8 @@
(deftest fk-in-result-metadata-test
(testing "FK target in card result_metadata is resolved to integer ID"
- (let [source (make-test-source)
- results (check-all source)
+ (let [sources (make-test-sources)
+ results (check-all sources)
result (get results "orders-with-fk")]
(is (some? result) "Card should be found")
(is (= "Orders With FK" (:name result)) "Card name should match")
@@ -242,8 +247,8 @@
(deftest check-specific-cards-test
(testing "Can check specific cards by entity-id"
- (let [source (make-test-source)
- results (check-specific source ["simple-orders" "native-orders"])]
+ (let [sources (make-test-sources)
+ results (check-specific sources ["simple-orders" "native-orders"])]
(is (>= (count results) 2) "Should check at least 2 cards")
(is (contains? results "simple-orders"))
(is (contains? results "native-orders"))
@@ -1211,5 +1216,5 @@
(fixtures-path)
;; Manually check all cards in fixtures
- (def source (make-test-source))
- (check-all source))
+ (def sources (make-test-sources))
+ (check-all sources))