The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Split source (#72666)
dpsutton113ee31093db
diff --git a/enterprise/backend/src/metabase_enterprise/checker/format/serdes_assets.clj b/enterprise/backend/src/metabase_enterprise/checker/format/serdes_assets.cljnew file mode 100644index 00000000000..75401d6d07a--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/checker/format/serdes_assets.clj@@ -0,0 +1,304 @@+(ns metabase-enterprise.checker.format.serdes-assets+ "Serdes export format — directory structure, path parsing, entity loading.++ Directory structure (on disk, names may be slugified):+ databases/<db-dir>/<db-dir>.yaml+ databases/<db-dir>/schemas/<schema-dir>/tables/<table-dir>/<table-dir>.yaml+ databases/<db-dir>/schemas/<schema-dir>/tables/<table-dir>/fields/<field>.yaml+ databases/<db-dir>/tables/<table-dir>/<table-dir>.yaml # schema-less+ databases/<db-dir>/tables/<table-dir>/fields/<field>.yaml # schema-less+ collections/<entity-id>_<slug>/<entity-id>_<slug>.yaml+ collections/.../cards/<entity-id>_<slug>.yaml+ collections/.../dashboards/<entity-id>_<slug>.yaml++ Directory names may differ from entity names (e.g. `analytics_data_warehouse`+ for \"Analytics Data Warehouse\", or `orders` for \"ORDERS\" in H2).++ ## Lazy resolution cache++ The `SerdesSource` maintains a single lazily-populated cache atom that maps+ real entity names (from YAML) to file locations:++ {db-name ; real name, e.g. \"Analytics Data Warehouse\"+ {schema ; real name or nil for schema-less+ ::not-indexed ; schema exists but tables not yet walked+ | {table-name ; real name from YAML+ {:table-file \"path/to/table.yaml\" ; path string to the table YAML+ :fields ::not-indexed ; fields not yet walked for this table+ | {field-name \"path\" ...}}}}} ; real name → field YAML path++ Two levels of laziness:+ 1. Schema → tables: realized when any table in that schema is resolved+ (via `resolve-table`, `resolve-field`, `fields-for-table`) or when+ `tables-for-database` is called. Walks the schema's `tables/` directory+ and reads each table's `name:` from YAML via fast regex.+ 2. Table → fields: realized when `resolve-field` or `fields-for-table` is+ called for that specific table. Walks the table's `fields/` directory+ and reads each field's `name:` via fast regex. Other tables in the same+ schema are unaffected.++ The `db-name->dir` map (built at startup from ~40 database YAMLs) handles the+ database name → directory name translation. Schema and table name → directory+ name translation is handled by case-insensitive directory matching during+ schema indexing."+ (:require+ [clojure.java.io :as io]+ [clojure.string :as str]+ [metabase-enterprise.checker.source :as source]+ [metabase.util.yaml :as yaml])+ (:import+ (java.io File)))++(set! *warn-on-reflection* true)++;;; ===========================================================================+;;; YAML Loading+;;; ===========================================================================++(defn load-yaml+ "Load and parse a YAML file."+ [path]+ (yaml/parse-string (slurp path)))++(defn- db-dir+ "Resolve a real database name to its directory on disk."+ ^File [^File databases-dir db-name->dir db-name]+ (let [dir-name (get db-name->dir db-name db-name)]+ (io/file databases-dir dir-name)))++(defn quick-extract+ "Extract a field from YAML using regex (fast) with full parse fallback."+ [file-path field-name pattern]+ (try+ (let [content (slurp file-path)]+ (if-let [[_ value] (re-find pattern content)]+ (str/trim value)+ (get (yaml/parse-string content) (keyword field-name))))+ (catch Exception _ nil)))++(defn- read-yaml-name+ "Read the `name:` field from a YAML file using regex (fast).+ Falls back to `fallback` if the file doesn't exist or can't be read."+ [^File yaml-file ^String fallback]+ (if (.exists yaml-file)+ (or (quick-extract (.getPath yaml-file) "name" #"(?m)^name:\s*(.+)")+ fallback)+ fallback))++(defn extract-entity-id+ "Extract the entity id from a yaml file without parsing it. Looking for a top level entity_id at start of line."+ [file-path]+ (quick-extract file-path "entity_id" #"(?m)^entity_id:\s*(\S+)"))++(defn extract-model+ "Extract the serdes model from a YAML file (e.g., \"Collection\", \"Card\", \"Dashboard\").+ Looks for the model field inside serdes/meta to avoid matching embedded refs."+ [file-path]+ (try+ (let [content (slurp file-path)]+ (when-let [[_ model] (re-find #"(?m)^serdes/meta:\s*\n\s*- id:.*\n\s+(?:label:.*\n\s+)?model:\s*(\S+)" content)]+ (str/trim model)))+ (catch Exception _ nil)))++(defn- index-segments-and-measures+ "Walk a table directory's segments/ and measures/ subdirectories.+ Reads entity_id from each YAML (only a handful exist).+ Returns a seq of {:kind :ref :file} entries."+ [^File table-dir]+ (into []+ (mapcat+ (fn [[subdir kind]]+ (let [^File d (io/file table-dir subdir)]+ (when (.isDirectory d)+ (keep (fn [^File f]+ (when (and (.isFile f) (str/ends-with? (.getName f) ".yaml"))+ (when-let [eid (extract-entity-id (.getPath f))]+ {:kind kind :ref eid :file (.getPath f)})))+ (.listFiles d))))))+ [["segments" :segment] ["measures" :measure]]))++(defn- list-table-dirs+ "Return a seq of table directory Files for a database.+ Walks schemas/<schema>/tables/<table>/ and tables/<table>/ directories."+ [^File databases-dir db-name->dir-map db-name]+ (let [^File db (db-dir databases-dir db-name->dir-map db-name)+ result (volatile! [])]+ (let [^File schemas-dir (io/file db "schemas")]+ (when (.isDirectory schemas-dir)+ (doseq [^File schema-d (.listFiles schemas-dir)+ :when (.isDirectory schema-d)]+ (let [^File tables-dir (io/file schema-d "tables")]+ (when (.isDirectory tables-dir)+ (doseq [^File td (.listFiles tables-dir)+ :when (.isDirectory td)]+ (vswap! result conj td)))))))+ (let [^File tables-dir (io/file db "tables")]+ (when (.isDirectory tables-dir)+ (doseq [^File td (.listFiles tables-dir)+ :when (.isDirectory td)]+ (vswap! result conj td))))+ @result))++(defn- index-schema-dir+ "Walk a databases directory and index databases, segments, and measures.+ Reads ~40 database YAMLs for real names, plus a handful of segment/measure YAMLs+ for entity_ids. Tables and fields are resolved on demand from the filesystem.++ When `include-segments-measures?` is true, also walks table directories+ to find segment and measure YAMLs (for export dirs). Set to false for+ schema-only dirs to avoid walking thousands of table directories.++ Returns {:entries [...]+ :db-name->dir {real-db-name dir-name}}."+ [schema-dir & {:keys [include-segments-measures?]}]+ (let [schema-dir (io/file schema-dir)+ db-name->dir (volatile! {})+ entries (volatile! [])]+ (doseq [^File db-dir (.listFiles schema-dir)+ :when (.isDirectory db-dir)]+ (let [dir-name (.getName db-dir)+ db-yaml (io/file db-dir (str dir-name ".yaml"))+ db-name (read-yaml-name db-yaml dir-name)]+ (vswap! db-name->dir assoc db-name dir-name)+ (vswap! entries conj {:kind :database :ref db-name+ :file (if (.exists db-yaml) (.getPath db-yaml) (.getPath db-dir))})+ ;; Walk table directories for segments and measures (export dirs only)+ (when include-segments-measures?+ (doseq [^File table-dir (list-table-dirs schema-dir @db-name->dir db-name)]+ (vswap! entries into (index-segments-and-measures table-dir))))))+ {:entries @entries+ :db-name->dir @db-name->dir}))++;;; ===========================================================================+;;; File Walking - Build index of all entities+;;; ===========================================================================++(def ^:private model->kind+ "Map serdes model names to index kinds."+ {"Card" :card+ "Dashboard" :dashboard+ "Collection" :collection+ "Document" :document+ "Measure" :measure+ "Segment" :segment+ "NativeQuerySnippet" :snippet+ "Transform" :transform})++(defn- index-collections-tree+ "Walk the collections/ directory and index all entities by their serdes model.+ Returns entries for cards, dashboards, and collections."+ [export-dir]+ (let [collections-dir (io/file export-dir "collections")]+ (when (.isDirectory collections-dir)+ (for [^File file (file-seq collections-dir)+ :when (.isFile file)+ :let [path (.getPath file)]+ :when (str/ends-with? (.getName file) ".yaml")+ :let [model (extract-model path)+ kind (get model->kind model)]+ :when kind+ :let [entity-id (extract-entity-id path)]+ :when entity-id]+ {:kind kind :ref entity-id :file path}))))++(defn build-file-index+ "Build index of all entity files in an export directory.+ Returns `{:index {kind {ref file-path}} :db-name->dir {real-name dir-name}}`+ where index also has `:duplicates` for refs appearing in multiple files."+ [export-dir]+ (let [databases-dir (io/file export-dir "databases")+ schema-result (when (.isDirectory databases-dir)+ (index-schema-dir databases-dir :include-segments-measures? true))+ entries (concat+ (:entries schema-result)+ (index-collections-tree export-dir))+ ;; Group by [kind ref] to find duplicates+ by-key (reduce (fn [m {:keys [kind ref file]}]+ (update m [kind ref] (fnil conj []) file))+ {}+ entries)+ ;; Build the index (first file wins) and collect duplicates+ index (reduce-kv (fn [idx [kind ref] files]+ (assoc-in idx [kind ref] (first files)))+ {}+ by-key)+ dupes (into []+ (keep (fn [[[kind ref] files]]+ (when (> (count files) 1)+ {:kind kind :ref ref :files files})))+ by-key)]+ (cond-> index+ (seq dupes) (assoc :duplicates dupes))))++(defn index-stats+ "Get statistics about a file index.+ Note: tables and fields are resolved on demand and won't appear in the index+ for schema-dir sources. They will be 0 for `build-database-dir-index` results."+ [index]+ {:databases (count (:database index))+ :tables (count (:table index))+ :fields (count (:field index))+ :measures (count (:measure index))+ :segments (count (:segment index))+ :cards (count (:card index))+ :dashboards (count (:dashboard index))+ :collections (count (:collection index))+ :database-names (vec (keys (:database index)))})++(deftype SerdesSource [databases-dir assets-index]+ source/AssetsSource+ (resolve-card [_ entity-id]+ (when-let [file (get-in assets-index [:card entity-id])]+ (load-yaml file)))++ (resolve-snippet [_ entity-id]+ (when-let [file (get-in assets-index [:snippet entity-id])]+ (load-yaml file)))++ (resolve-transform [_ entity-id]+ (when-let [file (get-in assets-index [:transform entity-id])]+ (load-yaml file)))++ (resolve-segment [_ entity-id]+ (when-let [file (get-in assets-index [:segment entity-id])]+ (load-yaml file)))++ (resolve-dashboard [_ entity-id]+ (when-let [file (get-in assets-index [:dashboard entity-id])]+ (load-yaml file)))++ (resolve-collection [_ entity-id]+ (when-let [file (get-in assets-index [:collection entity-id])]+ (load-yaml file)))++ (resolve-document [_ entity-id]+ (when-let [file (get-in assets-index [:document entity-id])]+ (load-yaml file)))++ (resolve-measure [_ entity-id]+ (when-let [file (get-in assets-index [:measure entity-id])]+ (load-yaml file))))++(defn make-source+ "Create a MetadataSource for a serdes export directory.+ The databases-dir for field resolution is export-dir/databases."+ [export-dir]+ (let [databases-dir (io/file export-dir "databases")+ index (build-file-index export-dir)+ assets-index (dissoc index :database)]+ (->SerdesSource databases-dir assets-index)))++(defn source-index+ "Get the assets index from a SerdesSource. Returns a flat map of+ {[kind ref] file-path} for cards, dashboards, segments, etc."+ [^SerdesSource source]+ (.-assets-index source))++;;; ===========================================================================+;;; Enumeration+;;; ===========================================================================++(defn all-card-ids+ "Get all card entity-ids from source."+ [^SerdesSource source]+ (keys (:card (.-assets-index source))))diff --git a/enterprise/backend/src/metabase_enterprise/checker/format/serdes.clj b/enterprise/backend/src/metabase_enterprise/checker/format/serdes_schema.cljsimilarity index 65%rename from enterprise/backend/src/metabase_enterprise/checker/format/serdes.cljrename to enterprise/backend/src/metabase_enterprise/checker/format/serdes_schema.cljindex 0845a829640..d3d7d5652d2 100644--- a/enterprise/backend/src/metabase_enterprise/checker/format/serdes.clj+++ b/enterprise/backend/src/metabase_enterprise/checker/format/serdes_schema.clj@@ -1,4 +1,4 @@-(ns metabase-enterprise.checker.format.serdes+(ns metabase-enterprise.checker.format.serdes-schema"Serdes export format — directory structure, path parsing, entity loading.Directory structure (on disk, names may be slugified):@@ -53,15 +53,47 @@(set! *warn-on-reflection* true)-;;; ===========================================================================-;;; YAML Loading-;;; ===========================================================================-(defn load-yaml"Load and parse a YAML file."[path](yaml/parse-string (slurp path)))+(defn- db-dir+ "Resolve a real database name to its directory on disk."+ ^File [^File databases-dir db-name->dir db-name]+ (let [dir-name (get db-name->dir db-name db-name)]+ (io/file databases-dir dir-name)))++;;; ===========================================================================+;;; MetadataSource Implementation+;;;+;;; Returns raw YAML data - the checker handles conversion to lib format.+;;; This keeps format knowledge here, lib knowledge in checker.+;;; ===========================================================================++(defn- resolve-schema-name+ "Resolve a real schema name to the key used in the model.++ Schema names are the one entity type with no canonical YAML file — the directory+ name is the filesystem key, and the real name comes from the `schema:` field in+ table YAMLs. These may differ (e.g. dir=`public`, real=`PUBLIC` for H2).++ Strategy:+ 1. Exact match in the model's :schemas+ 2. Case-insensitive match (handles H2-style uppercasing)+ 3. nil for schema-less databases++ To change schema resolution in the future, modify this function."+ [model db-name schema]+ (let [schemas (:schemas (get model db-name))+ lower (fn [s] (.toLowerCase (str s) Locale/US))]+ (cond+ (contains? schemas schema) schema+ schema (let [lowered (lower schema)]+ (some (fn [k] (when (and (string? k) (= (lower k) lowered)) k))+ (keys schemas)))+ :else (when (contains? schemas nil) nil))))+(defn quick-extract"Extract a field from YAML using regex (fast) with full parse fallback."[file-path field-name pattern]@@ -72,20 +104,14 @@(get (yaml/parse-string content) (keyword field-name))))(catch Exception _ nil)))-(defn extract-entity-id- "Extract the entity id from a yaml file without parsing it. Looking for a top level entity_id at start of line."- [file-path]- (quick-extract file-path "entity_id" #"(?m)^entity_id:\s*(\S+)"))--(defn extract-model- "Extract the serdes model from a YAML file (e.g., \"Collection\", \"Card\", \"Dashboard\").- Looks for the model field inside serdes/meta to avoid matching embedded refs."- [file-path]- (try- (let [content (slurp file-path)]- (when-let [[_ model] (re-find #"(?m)^serdes/meta:\s*\n\s*- id:.*\n\s+(?:label:.*\n\s+)?model:\s*(\S+)" content)]- (str/trim model)))- (catch Exception _ nil)))+(defn- read-yaml-schema+ "Read the `schema:` field from a YAML file using regex (fast).+ Returns nil if not found or if the value is `null`."+ [^File yaml-file]+ (when (.exists yaml-file)+ (let [v (quick-extract (.getPath yaml-file) "schema" #"(?m)^schema:\s*(\S+)")]+ (when (and v (not= v "null"))+ v))))(defn- read-yaml-name"Read the `name:` field from a YAML file using regex (fast).@@ -96,178 +122,6 @@fallback)fallback))-(defn- db-dir- "Resolve a real database name to its directory on disk."- ^File [^File databases-dir db-name->dir db-name]- (let [dir-name (get db-name->dir db-name db-name)]- (io/file databases-dir dir-name)))--(defn- list-table-dirs- "Return a seq of table directory Files for a database.- Walks schemas/<schema>/tables/<table>/ and tables/<table>/ directories."- [^File databases-dir db-name->dir-map db-name]- (let [^File db (db-dir databases-dir db-name->dir-map db-name)- result (volatile! [])]- (let [^File schemas-dir (io/file db "schemas")]- (when (.isDirectory schemas-dir)- (doseq [^File schema-d (.listFiles schemas-dir)- :when (.isDirectory schema-d)]- (let [^File tables-dir (io/file schema-d "tables")]- (when (.isDirectory tables-dir)- (doseq [^File td (.listFiles tables-dir)- :when (.isDirectory td)]- (vswap! result conj td)))))))- (let [^File tables-dir (io/file db "tables")]- (when (.isDirectory tables-dir)- (doseq [^File td (.listFiles tables-dir)- :when (.isDirectory td)]- (vswap! result conj td))))- @result))--(defn- index-segments-and-measures- "Walk a table directory's segments/ and measures/ subdirectories.- Reads entity_id from each YAML (only a handful exist).- Returns a seq of {:kind :ref :file} entries."- [^File table-dir]- (into []- (mapcat- (fn [[subdir kind]]- (let [^File d (io/file table-dir subdir)]- (when (.isDirectory d)- (keep (fn [^File f]- (when (and (.isFile f) (str/ends-with? (.getName f) ".yaml"))- (when-let [eid (extract-entity-id (.getPath f))]- {:kind kind :ref eid :file (.getPath f)})))- (.listFiles d))))))- [["segments" :segment] ["measures" :measure]]))--(defn- index-schema-dir- "Walk a databases directory and index databases, segments, and measures.- Reads ~40 database YAMLs for real names, plus a handful of segment/measure YAMLs- for entity_ids. Tables and fields are resolved on demand from the filesystem.-- When `include-segments-measures?` is true, also walks table directories- to find segment and measure YAMLs (for export dirs). Set to false for- schema-only dirs to avoid walking thousands of table directories.-- Returns {:entries [...]- :db-name->dir {real-db-name dir-name}}."- [schema-dir & {:keys [include-segments-measures?]}]- (let [schema-dir (io/file schema-dir)- db-name->dir (volatile! {})- entries (volatile! [])]- (doseq [^File db-dir (.listFiles schema-dir)- :when (.isDirectory db-dir)]- (let [dir-name (.getName db-dir)- db-yaml (io/file db-dir (str dir-name ".yaml"))- db-name (read-yaml-name db-yaml dir-name)]- (vswap! db-name->dir assoc db-name dir-name)- (vswap! entries conj {:kind :database :ref db-name- :file (if (.exists db-yaml) (.getPath db-yaml) (.getPath db-dir))})- ;; Walk table directories for segments and measures (export dirs only)- (when include-segments-measures?- (doseq [^File table-dir (list-table-dirs schema-dir @db-name->dir db-name)]- (vswap! entries into (index-segments-and-measures table-dir))))))- {:entries @entries- :db-name->dir @db-name->dir}))--;;; ===========================================================================-;;; File Walking - Build index of all entities-;;; ===========================================================================--(def ^:private model->kind- "Map serdes model names to index kinds."- {"Card" :card- "Dashboard" :dashboard- "Collection" :collection- "Document" :document- "Measure" :measure- "Segment" :segment- "NativeQuerySnippet" :snippet- "Transform" :transform})--(defn- index-collections-tree- "Walk the collections/ directory and index all entities by their serdes model.- Returns entries for cards, dashboards, and collections."- [export-dir]- (let [collections-dir (io/file export-dir "collections")]- (when (.isDirectory collections-dir)- (for [^File file (file-seq collections-dir)- :when (.isFile file)- :let [path (.getPath file)]- :when (str/ends-with? (.getName file) ".yaml")- :let [model (extract-model path)- kind (get model->kind model)]- :when kind- :let [entity-id (extract-entity-id path)]- :when entity-id]- {:kind kind :ref entity-id :file path}))))--(defn build-file-index- "Build index of all entity files in an export directory.- Returns `{:index {kind {ref file-path}} :db-name->dir {real-name dir-name}}`- where index also has `:duplicates` for refs appearing in multiple files."- [export-dir]- (let [databases-dir (io/file export-dir "databases")- schema-result (when (.isDirectory databases-dir)- (index-schema-dir databases-dir :include-segments-measures? true))- entries (concat- (:entries schema-result)- (index-collections-tree export-dir))- ;; Group by [kind ref] to find duplicates- by-key (reduce (fn [m {:keys [kind ref file]}]- (update m [kind ref] (fnil conj []) file))- {}- entries)- ;; Build the index (first file wins) and collect duplicates- index (reduce-kv (fn [idx [kind ref] files]- (assoc-in idx [kind ref] (first files)))- {}- by-key)- dupes (into []- (keep (fn [[[kind ref] files]]- (when (> (count files) 1)- {:kind kind :ref ref :files files})))- by-key)]- {:index (cond-> index- (seq dupes) (assoc :duplicates dupes))- :db-name->dir (:db-name->dir schema-result)}))--(defn build-database-dir-index- "Build index of databases from a schema directory.- Tables and fields are resolved on demand.-- Returns `{:index {:database {name file-path}} :db-name->dir {real-name dir-name}}`."- [databases-dir]- (let [{:keys [entries db-name->dir]} (index-schema-dir databases-dir)]- {:index (reduce (fn [idx {:keys [kind ref file]}]- (assoc-in idx [kind ref] file))- {}- entries)- :db-name->dir db-name->dir}))--(defn index-stats- "Get statistics about a file index.- Note: tables and fields are resolved on demand and won't appear in the index- for schema-dir sources. They will be 0 for `build-database-dir-index` results."- [index]- {:databases (count (:database index))- :tables (count (:table index))- :fields (count (:field index))- :measures (count (:measure index))- :segments (count (:segment index))- :cards (count (:card index))- :dashboards (count (:dashboard index))- :collections (count (:collection index))- :database-names (vec (keys (:database index)))})--;;; ===========================================================================-;;; MetadataSource Implementation-;;;-;;; Returns raw YAML data - the checker handles conversion to lib format.-;;; This keeps format knowledge here, lib knowledge in checker.-;;; ===========================================================================-(defn- table-real-name"Read the real table name from its YAML file via regex. Falls back to dir name."[^File table-dir]@@ -275,22 +129,6 @@^File yaml (io/file table-dir (str dir-name ".yaml"))](read-yaml-name yaml dir-name)))-(defn- field-real-name- "Read the real field name from its YAML file via regex. Falls back to filename without extension."- [^File field-yaml]- (let [fname (.getName field-yaml)- fallback (str/replace fname #"\.yaml$" "")]- (read-yaml-name field-yaml fallback)))--(defn- read-yaml-schema- "Read the `schema:` field from a YAML file using regex (fast).- Returns nil if not found or if the value is `null`."- [^File yaml-file]- (when (.exists yaml-file)- (let [v (quick-extract (.getPath yaml-file) "schema" #"(?m)^schema:\s*(\S+)")]- (when (and v (not= v "null"))- v))))-(defn- index-schema-tables"Walk a single schema's tables directory and return:{:real-schema-name string-or-nil ; the real schema name from YAML (may differ from dir name)@@ -314,48 +152,6 @@{:real-schema-name @real-schema:tables tables})))-(defn- index-table-fields- "Walk a table's fields/ directory and return a map of:- {real-field-name File ...}- Reads real field names from YAMLs via fast regex.- Skips FieldValues and FieldUserSettings files."- [table-file-path]- (let [^File table-dir (.getParentFile (io/file table-file-path))- ^File fields-dir (io/file table-dir "fields")]- (if (.isDirectory fields-dir)- (into {}- (keep (fn [^File f]- (let [fname (.getName f)]- (when (and (.isFile f)- (str/ends-with? fname ".yaml")- (not (str/includes? fname "___")))- [(field-real-name f) (.getPath f)]))))- (.listFiles fields-dir))- {})))--(defn- resolve-schema-name- "Resolve a real schema name to the key used in the model.-- Schema names are the one entity type with no canonical YAML file — the directory- name is the filesystem key, and the real name comes from the `schema:` field in- table YAMLs. These may differ (e.g. dir=`public`, real=`PUBLIC` for H2).-- Strategy:- 1. Exact match in the model's :schemas- 2. Case-insensitive match (handles H2-style uppercasing)- 3. nil for schema-less databases-- To change schema resolution in the future, modify this function."- [model db-name schema]- (let [schemas (:schemas (get model db-name))- lower (fn [s] (.toLowerCase (str s) Locale/US))]- (cond- (contains? schemas schema) schema- schema (let [lowered (lower schema)]- (some (fn [k] (when (and (string? k) (= (lower k) lowered)) k))- (keys schemas)))- :else (when (contains? schemas nil) nil))))-(defn- ensure-schema-indexed!"Ensure a schema's tables are indexed in the model. No-op if already done.@@ -384,6 +180,32 @@true(assoc-in [db-name :schemas final-key] tables))))))))+(defn- field-real-name+ "Read the real field name from its YAML file via regex. Falls back to filename without extension."+ [^File field-yaml]+ (let [fname (.getName field-yaml)+ fallback (str/replace fname #"\.yaml$" "")]+ (read-yaml-name field-yaml fallback)))++(defn- index-table-fields+ "Walk a table's fields/ directory and return a map of:+ {real-field-name File ...}+ Reads real field names from YAMLs via fast regex.+ Skips FieldValues and FieldUserSettings files."+ [table-file-path]+ (let [^File table-dir (.getParentFile (io/file table-file-path))+ ^File fields-dir (io/file table-dir "fields")]+ (if (.isDirectory fields-dir)+ (into {}+ (keep (fn [^File f]+ (let [fname (.getName f)]+ (when (and (.isFile f)+ (str/ends-with? fname ".yaml")+ (not (str/includes? fname "___")))+ [(field-real-name f) (.getPath f)]))))+ (.listFiles fields-dir))+ {})))+(defn- ensure-fields-indexed!"Ensure a table's fields are indexed in the model. No-op if already done."[^File databases-dir db-name->dir db-name schema table-name model]@@ -393,47 +215,53 @@(let [field-map (index-table-fields table-file)](swap! model assoc-in [db-name :schemas schema table-name :fields] field-map)))))-(defn- list-schemas- "List schema directory names for a database. Returns seq of strings, or [nil] for schema-less."- [^File databases-dir db-name->dir db-name]- (let [^File db (db-dir databases-dir db-name->dir db-name)- ^File schemas-dir (io/file db "schemas")- ^File tables-dir (io/file db "tables")]- (cond-> []- (.isDirectory schemas-dir)- (into (map #(.getName ^File %))- (filter #(.isDirectory ^File %) (.listFiles schemas-dir)))- (.isDirectory tables-dir)- (conj nil))))+(defn- index-schema-dir+ "Walk a databases directory and index databases, segments, and measures.+ Reads ~40 database YAMLs for real names, plus a handful of segment/measure YAMLs+ for entity_ids. Tables and fields are resolved on demand from the filesystem.-(defn- init-schema-model- "Build the initial schema model with db-file paths and ::not-indexed for each schema.+ When `include-segments-measures?` is true, also walks table directories+ to find segment and measure YAMLs (for export dirs). Set to false for+ schema-only dirs to avoid walking thousands of table directories.- Schema model shape:- {db-name {:db-file \"path/to/db.yaml\"- :schemas {schema ::not-indexed- | {table {:table-file \"path\" :fields ::not-indexed | {field \"path\"}}}}}}+ Returns {:entries [...]+ :db-name->dir {real-db-name dir-name}}."+ [schema-dir]+ (let [schema-dir (io/file schema-dir)+ db-name->dir (volatile! {})+ entries (volatile! [])]+ (doseq [^File db-dir (.listFiles schema-dir)+ :when (.isDirectory db-dir)]+ (let [dir-name (.getName db-dir)+ db-yaml (io/file db-dir (str dir-name ".yaml"))+ db-name (read-yaml-name db-yaml dir-name)]+ (vswap! db-name->dir assoc db-name dir-name)+ (vswap! entries conj {:kind :database :ref db-name+ :file (if (.exists db-yaml) (.getPath db-yaml) (.getPath db-dir))})))+ {:entries @entries+ :db-name->dir @db-name->dir}))- Two levels of laziness:- 1. Schema → tables: populated when the schema is first accessed- 2. Table → fields: populated when that table's fields are first accessed"- [^File databases-dir db-name->dir db-file-index]- (atom- (into {}- (for [[db-name _dir-name] db-name->dir]- [db-name {:db-file (get db-file-index db-name)- :schemas (into {}- (for [schema (list-schemas databases-dir db-name->dir db-name)]- [schema ::not-indexed]))}]))))+(defn build-database-dir-index+ "Build index of databases from a schema directory.+ Tables and fields are resolved on demand.++ Returns `{:index {:database {name file-path}} :db-name->dir {real-name dir-name}}`."+ [databases-dir]+ (let [{:keys [entries db-name->dir]} (index-schema-dir databases-dir)]+ {:index (reduce (fn [idx {:keys [kind ref file]}]+ (assoc-in idx [kind ref] file))+ {}+ entries)+ :db-name->dir db-name->dir}))(p.types/defprotocol+ ISchemaModelDebug"Non exported protocol to get the schema of the SerdesSource. Useful for debugging and asserting on the schema modelbuilt up during serdes."(schema-model [_] "Deref and return the schema model. See [[init-schema-model]] for details of this structure."))-(deftype SerdesSource [databases-dir assets-index db-name->dir- ;; See init-schema-model for shape documentation- schema-model]+(deftype SerdesSchemaSource [databases-dir db-name->dir+ ;; See init-schema-model for shape documentation+ schema-model]source/SchemaSource(resolve-database [_ db-name](when-let [file (get-in @schema-model [db-name :db-file])]@@ -499,51 +327,41 @@table-name (keys tables)][db-name schema table-name])))- source/AssetsSource- (resolve-card [_ entity-id]- (when-let [file (get-in assets-index [:card entity-id])]- (load-yaml file)))-- (resolve-snippet [_ entity-id]- (when-let [file (get-in assets-index [:snippet entity-id])]- (load-yaml file)))-- (resolve-transform [_ entity-id]- (when-let [file (get-in assets-index [:transform entity-id])]- (load-yaml file)))-- (resolve-segment [_ entity-id]- (when-let [file (get-in assets-index [:segment entity-id])]- (load-yaml file)))-- (resolve-dashboard [_ entity-id]- (when-let [file (get-in assets-index [:dashboard entity-id])]- (load-yaml file)))+ ISchemaModelDebug+ (schema-model [_] (deref schema-model)))- (resolve-collection [_ entity-id]- (when-let [file (get-in assets-index [:collection entity-id])]- (load-yaml file)))+(defn- list-schemas+ "List schema directory names for a database. Returns seq of strings, or [nil] for schema-less."+ [^File databases-dir db-name->dir db-name]+ (let [^File db (db-dir databases-dir db-name->dir db-name)+ ^File schemas-dir (io/file db "schemas")+ ^File tables-dir (io/file db "tables")]+ (cond-> []+ (.isDirectory schemas-dir)+ (into (map #(.getName ^File %))+ (filter #(.isDirectory ^File %) (.listFiles schemas-dir)))+ (.isDirectory tables-dir)+ (conj nil))))- (resolve-document [_ entity-id]- (when-let [file (get-in assets-index [:document entity-id])]- (load-yaml file)))+(defn- init-schema-model+ "Build the initial schema model with db-file paths and ::not-indexed for each schema.- (resolve-measure [_ entity-id]- (when-let [file (get-in assets-index [:measure entity-id])]- (load-yaml file)))- ISchemaModelDebug- (schema-model [_] (deref schema-model)))+ Schema model shape:+ {db-name {:db-file \"path/to/db.yaml\"+ :schemas {schema ::not-indexed+ | {table {:table-file \"path\" :fields ::not-indexed | {field \"path\"}}}}}}-(defn make-source- "Create a MetadataSource for a serdes export directory.- The databases-dir for field resolution is export-dir/databases."- [export-dir]- (let [databases-dir (io/file export-dir "databases")- {:keys [index db-name->dir]} (build-file-index export-dir)- db-files (:database index)- assets-index (dissoc index :database)]- (->SerdesSource databases-dir assets-index db-name->dir- (init-schema-model databases-dir db-name->dir db-files))))+ Two levels of laziness:+ 1. Schema → tables: populated when the schema is first accessed+ 2. Table → fields: populated when that table's fields are first accessed"+ [^File databases-dir db-name->dir db-file-index]+ (atom+ (into {}+ (for [[db-name _dir-name] db-name->dir]+ [db-name {:db-file (get db-file-index db-name)+ :schemas (into {}+ (for [schema (list-schemas databases-dir db-name->dir db-name)]+ [schema ::not-indexed]))}]))))(defn make-database-source"Create a MetadataSource for a directory that IS the databases directory.@@ -553,20 +371,5 @@(let [{:keys [index db-name->dir]} (build-database-dir-index databases-dir)databases-dir (io/file databases-dir)db-files (:database index)]- (->SerdesSource databases-dir {} db-name->dir- (init-schema-model databases-dir db-name->dir db-files))))--(defn source-index- "Get the assets index from a SerdesSource. Returns a flat map of- {[kind ref] file-path} for cards, dashboards, segments, etc."- [^SerdesSource source]- (.-assets-index source))--;;; ===========================================================================-;;; Enumeration-;;; ===========================================================================--(defn all-card-ids- "Get all card entity-ids from source."- [^SerdesSource source]- (keys (:card (.-assets-index source))))+ (->SerdesSchemaSource databases-dir db-name->dir+ (init-schema-model databases-dir db-name->dir db-files))))diff --git a/enterprise/backend/src/metabase_enterprise/checker/semantic.clj b/enterprise/backend/src/metabase_enterprise/checker/semantic.cljindex 8a9c656aac9..1d905ae2e5c 100644--- a/enterprise/backend/src/metabase_enterprise/checker/semantic.clj+++ b/enterprise/backend/src/metabase_enterprise/checker/semantic.clj@@ -11,7 +11,8 @@- `(setup export-dir schema-dir)` + `(check-one ctx entity-id)` — REPL workflow"(:require[clojure.string :as str]- [metabase-enterprise.checker.format.serdes :as serdes]+ [metabase-enterprise.checker.format.serdes-assets :as serdes-assets]+ [metabase-enterprise.checker.format.serdes-schema :as serdes.schema][metabase-enterprise.checker.provider :as provider][metabase-enterprise.checker.store :as store][metabase-enterprise.dependencies.analysis :as deps.analysis]@@ -717,9 +718,9 @@"Build schema and assets sources from `schema-dir` and `export-dir`,and a merged assets index (cards, dashboards, segments, etc.)."[export-dir schema-dir]- (let [schema-source (serdes/make-database-source schema-dir)- assets-source (serdes/make-source export-dir)- index (serdes/source-index assets-source)]+ (let [schema-source (serdes.schema/make-database-source schema-dir)+ assets-source (serdes-assets/make-source export-dir)+ index (serdes-assets/source-index assets-source)]{:schema-source schema-source :assets-source assets-source :index index}))(defn check@@ -770,7 +771,13 @@(comment;; REPL workflow:(check "/Users/dan/projects/work/stats-remote-sync"- "/tmp/metadata/metadata/databases")+ "/Users/dan/projects/work/exports-root/metadata/metadata/databases")++ (->> (check "/Users/dan/projects/work/representations/examples/v1"+ "/Users/dan/projects/work/yaml-checked-files-v1/exports/sqlite-based/databases")+ (into {} (filter (fn [[_id stuff]]+ (letfn [(bad [x] ((some-fn :bad-refs :unresolved :native-errors) x))]+ (bad stuff))))))(setup"/Users/dan/projects/work/stats-remote-sync""/Users/dan/projects/work/stats-remote-sync/databases")diff --git a/enterprise/backend/test/metabase_enterprise/checker/format/serdes_assets_test.clj b/enterprise/backend/test/metabase_enterprise/checker/format/serdes_assets_test.cljnew file mode 100644index 00000000000..55fc8b3e21c--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/checker/format/serdes_assets_test.clj@@ -0,0 +1,178 @@+(ns metabase-enterprise.checker.format.serdes-assets-test+ "Tests for the serdes assets format module — YAML extraction, file indexing, and assets source resolution."+ (:require+ [clojure.java.io :as io]+ [clojure.test :refer [deftest is testing]]+ [metabase-enterprise.checker.format.serdes-assets :as serdes]+ [metabase-enterprise.checker.source :as source]+ [metabase.util.yaml :as yaml]))++(set! *warn-on-reflection* true)++;;; ===========================================================================+;;; Helpers+;;; ===========================================================================++(def ^:private fixtures-dir "test_resources/yaml_checks")++(defn- fixtures-path []+ (let [f (io/file fixtures-dir)]+ (if (.isAbsolute f)+ fixtures-dir+ (.getPath (io/file (System/getProperty "user.dir") fixtures-dir)))))++(defn- with-temp-dir+ "Create a temp directory, call (f dir-path), then clean up."+ [f]+ (let [dir (java.io.File/createTempFile "serdes-test" "")]+ (.delete dir)+ (.mkdirs dir)+ (try+ (f (.getPath dir))+ (finally+ (doseq [fl (reverse (file-seq dir))]+ (.delete ^java.io.File fl))))))++(defn- write-yaml!+ "Write a YAML file, creating parent directories as needed."+ [dir & path-and-data]+ (let [data (last path-and-data)+ path-segs (butlast path-and-data)+ ^java.io.File file (apply io/file dir (map str path-segs))]+ (.mkdirs (.getParentFile file))+ (spit file (yaml/generate-string data))+ (.getPath file)))++;;; ===========================================================================+;;; YAML extraction tests (pure, no files needed)+;;; ===========================================================================++(deftest extract-entity-id-from-fixture-test+ (testing "extract-entity-id reads entity_id from a card YAML"+ (let [card-file (io/file (fixtures-path) "collections" "cards" "simple-orders_simple_orders.yaml")]+ (is (= "simple-orders" (serdes/extract-entity-id (.getPath card-file)))))))++(deftest extract-model-from-fixture-test+ (testing "extract-model reads the serdes model from a card YAML"+ (let [card-file (io/file (fixtures-path) "collections" "cards" "simple-orders_simple_orders.yaml")]+ (is (= "Card" (serdes/extract-model (.getPath card-file)))))))++(deftest extract-entity-id-from-temp-file-test+ (testing "extract-entity-id with various formats"+ (with-temp-dir+ (fn [dir]+ (let [f (write-yaml! dir "test.yaml" {:name "X" :entity_id "xYz123AbC"})]+ (is (= "xYz123AbC" (serdes/extract-entity-id f))))))))++;;; ===========================================================================+;;; Index tests — databases are indexed, tables/fields are on-demand+;;; ===========================================================================++(deftest build-file-index-databases-test+ (testing "build-file-index indexes databases from fixtures"+ (let [index (serdes/build-file-index (fixtures-path))]+ (is (contains? (:database index) "Test Database"))+ (is (contains? (:database index) "SQLite DB")))))++(deftest build-file-index-cards-test+ (testing "build-file-index indexes cards by entity_id"+ (let [index (serdes/build-file-index (fixtures-path))]+ (is (contains? (:card index) "simple-orders"))+ (is (contains? (:card index) "native-orders"))+ (is (contains? (:card index) "orders-with-products")))))++(deftest build-file-index-no-duplicates-test+ (testing "fixture index has no duplicates"+ (let [index (serdes/build-file-index (fixtures-path))]+ (is (nil? (:duplicates index))))))++;;; ===========================================================================+;;; AssetsSource protocol — resolution tests+;;; ===========================================================================++(deftest source-resolves-card-test+ (testing "SerdesSource resolves cards by entity-id"+ (let [source (serdes/make-source (fixtures-path))+ card (source/resolve-card source "simple-orders")]+ (is (some? card))+ (is (= "Simple Orders" (:name card))))))++(deftest source-returns-nil-for-unknown-test+ (testing "SerdesSource returns nil for unknown entities"+ (let [source (serdes/make-source (fixtures-path))]+ (is (nil? (source/resolve-card source "no-such-card"))))))++;;; ===========================================================================+;;; Segments and measures — indexed from export dir+;;; ===========================================================================++(deftest export-dir-indexes-segments-and-measures-test+ (testing "build-file-index (export dir) finds segments and measures under databases/"+ (let [index (serdes/build-file-index (fixtures-path))]+ (is (contains? (:segment index) "big-orders-segment")+ "segment from with-schema table should be indexed")+ (is (contains? (:segment index) "recent-orders-segment")+ "segment from schema-less table should be indexed")+ (is (contains? (:measure index) "total-revenue-measure")+ "measure should be indexed"))))++(deftest segments-resolve-from-export-source-test+ (testing "segments can be resolved from the export source"+ (let [source (serdes/make-source (fixtures-path))]+ (let [seg (source/resolve-segment source "big-orders-segment")]+ (is (some? seg))+ (is (= "Big Orders" (:name seg))))+ (let [seg (source/resolve-segment source "recent-orders-segment")]+ (is (some? seg))+ (is (= "Recent Orders" (:name seg)))))))++(deftest measures-resolve-from-export-source-test+ (testing "measures can be resolved from the export source"+ (let [source (serdes/make-source (fixtures-path))+ m (source/resolve-measure source "total-revenue-measure")]+ (is (some? m))+ (is (= "Total Revenue" (:name m))))))++;;; ===========================================================================+;;; Temp file edge cases+;;; ===========================================================================++(deftest build-file-index-empty-dir-test+ (testing "build-file-index on empty directory returns empty index"+ (with-temp-dir+ (fn [dir]+ (let [index (serdes/build-file-index dir)]+ (is (nil? (:database index)))+ (is (nil? (:card index))))))))++(def ^:private card-yaml-template+ "name: %s\nentity_id: %s\nserdes/meta:\n- id: %s\n model: Card\n")++(deftest build-file-index-detects-duplicates-test+ (testing "build-file-index reports duplicate entity_ids"+ (with-temp-dir+ (fn [dir]+ ;; Create two cards with the same entity_id in different directories+ (let [^java.io.File f1 (io/file dir "collections" "a" "cards" "dup.yaml")+ ^java.io.File f2 (io/file dir "collections" "b" "cards" "dup.yaml")]+ (.mkdirs (.getParentFile f1))+ (.mkdirs (.getParentFile f2))+ (spit f1 (format card-yaml-template "Card A" "dup-id" "dup-id"))+ (spit f2 (format card-yaml-template "Card B" "dup-id" "dup-id"))+ (let [index (serdes/build-file-index dir)]+ (is (seq (:duplicates index)) "Should detect duplicate entity_ids")+ (is (= "dup-id" (:ref (first (:duplicates index)))))))))))++(deftest source-index-accessor-test+ (testing "source-index returns the assets index (no databases — those are in the schema source)"+ (let [source (serdes/make-source (fixtures-path))+ index (serdes/source-index source)]+ (is (map? index))+ (is (not (contains? index :database)) "databases are in the schema source, not the assets index")+ (is (contains? index :card)))))++(deftest all-card-ids-test+ (testing "all-card-ids returns all card entity-ids"+ (let [source (serdes/make-source (fixtures-path))]+ (is (>= (count (serdes/all-card-ids source)) 5))+ (is (contains? (set (serdes/all-card-ids source)) "simple-orders")))))diff --git a/enterprise/backend/test/metabase_enterprise/checker/format/serdes_test.clj b/enterprise/backend/test/metabase_enterprise/checker/format/serdes_schema_test.cljsimilarity index 57%rename from enterprise/backend/test/metabase_enterprise/checker/format/serdes_test.cljrename to enterprise/backend/test/metabase_enterprise/checker/format/serdes_schema_test.cljindex 9ff088efa38..a8ec4eb9033 100644--- a/enterprise/backend/test/metabase_enterprise/checker/format/serdes_test.clj+++ b/enterprise/backend/test/metabase_enterprise/checker/format/serdes_schema_test.clj@@ -1,9 +1,9 @@-(ns metabase-enterprise.checker.format.serdes-test- "Tests for the serdes format module — YAML extraction, file indexing, and source resolution."+(ns metabase-enterprise.checker.format.serdes-schema-test+ "Tests for the serdes schema format module — database/table/field resolution and lazy indexing."(:require[clojure.java.io :as io]- [clojure.test :refer [deftest is testing]]- [metabase-enterprise.checker.format.serdes :as serdes]+ [clojure.test :refer :all]+ [metabase-enterprise.checker.format.serdes-schema :as serdes-schema][metabase-enterprise.checker.source :as source][metabase.util.yaml :as yaml]))@@ -43,53 +43,10 @@(spit file (yaml/generate-string data))(.getPath file)))-;;; ===========================================================================-;;; YAML extraction tests (pure, no files needed)-;;; ===========================================================================--(deftest extract-entity-id-from-fixture-test- (testing "extract-entity-id reads entity_id from a card YAML"- (let [card-file (io/file (fixtures-path) "collections" "cards" "simple-orders_simple_orders.yaml")]- (is (= "simple-orders" (serdes/extract-entity-id (.getPath card-file)))))))--(deftest extract-model-from-fixture-test- (testing "extract-model reads the serdes model from a card YAML"- (let [card-file (io/file (fixtures-path) "collections" "cards" "simple-orders_simple_orders.yaml")]- (is (= "Card" (serdes/extract-model (.getPath card-file)))))))--(deftest extract-entity-id-from-temp-file-test- (testing "extract-entity-id with various formats"- (with-temp-dir- (fn [dir]- (let [f (write-yaml! dir "test.yaml" {:name "X" :entity_id "xYz123AbC"})]- (is (= "xYz123AbC" (serdes/extract-entity-id f))))))))--;;; ===========================================================================-;;; Index tests — databases are indexed, tables/fields are on-demand-;;; ===========================================================================--(deftest build-file-index-databases-test- (testing "build-file-index indexes databases from fixtures"- (let [{:keys [index]} (serdes/build-file-index (fixtures-path))]- (is (contains? (:database index) "Test Database"))- (is (contains? (:database index) "SQLite DB")))))--(deftest build-file-index-cards-test- (testing "build-file-index indexes cards by entity_id"- (let [{:keys [index]} (serdes/build-file-index (fixtures-path))]- (is (contains? (:card index) "simple-orders"))- (is (contains? (:card index) "native-orders"))- (is (contains? (:card index) "orders-with-products")))))--(deftest build-file-index-no-duplicates-test- (testing "fixture index has no duplicates"- (let [{:keys [index]} (serdes/build-file-index (fixtures-path))]- (is (nil? (:duplicates index))))))-(deftest build-database-dir-index-test(testing "build-database-dir-index indexes databases and provides db-name->dir mapping"(let [db-dir (str (fixtures-path) "/databases")- {:keys [index db-name->dir]} (serdes/build-database-dir-index db-dir)]+ {:keys [index db-name->dir]} (serdes-schema/build-database-dir-index db-dir)](is (contains? (:database index) "Test Database"))(is (contains? (:database index) "SQLite DB"))(is (contains? db-name->dir "Test Database"))@@ -110,7 +67,7 @@;; Create a database with a slugified directory name but real name in YAML(write-yaml! dir "databases" "my_fancy_db" "my_fancy_db.yaml"{:name "My Fancy DB" :engine "h2"})- (let [{:keys [db-name->dir]} (serdes/build-database-dir-index (str dir "/databases"))]+ (let [{:keys [db-name->dir]} (serdes-schema/build-database-dir-index (str dir "/databases"))](is (= "my_fancy_db" (get db-name->dir "My Fancy DB")))))))(testing "db-name->dir includes ALL databases, not just some"@@ -120,7 +77,7 @@(let [slug (str "db_" i)](write-yaml! dir "databases" slug (str slug ".yaml"){:name (str "Database " i) :engine "h2"})))- (let [{:keys [db-name->dir]} (serdes/build-database-dir-index (str dir "/databases"))]+ (let [{:keys [db-name->dir]} (serdes-schema/build-database-dir-index (str dir "/databases"))](is (= 10 (count db-name->dir)))(doseq [i (range 10)](is (= (str "db_" i) (get db-name->dir (str "Database " i))))))))))@@ -130,50 +87,42 @@;;; ===========================================================================(deftest source-resolves-database-test- (testing "SerdesSource resolves databases by name"- (let [source (serdes/make-source (fixtures-path))+ (testing "SerdesSchemaSource resolves databases by name"+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))db (source/resolve-database source "Test Database")](is (some? db))(is (= "Test Database" (:name db))))))(deftest source-resolves-table-on-demand-test- (testing "SerdesSource resolves tables by path on demand (no pre-indexing)"- (let [source (serdes/make-source (fixtures-path))+ (testing "SerdesSchemaSource resolves tables by path on demand (no pre-indexing)"+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))table (source/resolve-table source ["Test Database" "public" "orders"])](is (some? table))(is (= "orders" (:name table))))(testing "schema-less tables"- (let [source (serdes/make-source (fixtures-path))+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))table (source/resolve-table source ["SQLite DB" nil "orders"])](is (some? table))(is (= "orders" (:name table)))))))(deftest source-resolves-field-on-demand-test- (testing "SerdesSource resolves fields by path on demand"- (let [source (serdes/make-source (fixtures-path))+ (testing "SerdesSchemaSource resolves fields by path on demand"+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))field (source/resolve-field source ["Test Database" "public" "orders" "id"])](is (some? field))(is (= "id" (:name field))))(testing "schema-less fields"- (let [source (serdes/make-source (fixtures-path))+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))field (source/resolve-field source ["SQLite DB" nil "orders" "id"])](is (some? field))(is (= "id" (:name field)))))))-(deftest source-resolves-card-test- (testing "SerdesSource resolves cards by entity-id"- (let [source (serdes/make-source (fixtures-path))- card (source/resolve-card source "simple-orders")]- (is (some? card))- (is (= "Simple Orders" (:name card))))))-(deftest source-returns-nil-for-unknown-test- (testing "SerdesSource returns nil for unknown entities"- (let [source (serdes/make-source (fixtures-path))]+ (testing "SerdesSchemaSource returns nil for unknown entities"+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))](is (nil? (source/resolve-database source "Nonexistent")))(is (nil? (source/resolve-table source ["DB" "x" "y"])))- (is (nil? (source/resolve-field source ["DB" "x" "y" "z"])))- (is (nil? (source/resolve-card source "no-such-card"))))))+ (is (nil? (source/resolve-field source ["DB" "x" "y" "z"]))))));;; ===========================================================================;;; fields-for-table and all-table-paths — on-demand enumeration@@ -181,23 +130,23 @@(deftest fields-for-table-test(testing "fields-for-table returns field paths for a table"- (let [source (serdes/make-source (fixtures-path))+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))fields (source/fields-for-table source ["Test Database" "public" "orders"])](is (set? fields))(is (contains? fields ["Test Database" "public" "orders" "id"]))(is (contains? fields ["Test Database" "public" "orders" "total"]))))(testing "fields-for-table works for schema-less tables"- (let [source (serdes/make-source (fixtures-path))+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))fields (source/fields-for-table source ["SQLite DB" nil "orders"])](is (set? fields))(is (contains? fields ["SQLite DB" nil "orders" "id"]))))(testing "fields-for-table returns nil for unknown table"- (let [source (serdes/make-source (fixtures-path))]+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))](is (nil? (source/fields-for-table source ["Nope" "x" "y"]))))))(deftest all-table-paths-test(testing "all-table-paths enumerates all tables across databases"- (let [source (serdes/make-source (fixtures-path))+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))tables (source/all-table-paths source)](is (seq tables))(is (some #(= ["Test Database" "public" "orders"] %) tables))@@ -205,7 +154,7 @@(deftest all-database-names-test(testing "all-database-names returns database names"- (let [source (serdes/make-source (fixtures-path))]+ (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))](is (= #{"Test Database" "SQLite DB"} (set (source/all-database-names source)))))));;; ===========================================================================@@ -233,7 +182,7 @@{:id "public" :model "Schema"}{:id "orders" :model "Table"}{:id "id" :model "Field"}]})- (let [source (serdes/make-database-source db-dir)]+ (let [source (serdes-schema/make-database-source db-dir)];; Database resolves by real name(is (some? (source/resolve-database source "Analytics Data Warehouse")));; Table resolves using real db name@@ -253,7 +202,7 @@(deftest index-does-not-contain-tables-or-fields-test(testing "the file index only contains databases — tables and fields are resolved on demand"(let [db-dir (str (fixtures-path) "/databases")- {:keys [index]} (serdes/build-database-dir-index db-dir)]+ {:keys [index]} (serdes-schema/build-database-dir-index db-dir)](is (seq (:database index)) "databases should be in the index")(is (nil? (:table index)) "tables should NOT be in the index")(is (nil? (:field index)) "fields should NOT be in the index"))))@@ -262,7 +211,7 @@;;; Cache shape and lazy resolution;;; ===========================================================================-(def ^:private not-indexed :metabase-enterprise.checker.format.serdes/not-indexed)+(def ^:private not-indexed ::serdes-schema/not-indexed)(deftest schema-model-shape-test(testing "schema model starts with ::not-indexed for each schema, populates lazily"@@ -277,24 +226,24 @@{:name "T2" :schema "s2"})(write-yaml! dir "databases" "db" "schemas" "s2" "tables" "t2" "fields" "f2.yaml"{:name "F2" :base_type "type/Text"})- (let [source (serdes/make-database-source (str dir "/databases"))]+ (let [source (serdes-schema/make-database-source (str dir "/databases"))];; Initial: db-file set, all schemas ::not-indexed(is (=? {"DB" {:db-file string?:schemas {"s1" not-indexed"s2" not-indexed}}}- (serdes/schema-model source)))+ (serdes-schema/schema-model source)));; After resolving a table in s1: s1 indexed with tables, s2 untouched(source/resolve-table source ["DB" "s1" "T1"])(is (=? {"DB" {:schemas {"s1" {"T1" {:table-file string?:fields not-indexed}}"s2" not-indexed}}}- (serdes/schema-model source)))+ (serdes-schema/schema-model source)));; After resolving fields for T1: T1 fields indexed, s2 still untouched(source/fields-for-table source ["DB" "s1" "T1"])(is (=? {"DB" {:schemas {"s1" {"T1" {:table-file string?:fields {"F1" string?}}}"s2" not-indexed}}}- (serdes/schema-model source))))))))+ (serdes-schema/schema-model source))))))))(deftest case-insensitive-schema-resolution-test(testing "schemas with different casing on disk vs YAML resolve correctly"@@ -306,91 +255,9 @@{:name "ORDERS" :schema "PUBLIC"})(write-yaml! dir "databases" "db" "schemas" "public" "tables" "orders" "fields" "id.yaml"{:name "ID" :base_type "type/Integer"})- (let [source (serdes/make-database-source (str dir "/databases"))]+ (let [source (serdes-schema/make-database-source (str dir "/databases"))];; Resolve by real names (uppercase) — schema re-keys from "public" to "PUBLIC"(is (some? (source/resolve-table source ["DB" "PUBLIC" "ORDERS"])))(is (some? (source/resolve-field source ["DB" "PUBLIC" "ORDERS" "ID"])))(is (=? {"DB" {:schemas {"PUBLIC" {"ORDERS" {:fields {"ID" string?}}}}}}- (serdes/schema-model source))))))))--;;; ===========================================================================-;;; Segments and measures — indexed from export dir, NOT schema dir-;;; ===========================================================================--(deftest export-dir-indexes-segments-and-measures-test- (testing "build-file-index (export dir) finds segments and measures under databases/"- (let [{:keys [index]} (serdes/build-file-index (fixtures-path))]- (is (contains? (:segment index) "big-orders-segment")- "segment from with-schema table should be indexed")- (is (contains? (:segment index) "recent-orders-segment")- "segment from schema-less table should be indexed")- (is (contains? (:measure index) "total-revenue-measure")- "measure should be indexed"))))--(deftest schema-dir-does-not-index-segments-test- (testing "build-database-dir-index (schema dir) does NOT index segments or measures"- (let [db-dir (str (fixtures-path) "/databases")- {:keys [index]} (serdes/build-database-dir-index db-dir)]- (is (nil? (:segment index)) "schema dir should not contain segments")- (is (nil? (:measure index)) "schema dir should not contain measures"))))--(deftest segments-resolve-from-export-source-test- (testing "segments can be resolved from the export source"- (let [source (serdes/make-source (fixtures-path))]- (let [seg (source/resolve-segment source "big-orders-segment")]- (is (some? seg))- (is (= "Big Orders" (:name seg))))- (let [seg (source/resolve-segment source "recent-orders-segment")]- (is (some? seg))- (is (= "Recent Orders" (:name seg)))))))--(deftest measures-resolve-from-export-source-test- (testing "measures can be resolved from the export source"- (let [source (serdes/make-source (fixtures-path))- m (source/resolve-measure source "total-revenue-measure")]- (is (some? m))- (is (= "Total Revenue" (:name m))))))--;;; ===========================================================================-;;; Temp file edge cases-;;; ===========================================================================--(deftest build-file-index-empty-dir-test- (testing "build-file-index on empty directory returns empty index"- (with-temp-dir- (fn [dir]- (let [{:keys [index]} (serdes/build-file-index dir)]- (is (nil? (:database index)))- (is (nil? (:card index))))))))--(def ^:private card-yaml-template- "name: %s\nentity_id: %s\nserdes/meta:\n- id: %s\n model: Card\n")--(deftest build-file-index-detects-duplicates-test- (testing "build-file-index reports duplicate entity_ids"- (with-temp-dir- (fn [dir]- ;; Create two cards with the same entity_id in different directories- (let [^java.io.File f1 (io/file dir "collections" "a" "cards" "dup.yaml")- ^java.io.File f2 (io/file dir "collections" "b" "cards" "dup.yaml")]- (.mkdirs (.getParentFile f1))- (.mkdirs (.getParentFile f2))- (spit f1 (format card-yaml-template "Card A" "dup-id" "dup-id"))- (spit f2 (format card-yaml-template "Card B" "dup-id" "dup-id"))- (let [{:keys [index]} (serdes/build-file-index dir)]- (is (seq (:duplicates index)) "Should detect duplicate entity_ids")- (is (= "dup-id" (:ref (first (:duplicates index)))))))))))--(deftest source-index-accessor-test- (testing "source-index returns the assets index (no databases — those are in the schema model)"- (let [source (serdes/make-source (fixtures-path))- index (serdes/source-index source)]- (is (map? index))- (is (not (contains? index :database)) "databases are in the schema model, not the assets index")- (is (contains? index :card)))))--(deftest all-card-ids-test- (testing "all-card-ids returns all card entity-ids"- (let [source (serdes/make-source (fixtures-path))]- (is (>= (count (serdes/all-card-ids source)) 5))- (is (contains? (set (serdes/all-card-ids source)) "simple-orders")))))+ (serdes-schema/schema-model source))))))))diff --git a/enterprise/backend/test/metabase_enterprise/checker/semantic_test.clj b/enterprise/backend/test/metabase_enterprise/checker/semantic_test.cljindex 68064b532a1..7f3c7775c1e 100644--- a/enterprise/backend/test/metabase_enterprise/checker/semantic_test.clj+++ b/enterprise/backend/test/metabase_enterprise/checker/semantic_test.clj@@ -6,7 +6,8 @@(:require[clojure.java.io :as io][clojure.test :refer [deftest is testing]]- [metabase-enterprise.checker.format.serdes :as serdes-format]+ [metabase-enterprise.checker.format.serdes-assets :as serdes-format]+ [metabase-enterprise.checker.format.serdes-schema :as serdes-schema][metabase-enterprise.checker.provider :as provider][metabase-enterprise.checker.semantic :as checker][metabase-enterprise.checker.source :as source]@@ -30,14 +31,18 @@test-fixtures-dir(.getPath (io/file (System/getProperty "user.dir") test-fixtures-dir)))))-(defn- make-test-source []- (serdes-format/make-source (fixtures-path)))+(defn- make-test-sources+ "Create both schema and assets sources from fixtures.+ Returns [schema-source assets-source]."+ []+ [(serdes-schema/make-database-source (str (fixtures-path) "/databases"))+ (serdes-format/make-source (fixtures-path))])-(defn- check-all [source]- (checker/check-entities source source (serdes-format/source-index source)))+(defn- check-all [[schema-source assets-source]]+ (checker/check-entities schema-source assets-source (serdes-format/source-index assets-source)))-(defn- check-specific [source entity-ids]- (checker/check-entities source source (serdes-format/source-index source) entity-ids))+(defn- check-specific [[schema-source assets-source] entity-ids]+ (checker/check-entities schema-source assets-source (serdes-format/source-index assets-source) entity-ids));;; ===========================================================================;;; Tests Using Real YAML Files@@ -45,20 +50,20 @@(deftest source-index-test(testing "Source correctly indexes databases and cards; tables/fields are on-demand"- (let [source (make-test-source)- index (serdes-format/source-index source)]- ;; Databases are in the schema model, not the assets index- (is (= #{"Test Database" "SQLite DB"} (set (source/all-database-names source))))+ (let [[schema-source assets-source] (make-test-sources)+ index (serdes-format/source-index assets-source)]+ ;; Databases are in the schema source+ (is (= #{"Test Database" "SQLite DB"} (set (source/all-database-names schema-source))));; Cards are in the assets index(is (= 5 (count (:card index))) "Should have 5 cards")- ;; Tables and fields are resolved on demand- (is (= 4 (count (source/all-table-paths source))) "Should have 4 tables via source")- (is (= 11 (count (source/all-field-paths source))) "Should have 11 fields via source"))))+ ;; Tables and fields are resolved on demand via schema source+ (is (= 4 (count (source/all-table-paths schema-source))) "Should have 4 tables via source")+ (is (= 11 (count (source/all-field-paths schema-source))) "Should have 11 fields via source"))))(deftest simple-mbql-query-test(testing "Simple MBQL query on orders table validates successfully"- (let [source (make-test-source)- results (check-all source)+ (let [sources (make-test-sources)+ results (check-all sources)result (get results "simple-orders")](is (some? result) "Card should be found")(is (= "Simple Orders" (:name result)) "Card name should match")@@ -70,8 +75,8 @@(deftest native-query-test(testing "Native SQL query validates successfully"- (let [source (make-test-source)- results (check-all source)+ (let [sources (make-test-sources)+ results (check-all sources)result (get results "native-orders")](is (some? result) "Card should be found")(is (= "Native Orders" (:name result)) "Card name should match")@@ -80,8 +85,8 @@(deftest mbql-with-joins-test(testing "MBQL query with joins validates successfully"- (let [source (make-test-source)- results (check-all source)+ (let [sources (make-test-sources)+ results (check-all sources)result (get results "orders-with-products")](is (some? result) "Card should be found")(is (= "Orders With Products" (:name result)) "Card name should match")@@ -94,8 +99,8 @@(deftest all-cards-checked-test(testing "All cards in fixtures are checked"- (let [source (make-test-source)- results (check-all source)]+ (let [sources (make-test-sources)+ results (check-all sources)](is (>= (count results) 5) "Should check at least 5 cards")(is (contains? results "simple-orders"))(is (contains? results "native-orders"))@@ -112,14 +117,14 @@(deftest schemaless-database-index-test(testing "Schema-less databases are indexed correctly with nil schema"- (let [source (make-test-source)]- (is (= #{"Test Database" "SQLite DB"} (set (source/all-database-names source))))- (is (= 4 (count (source/all-table-paths source))) "Should have 4 tables via source"))))+ (let [[schema-source _] (make-test-sources)]+ (is (= #{"Test Database" "SQLite DB"} (set (source/all-database-names schema-source))))+ (is (= 4 (count (source/all-table-paths schema-source))) "Should have 4 tables via source"))))(deftest schemaless-query-test(testing "Query on schema-less database validates successfully"- (let [source (make-test-source)- results (check-all source)+ (let [sources (make-test-sources)+ results (check-all sources)result (get results "schemaless-query")](is (some? result) "Card should be found")(is (= "Schemaless Query" (:name result)) "Card name should match")@@ -136,9 +141,9 @@(deftest fk-in-field-metadata-test(testing "FK target in field metadata is resolved to integer ID"- (let [source (make-test-source)- index (serdes-format/source-index source)- provider (provider/make-provider (store/make-store source source index))+ (let [[schema-source assets-source] (make-test-sources)+ index (serdes-format/source-index assets-source)+ provider (provider/make-provider (store/make-store schema-source assets-source index));; Get the product_id field which has FK to products.idfields (lib.metadata.protocols/metadatasprovider {:lib/type :metadata/column})@@ -150,8 +155,8 @@(deftest fk-in-result-metadata-test(testing "FK target in card result_metadata is resolved to integer ID"- (let [source (make-test-source)- results (check-all source)+ (let [sources (make-test-sources)+ results (check-all sources)result (get results "orders-with-fk")](is (some? result) "Card should be found")(is (= "Orders With FK" (:name result)) "Card name should match")@@ -242,8 +247,8 @@(deftest check-specific-cards-test(testing "Can check specific cards by entity-id"- (let [source (make-test-source)- results (check-specific source ["simple-orders" "native-orders"])]+ (let [sources (make-test-sources)+ results (check-specific sources ["simple-orders" "native-orders"])](is (>= (count results) 2) "Should check at least 2 cards")(is (contains? results "simple-orders"))(is (contains? results "native-orders"))@@ -1211,5 +1216,5 @@(fixtures-path);; Manually check all cards in fixtures- (def source (make-test-source))- (check-all source))+ (def sources (make-test-sources))+ (check-all sources))