Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Metadata File Import (#74524)

Timothy S. Dean2fe12d63ee94
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 4654efeccf1..35074be78cb 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -429,6 +429,7 @@
instaparse.core/defparser clojure.core/def
metabase-enterprise.action-v2.test-util/with-test-tables! clojure.core/let
metabase-enterprise.content-translation.routes-test/with-new-secret-key! clojure.core/fn
+ metabase-enterprise.serialization.metadata-file-import.wire-format-test/with-tiny-fixture! clojure.core/fn
metabase-enterprise.serialization.test-util/with-dbs clojure.core/fn
metabase-enterprise.serialization.test-util/with-random-dump-dir clojure.core/let
metabase-enterprise.workspaces.test-util/with-resources! clojure.core/let
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index f343247aba0..ab49279d886 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3308,6 +3308,7 @@
:api #{metabase-enterprise.serialization.api
metabase-enterprise.serialization.cmd
metabase-enterprise.serialization.core
+ metabase-enterprise.serialization.init
metabase-enterprise.serialization.v2.backfill-ids}
:uses #{analytics
api
@@ -3324,7 +3325,9 @@
search
server
setup
- util}
+ sync
+ util
+ warehouse-schema}
:model-imports :bypass}
enterprise/snippet-collections
diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.clj
index 31dc597a85e..8abb55639c3 100644
--- a/enterprise/backend/src/metabase_enterprise/core/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj
@@ -20,6 +20,7 @@
[metabase-enterprise.scim.init]
[metabase-enterprise.security-center.init]
[metabase-enterprise.semantic-search.init]
+ [metabase-enterprise.serialization.init]
[metabase-enterprise.sso.init]
[metabase-enterprise.stale.init]
[metabase-enterprise.support-access-grants.init]
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
index 440a8399c21..39a5e8d1d4c 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
@@ -4,6 +4,7 @@
[clojure.string :as str]
[java-time.api :as t]
[metabase-enterprise.serialization.export :as export]
+ [metabase-enterprise.serialization.metadata-file-import :as metadata-file-import]
[metabase-enterprise.serialization.schema :as schema]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase-enterprise.serialization.v2.ingest :as v2.ingest]
@@ -26,9 +27,10 @@
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[metabase.util.random :as u.random]
- [ring.core.protocols :as ring.protocols])
+ [ring.core.protocols :as ring.protocols]
+ [ring.util.response :as response])
(:import
- (java.io ByteArrayOutputStream File)))
+ (java.io ByteArrayOutputStream File InputStream)))
(set! *warn-on-reflection* true)
@@ -332,6 +334,83 @@
(sr/streaming-response {:content-type "application/json; charset=utf-8"} [os _]
(export/export-metadata! os opts))))
+;;; ----------------------------------- POST /api/ee/serialization/metadata/import -----------------------------------
+
+(defn- spool-to-temp-file!
+ "Stream `is` to a freshly-created temp file in 64 KB chunks. Returns the
+ `File`. If streaming fails the temp file is removed before the throw
+ propagates."
+ ^File [^InputStream is]
+ (let [tmp (File/createTempFile "metadata-import-" ".json")]
+ (try
+ (with-open [os (io/output-stream tmp)]
+ (io/copy is os :buffer-size (* 64 1024)))
+ tmp
+ (catch Throwable t
+ (.delete tmp)
+ (throw t)))))
+
+(api.macros/defendpoint :post "/metadata/import"
+ :- [:map
+ [:status [:= 202]]
+ [:body [:map {:closed true}
+ [:queued :boolean]
+ [:import-id :string]]]]
+ "Import warehouse metadata previously emitted by `POST /metadata/export`. The
+ request body is the JSON document `{databases, tables, fields}`; sections are
+ parsed incrementally so memory stays bounded regardless of payload size.
+
+ To bypass the JSON-parsing request middleware, send with `Content-Type:
+ application/octet-stream`. Restricted to superusers.
+
+ Returns `202` immediately with an `:import-id`; the import runs
+ asynchronously. Poll `GET /metadata/import/:id` for its outcome."
+ [_route-params
+ _query-params
+ _body
+ {:keys [body], :as _request}]
+ (api/check-superuser)
+ (cond
+ (nil? body)
+ (throw (ex-info "Empty request body" {:status-code 400}))
+
+ (not (instance? InputStream body))
+ (throw (ex-info (str "Expected a raw stream body. Send the request with "
+ "Content-Type: application/octet-stream so the JSON "
+ "middleware does not pre-parse the payload.")
+ {:status-code 415}))
+
+ :else
+ (let [tmp (spool-to-temp-file! body)
+ id (metadata-file-import/enqueue-import! tmp {:delete-after? true})]
+ (-> (response/response {:queued true :import-id id})
+ (assoc :status 202)))))
+
+;;; ------------------------------- GET /api/ee/serialization/metadata/import/:id -------------------------------
+
+(defn- present-import-status
+ "Project an internal registry record onto the public wire shape. Deliberately
+ omits the server-side `:file` path; stringifies the status keyword and the
+ timestamps."
+ [record]
+ {:id (:id record)
+ :status (name (:status record))
+ :enqueued-at (str (:enqueued-at record))
+ :started-at (some-> (:started-at record) str)
+ :finished-at (some-> (:finished-at record) str)
+ :wall-ms (:wall-ms record)
+ :error (:error record)})
+
+(api.macros/defendpoint :get "/metadata/import/:id"
+ :- ::schema/import-status-response
+ "Status of a metadata import previously started by `POST /metadata/import`.
+ Status is retained in-memory and is not durable across server restarts.
+ Restricted to superusers."
+ [{:keys [id]} :- [:map [:id ms/UUIDString]]]
+ (api/check-superuser)
+ (or (some-> (metadata-file-import/import-status id) present-import-status)
+ (throw (ex-info "Unknown or expired import id" {:status-code 404}))))
+
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/serialization` routes."
(api.macros/ns-handler *ns* +auth))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/core.clj b/enterprise/backend/src/metabase_enterprise/serialization/core.clj
index a1a88a32d9e..d4269100354 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/core.clj
@@ -1,6 +1,10 @@
(ns metabase-enterprise.serialization.core
(:require
[metabase-enterprise.serialization.dump]
+ [metabase-enterprise.serialization.export]
+ ;; Required (without alias) so loading `serialization.core` populates the malli
+ ;; registry — callers can then refer to `::serialization.schema/...` keywords.
+ [metabase-enterprise.serialization.schema]
[metabase-enterprise.serialization.v2.extract]
[metabase-enterprise.serialization.v2.ingest]
[metabase-enterprise.serialization.v2.load]
@@ -11,6 +15,7 @@
(comment
metabase-enterprise.serialization.dump/keep-me
+ metabase-enterprise.serialization.export/keep-me
metabase-enterprise.serialization.v2.extract/keep-me
metabase-enterprise.serialization.v2.ingest/keep-me
metabase-enterprise.serialization.v2.load/keep-me
@@ -21,6 +26,8 @@
(p/import-vars
[metabase-enterprise.serialization.dump
serialization-deep-sort]
+ [metabase-enterprise.serialization.export
+ export-metadata!]
[metabase-enterprise.serialization.v2.extract
make-targets-of-type
extract]
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/init.clj b/enterprise/backend/src/metabase_enterprise/serialization/init.clj
new file mode 100644
index 00000000000..b6cd5a49416
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/init.clj
@@ -0,0 +1,5 @@
+(ns metabase-enterprise.serialization.init
+ (:require
+ [metabase-enterprise.serialization.metadata-file-import :as metadata-file-import]))
+
+(metadata-file-import/init!)
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import.clj
new file mode 100644
index 00000000000..99c549d229f
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import.clj
@@ -0,0 +1,308 @@
+(ns metabase-enterprise.serialization.metadata-file-import
+ "Streams a metadata file into the appdb, populating
+ `:model/Database` (matched, never created), `:model/Table`, and `:model/Field`.
+
+ Composes three building blocks:
+
+ - [[metabase-enterprise.serialization.metadata-file-import.parsers]] streams
+ batches of `[line-num row]` tuples from a JSON file.
+ - [[metabase-enterprise.serialization.metadata-file-import.processors]] provides
+ the per-batch and per-depth SQL building blocks.
+ - [[metabase-enterprise.serialization.metadata-file-import.schemas]] defines
+ the per-row Malli schemas, shared with the parsers' callers.
+
+ Strict-consistency invariants enforced by the pre-flight + depth-tagging
+ steps mean the merge loop has no orphan-handling code paths. The whole
+ merge is set-based SQL, one statement per phase per depth level."
+ (:require
+ [metabase-enterprise.serialization.metadata-file-import.parsers :as parsers]
+ [metabase-enterprise.serialization.metadata-file-import.processors :as processors]
+ [metabase.app-db.core :as mdb]
+ [metabase.sync.util :as sync.util]
+ [metabase.util.log :as log]
+ [metabase.warehouse-schema.models.table :as schema.table]
+ [toucan2.core :as t2])
+ (:import
+ (java.io File)
+ (java.sql Connection PreparedStatement)
+ (java.time Instant)
+ (java.util UUID)
+ (org.postgresql PGConnection)
+ (org.postgresql.copy CopyIn CopyManager)))
+
+(set! *warn-on-reflection* true)
+
+;;; ============================== drain + database matching ==============================
+
+(defn- process-databases-batch!
+ "Per-batch handler for `:databases`. Updates `databases-by-source-id`
+ with `source-id → name` for every row, conjs target-db-ids onto
+ `matched-ids` for matches, and WARN-logs unmatched rows."
+ [matched-ids databases-by-source-id batch]
+ (reduce (fn [_ result]
+ (vswap! databases-by-source-id assoc (:source-id result) (:name result))
+ (case (:status result)
+ :matched
+ (vswap! matched-ids conj (:target-id result))
+
+ :no-match
+ (log/warnf "metadata-file-import: skipped source database %s (no matching target): %s"
+ (pr-str (:name result)) (:detail result)))
+ nil)
+ nil
+ (processors/process-databases! batch)))
+
+(defn- drain-via-jdbc-batch!
+ "Drive parser batches through JDBC `executeBatch` against per-staging-table
+ `PreparedStatement`s. The fallback path for non-Postgres app-dbs."
+ [^Connection conn ^File file matched-ids databases-by-source-id]
+ (with-open [^PreparedStatement tables-ps (.prepareStatement conn (processors/tables-insert-sql))
+ ^PreparedStatement fields-ps (.prepareStatement conn (processors/fields-insert-sql))]
+ (parsers/stream-keyed-arrays!
+ file processors/import-batch-size
+ {:databases (partial process-databases-batch! matched-ids databases-by-source-id)
+ :tables (fn [batch] (processors/drain-tables-batch-jdbc! tables-ps @databases-by-source-id batch))
+ :fields (fn [batch] (processors/drain-fields-batch-jdbc! fields-ps batch))})))
+
+;; Only one COPY can be active per connection at a time, so we track the
+;; current handle (`:tables` or `:fields`) and close it before opening the next.
+(defn- drain-via-pg-copy!
+ "Drive parser batches through PG's COPY wire protocol via `CopyManager`,
+ streaming TSV-formatted bytes directly to the server."
+ [^Connection conn ^File file matched-ids databases-by-source-id]
+ (let [^PGConnection pg-conn (.unwrap conn PGConnection)
+ ^CopyManager copy-mgr (.getCopyAPI pg-conn)
+ current (volatile! {:kind nil :copy-in nil})
+ close-current! (fn []
+ (when-let [^CopyIn ci (:copy-in @current)]
+ (.endCopy ci)
+ (vreset! current {:kind nil :copy-in nil})))
+ ensure-copy! (fn [kind ^String sql]
+ (when (not= kind (:kind @current))
+ (close-current!)
+ (vreset! current {:kind kind :copy-in (.copyIn copy-mgr sql)})))]
+ (try
+ (parsers/stream-keyed-arrays!
+ file processors/import-batch-size
+ {:databases (partial process-databases-batch! matched-ids databases-by-source-id)
+ :tables (fn [batch]
+ (ensure-copy! :tables processors/tables-copy-sql)
+ (processors/drain-tables-batch-pg-copy! (:copy-in @current) @databases-by-source-id batch))
+ :fields (fn [batch]
+ (ensure-copy! :fields processors/fields-copy-sql)
+ (processors/drain-fields-batch-pg-copy! (:copy-in @current) batch))})
+ (close-current!)
+ (catch Throwable t
+ (try (close-current!) (catch Throwable _ nil))
+ (throw t)))))
+
+;; Opens one JDBC connection with autoCommit=false and commits at the end.
+;; c3p0's autoCommitOnClose=false means we must explicitly restore
+;; autoCommit=true before returning the connection to the pool — otherwise the
+;; next pool user inherits an open-transaction-capable connection and can leak
+;; an idle-in-transaction backend.
+(defn- drain-and-match-databases!
+ "Single-pass walk of `file`: match `:databases` against the live appdb,
+ drain `:tables` and `:fields` into staging. Returns the set of matched
+ target-db-ids."
+ [^File file]
+ (let [matched-ids (volatile! #{})
+ databases-by-source-id (volatile! {})
+ ds (mdb/data-source)]
+ (with-open [^Connection conn (.getConnection ds)]
+ (.setAutoCommit conn false)
+ (try
+ ;; PG COPY fast path; JDBC executeBatch fallback for H2/MySQL.
+ (case (mdb/db-type)
+ :postgres (drain-via-pg-copy! conn file matched-ids databases-by-source-id)
+ (drain-via-jdbc-batch! conn file matched-ids databases-by-source-id))
+ (.commit conn)
+ (catch Throwable t
+ (try (.rollback conn) (catch Throwable _ nil))
+ (throw t))
+ (finally
+ (try (.setAutoCommit conn true) (catch Throwable _ nil)))))
+ @matched-ids))
+
+;;; ============================== Top-level orchestration ==============================
+
+;; Writes via raw `:metabase_database` to bypass `:model/Database`'s
+;; `:after-update` hook, which would schedule per-database Quartz sync triggers
+;; — undesirable here, since the whole point of metadata-file-import is to skip
+;; warehouse sync. App startup re-registers triggers.
+(defn- mark-databases-sync-complete!
+ "Flip every matched target Database's `initial_sync_status` to `\"complete\"`
+ so the UI surfaces tables immediately."
+ [matched-target-db-ids]
+ (when (seq matched-target-db-ids)
+ (t2/query {:update :metabase_database
+ :set {:initial_sync_status "complete"}
+ :where [:in :id (vec matched-target-db-ids)]})))
+
+(defn import-metadata-file!
+ "Run the full metadata import pipeline against the given file.
+
+ `metadata-file` — `java.io.File` or path-string for the metadata file
+ (databases / tables / fields).
+
+ Returns `:ok` on success; throws on hard-fail conditions (file not
+ readable, file incomplete, cycle in reference graph, mid-merge errors)."
+ [metadata-file]
+ (let [^File m-file (if (instance? File metadata-file)
+ metadata-file
+ (File. ^String metadata-file))]
+ ;; with-staging-tables clears the staging tables on entry and exit
+ ;; (try/finally), so a crashed prior attempt can't leak rows into this run.
+ (processors/with-staging-tables
+ ;; --- drain + database matching (single pass over the file) ---
+ (let [matched-target-db-ids (drain-and-match-databases! m-file)]
+ ;; --- analyze: feed the planner real selectivity for downstream UPDATEs ---
+ (processors/analyze-staging-tables!)
+ ;; --- pre-flight: every parent ref must resolve within the file ---
+ (processors/assert-no-orphan-refs!)
+ ;; --- pre-flight: NULL orphan fk-target refs (informational, not fatal) ---
+ (let [{:keys [count sample]} (processors/null-orphan-fk-target-refs!)]
+ (when (pos? count)
+ (log/warnf "metadata-file-import: %d orphan fk_target_field_id ref(s) NULLed (target not present in file); sample=%s"
+ count (pr-str sample))))
+ ;; --- depth tagging: assign 0..max-depth, detect cycles ---
+ (processors/compute-staging-depth!)
+ ;; --- table resolve round 1: matched-vs-insert decision for merge-tables! ---
+ (processors/resolve-target-table-ids-in-staging!)
+ ;; --- merge (one txn — every live-data write all-or-nothing) ---
+ (let [new-tables
+ (t2/with-transaction [_]
+ (let [insert-source-ids (processors/merge-tables!)]
+ ;; round 2 captures INSERT-assigned ids back into table staging
+ (processors/resolve-target-table-ids-in-staging!)
+ ;; copy target-table-ids from table staging onto field staging
+ (processors/resolve-target-table-ids-for-fields-in-staging!)
+ ;; depth-walk: per-depth resolve + UPDATE matched + INSERT new
+ (processors/merge-fields-by-depth!)
+ (mark-databases-sync-complete! matched-target-db-ids)
+ (processors/new-target-tables-from-staging insert-source-ids)))]
+ ;; Permission grants run outside the merge txn — the cluster lock
+ ;; shouldn't be held across the long merge work.
+ (doseq [[db-id rows] (group-by :db_id new-tables)]
+ (schema.table/set-new-tables-permissions! db-id rows)))
+ (log/infof "metadata-file-import: complete (matched-databases=%d)"
+ (count matched-target-db-ids))))
+ :ok))
+
+;;; ============================== Import registry + agent ==============================
+;;; In-JVM only — matches the scope of `metabase.sync.util/operation->db-ids`.
+;;; Status is not durable across server restarts.
+
+(def ^:private terminal-record-limit
+ "Max number of completed (`:ok`/`:error`) records retained in the registry.
+ In-flight (`:queued`/`:running`) records are never evicted regardless of count."
+ 10)
+
+;; id -> {:id :status :file :enqueued-at :started-at :finished-at :wall-ms :error}
+(defonce ^:private import-registry (atom {}))
+
+(defonce ^:private import-agent
+ (agent ::serializer :error-mode :continue))
+
+;; The id currently being run by the agent, or nil. A single-element index over
+;; `import-registry` so the sync busy-check is O(1) instead of scanning.
+(defonce ^:private running-import-id (atom nil))
+
+(defn- terminal-status? [status]
+ (contains? #{:ok :error} status))
+
+(defn- prune-terminal-records
+ "Drop the oldest terminal records past `terminal-record-limit`, ordered by
+ `:finished-at`. Non-terminal records are retained regardless of count."
+ [registry]
+ (let [terminal (filter (comp terminal-status? :status val) registry)]
+ (if (<= (count terminal) terminal-record-limit)
+ registry
+ (->> terminal
+ (sort-by (comp :finished-at val))
+ (drop-last terminal-record-limit)
+ (map key)
+ (apply dissoc registry)))))
+
+(defn- finalize-record
+ "Merge terminal `attrs` onto record `id`, then prune."
+ [registry id attrs]
+ (-> registry
+ (update id merge attrs)
+ prune-terminal-records))
+
+(defn import-status
+ "Registry record for import `id`, or nil if the id is unknown or has been
+ evicted."
+ [id]
+ (get @import-registry id))
+
+(defn import-running?
+ "Truthy iff a metadata-file-import is currently in flight on this JVM."
+ []
+ (some? @running-import-id))
+
+(defn- import-busy-reason []
+ (when-let [id @running-import-id]
+ (let [{:keys [file started-at]} (get @import-registry id)]
+ {:reason (format "metadata-file-import in progress (file=%s, since=%s)"
+ file started-at)})))
+
+(defonce ^:private initialized? (atom false))
+
+(defn init!
+ "Register hooks with the subsystems this module coordinates with (sync).
+ Called from `metabase-enterprise.serialization.init` at boot. Idempotent —
+ callers don't need to guard against multiple invocations."
+ []
+ (when (compare-and-set! initialized? false true)
+ (sync.util/register-busy-predicate! import-busy-reason)))
+
+(defn- run-import*
+ "Agent body. Always returns `::serializer`; `:error-mode :continue` plus
+ the inner try/catch means a failing import never poisons the agent."
+ [_serializer id ^File file {:keys [delete-after?]}]
+ (let [path (.getAbsolutePath file)
+ t0 (System/nanoTime)]
+ (swap! import-registry update id merge {:status :running :started-at (Instant/now)})
+ (reset! running-import-id id)
+ (log/infof "metadata-file-import: starting (id=%s, file=%s)" id path)
+ (try
+ (import-metadata-file! file)
+ (let [wall-ms (/ (- (System/nanoTime) t0) 1e6)]
+ (log/infof "metadata-file-import: complete (id=%s, file=%s, wall-ms=%.0f)" id path wall-ms)
+ (swap! import-registry finalize-record id
+ {:status :ok :finished-at (Instant/now) :wall-ms wall-ms :error nil}))
+ (catch Throwable t
+ (let [wall-ms (/ (- (System/nanoTime) t0) 1e6)]
+ (log/errorf t "metadata-file-import: failed (id=%s, file=%s, wall-ms=%.0f)" id path wall-ms)
+ (swap! import-registry finalize-record id
+ {:status :error :finished-at (Instant/now) :wall-ms wall-ms :error (str t)})))
+ (finally
+ (reset! running-import-id nil)
+ (when delete-after?
+ (try (.delete file) (catch Throwable _ nil)))))
+ ::serializer))
+
+(defn enqueue-import!
+ "Submit `file` to the import agent and return its import id (a string UUID).
+ Imports execute in arrival order; a `:queued` registry record is created
+ synchronously before this returns, so the caller can immediately look the id
+ up via `import-status`. With `{:delete-after? true}`, the agent deletes
+ `file` once it finishes (success or failure)."
+ ([^File file]
+ (enqueue-import! file {}))
+ ([^File file opts]
+ (let [id (str (UUID/randomUUID))
+ path (.getAbsolutePath file)]
+ (swap! import-registry assoc id {:id id
+ :status :queued
+ :file path
+ :enqueued-at (Instant/now)
+ :started-at nil
+ :finished-at nil
+ :wall-ms nil
+ :error nil})
+ (log/infof "metadata-file-import: queued (id=%s, file=%s)" id path)
+ (send-off import-agent run-import* id file opts)
+ id)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/parsers.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/parsers.clj
new file mode 100644
index 00000000000..dcf85bbce5b
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/parsers.clj
@@ -0,0 +1,132 @@
+(ns metabase-enterprise.serialization.metadata-file-import.parsers
+ "Streaming JSON parser for the metadata file importer. Walks a top-level JSON
+ object, advances to a named array, and emits its items in batches via a
+ callback. Memory bounded by the in-flight item plus the current batch buffer."
+ (:require
+ [metabase.util.performance :as perf])
+ (:import
+ (com.fasterxml.jackson.core JsonParser JsonToken)
+ (com.fasterxml.jackson.databind ObjectMapper)
+ (java.io File FileInputStream InputStreamReader Reader)
+ (java.nio.charset StandardCharsets)
+ (java.util LinkedHashMap)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private ^ObjectMapper object-mapper (ObjectMapper.))
+
+(defn- advance-to-array!
+ "Advance `parser` from start-of-input through a top-level JSON object to the
+ array valued at `target-key` (parser has just consumed the START_ARRAY
+ token). Throws `:bad-shape` if the document doesn't begin with an object or
+ the value at `target-key` isn't an array; throws `:missing-key` if the key
+ is absent."
+ [^JsonParser parser ^String target-key]
+ (let [t (.nextToken parser)]
+ (when-not (= t JsonToken/START_OBJECT)
+ (throw (ex-info "Expected JSON document to begin with an object"
+ {:kind :bad-shape, :target-key target-key}))))
+ (loop []
+ (let [t (.nextToken parser)]
+ (cond
+ (or (nil? t) (= t JsonToken/END_OBJECT))
+ (throw (ex-info (format "Key %s not found in top-level object" (pr-str target-key))
+ {:kind :missing-key, :key target-key}))
+
+ (= t JsonToken/FIELD_NAME)
+ (if (= target-key (.getCurrentName parser))
+ (let [vt (.nextToken parser)]
+ (when-not (= vt JsonToken/START_ARRAY)
+ (throw (ex-info (format "Value of %s must be an array" (pr-str target-key))
+ {:kind :bad-shape, :key target-key}))))
+ (do (.nextToken parser) (.skipChildren parser) (recur)))
+
+ :else (recur)))))
+
+(defn- consume-array-as-batches!
+ "After START_ARRAY has been consumed, walk array items into batches of
+ `[line-num row]` tuples (up to `batch-size`), calling `process-batch!` per
+ batch. Returns when END_ARRAY is consumed."
+ [^JsonParser parser array-key batch-size process-batch!]
+ (loop [batch (transient [])
+ line-num 0]
+ (let [t (.nextToken parser)]
+ (cond
+ (= t JsonToken/END_ARRAY)
+ (when (pos? (count batch))
+ (process-batch! (persistent! batch)))
+
+ (= t JsonToken/START_OBJECT)
+ ;; LinkedHashMap fails (map? x), so coerce before keywordize-keys.
+ (let [raw (.readValueAs parser LinkedHashMap)
+ item (perf/keywordize-keys (into {} raw))
+ ln (inc line-num)
+ next-batch (conj! batch [ln item])]
+ (if (>= (count next-batch) batch-size)
+ (do (process-batch! (persistent! next-batch))
+ (recur (transient []) ln))
+ (recur next-batch ln)))
+
+ (nil? t)
+ (throw (ex-info (format "Unexpected end-of-input in array %s" (pr-str array-key))
+ {:kind :bad-shape, :key array-key}))
+
+ :else
+ (throw (ex-info (format "Unexpected token %s in array %s"
+ (some-> t .asString) (pr-str array-key))
+ {:kind :bad-shape, :key array-key}))))))
+
+(defn- open-parser
+ ^JsonParser [^Reader reader]
+ (.createParser (.getFactory object-mapper) reader))
+
+(defn stream-array-batches!
+ "Stream `file`'s top-level named array (string or keyword `array-key`) through
+ `process-batch!`. Each batch is a vector of `[line-num row]` tuples (up to
+ `batch-size`); `line-num` is 1-indexed and continues across batch boundaries.
+ Items are parsed into Clojure maps with keyword keys. Caller's responsibility
+ to ensure the file exists and is readable; opening errors propagate."
+ [^File file array-key batch-size process-batch!]
+ (with-open [is (FileInputStream. file)
+ reader (InputStreamReader. is StandardCharsets/UTF_8)
+ parser (open-parser reader)]
+ (advance-to-array! parser (name array-key))
+ (consume-array-as-batches! parser array-key batch-size process-batch!)))
+
+(defn stream-keyed-arrays!
+ "Single-pass walk of `file`'s top-level JSON object, dispatching arrays to
+ per-key handlers. `handlers` is a map of keyword → fn-of-batch (each batch is
+ a vector of `[line-num row]` tuples, up to `batch-size`). Arrays for keys not
+ in `handlers` are skipped without materialization.
+
+ Throws `:bad-shape` if the document doesn't begin with an object or if a
+ known key's value isn't an array. Missing keys are silently OK."
+ [^File file batch-size handlers]
+ (with-open [is (FileInputStream. file)
+ reader (InputStreamReader. is StandardCharsets/UTF_8)
+ parser (open-parser reader)]
+ (let [t (.nextToken parser)]
+ (when-not (= t JsonToken/START_OBJECT)
+ (throw (ex-info "Expected JSON document to begin with an object"
+ {:kind :bad-shape}))))
+ (loop []
+ (let [t (.nextToken parser)]
+ (cond
+ (or (nil? t) (= t JsonToken/END_OBJECT))
+ nil
+
+ (= t JsonToken/FIELD_NAME)
+ (let [field-name (.getCurrentName parser)
+ handler-fn (get handlers (keyword field-name))]
+ (if handler-fn
+ (let [vt (.nextToken parser)]
+ (when-not (= vt JsonToken/START_ARRAY)
+ (throw (ex-info (format "Value of %s must be an array" (pr-str field-name))
+ {:kind :bad-shape, :key field-name})))
+ (consume-array-as-batches! parser field-name batch-size handler-fn)
+ (recur))
+ (do (.nextToken parser)
+ (.skipChildren parser)
+ (recur))))
+
+ :else (recur))))))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/processors.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/processors.clj
new file mode 100644
index 00000000000..898a1adb765
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/processors.clj
@@ -0,0 +1,834 @@
+(ns metabase-enterprise.serialization.metadata-file-import.processors
+ "SQL building blocks for the metadata file importer's drain-then-merge flow.
+
+ Errors propagate as `ex-info` with `:kind` for typed failure handling
+ (`:invalid-input`, `:file-incomplete`, `:cycle-in-field-graph`, `:depth-tagging-cap-exceeded`)."
+ (:require
+ [malli.error :as me]
+ [metabase-enterprise.serialization.metadata-file-import.schemas :as schemas]
+ [metabase.app-db.core :as mdb]
+ [metabase.models.humanization :as humanization]
+ [metabase.util.json :as json]
+ [metabase.util.malli.registry :as mr]
+ [toucan2.core :as t2])
+ (:import
+ (java.nio.charset StandardCharsets)
+ (java.sql PreparedStatement Types)
+ (org.postgresql.copy CopyIn)))
+
+(set! *warn-on-reflection* true)
+
+(def import-batch-size
+ "Row batch size shared by the drain and merge processors."
+ ;; 250 was the sweet spot in local benchmarks
+ 250)
+
+;;; ============================== Validation ==============================
+
+(defn- validate-line!
+ "Validate `line` against the registered Malli `schema-ref`.
+ On failure throws `ex-info` with `:kind :invalid-input`, `:line`, a humanized `:detail`,
+ and `extras` merged into the ex-data."
+ [schema-ref line-num line extras]
+ (when-let [humanized (me/humanize (mr/explain schema-ref line))]
+ (throw (ex-info (format "Invalid %s row on line %d" (name schema-ref) line-num)
+ (merge extras
+ {:kind :invalid-input
+ :line line-num
+ :detail (pr-str humanized)})))))
+
+;;; ============================== Staging tables ==============================
+
+(defn clear-staging-tables!
+ "Empty `metabase_table_import` and `metabase_field_import`."
+ []
+ ;; TRUNCATE (not DELETE) so multiple imports in one process lifetime don't
+ ;; accumulate dead tuples in staging.
+ ;; Called outside any outer transaction, so MySQL's implicit-commit-on-
+ ;; TRUNCATE doesn't break composition.
+ (t2/query {:truncate :metabase_table_import})
+ (t2/query {:truncate :metabase_field_import}))
+
+(defn analyze-staging-tables!
+ "Refresh planner statistics on `metabase_table_import` and `metabase_field_import`."
+ []
+ (case (mdb/db-type)
+ :postgres
+ (do (t2/query "ANALYZE metabase_table_import")
+ (t2/query "ANALYZE metabase_field_import"))
+ :mysql
+ (do (t2/query "ANALYZE TABLE metabase_table_import")
+ (t2/query "ANALYZE TABLE metabase_field_import"))
+ :h2
+ (t2/query "ANALYZE")))
+
+(defmacro with-staging-tables
+ "Run `body` with staging tables cleared on entry and on exit."
+ [& body]
+ `(do (clear-staging-tables!)
+ (try ~@body
+ (finally (clear-staging-tables!)))))
+
+;;; ============================== databases (batch) ==============================
+
+(defn- engine-name
+ "Normalize `engine` (string or keyword) to a string for natural-key comparison.
+ Toucan2's `:model/Database` reads `:engine` as a keyword via `define-after-select`,
+ but file-imported rows always carry strings. Normalize both sides to string."
+ [engine]
+ (when engine (name engine)))
+
+(defn- match-databases-batch
+ "Look up every existing target Database matching any source row in the
+ batch. Returns `{[name engine-string] → existing-id}` for matching rows."
+ [lines]
+ (let [pairs (into #{}
+ (map (fn [{:keys [name engine]}] [name (engine-name engine)]))
+ lines)
+ names (into #{} (map first) pairs)
+ engines (into #{} (map second) pairs)]
+ (if (or (empty? names) (empty? engines))
+ {}
+ ;; Over-include via `(name IN ..., engine IN ...)` then intersect in
+ ;; Clojure — keeps the SQL portable (no `(col, col) IN ((?, ?), ...)`
+ ;; tuple form across Postgres / H2 / MySQL).
+ (let [rows (t2/select [:model/Database :id :name :engine]
+ {:where [:and
+ [:in :name names]
+ [:in :engine engines]]})]
+ (into {}
+ (comp (map (fn [{:keys [id name engine]}]
+ [[name (engine-name engine)] id]))
+ (filter (fn [[pair _]] (contains? pairs pair))))
+ rows)))))
+
+(defn process-databases!
+ "Process a batch of database rows. Returns an eduction of result maps.
+
+ Result shapes:
+ `{:source-id <int> :name <string> :target-id <int> :status :matched}`
+ `{:source-id <int> :name <string> :status :no-match :line L :detail S}`
+
+ Validation failures throw; lookup misses produce `:no-match` results
+ (non-fatal)."
+ [batch]
+ (doseq [[ln line] batch]
+ (validate-line! ::schemas/database-info ln line {:source-id (:id line)}))
+ (let [match-idx (match-databases-batch (mapv second batch))]
+ (eduction
+ (map (fn [[ln {:keys [id name engine]}]]
+ (if-let [target (match-idx [name (engine-name engine)])]
+ {:source-id id :name name :target-id target :status :matched}
+ {:source-id id :name name
+ :status :no-match
+ :line ln
+ :detail (format "No database with name=%s engine=%s"
+ (pr-str name) (pr-str (engine-name engine)))})))
+ batch)))
+
+;;; ============================== tables — drain + merge ==============================
+
+(defn- encode-path-or-nil
+ "Encode an `:nfc_path` coll as a JSON string. NULL for empty/nil."
+ [coll]
+ (when (seq coll) (json/encode (vec coll))))
+
+;;; ============================== JDBC drain (perf path) ==============================
+;;;
+;;; Hand-rolled JDBC `executeBatch` against single per-staging-table prepared
+;;; statements. Used by the loader's `drain-and-match-databases!` to bypass
+;;; the t2/insert! + HoneySQL VALUES formatting overhead — measured ~15×
+;;; slower per row than PG's COPY floor on multi-million-row drains.
+
+(defn tables-insert-sql
+ "Parameterized INSERT for `metabase_table_import`. Column order matches the
+ `set*-batch!` parameter-binding order in [[drain-tables-batch-jdbc!]]."
+ []
+ (str "INSERT INTO metabase_table_import"
+ ;; `schema` is a reserved word in MySQL/MariaDB
+ " (source_id, source_db_id, db_name, " (mdb/quote-for-application-db "schema")
+ ", name, description, display_name)"
+ " VALUES (?, ?, ?, ?, ?, ?, ?)"))
+
+(defn fields-insert-sql
+ "Parameterized INSERT for `metabase_field_import`. Column order matches the
+ `set*-batch!` parameter-binding order in [[drain-fields-batch-jdbc!]]."
+ []
+ (str "INSERT INTO metabase_field_import"
+ " (source_id, source_table_id, source_parent_id, source_fk_target_id,"
+ " name, base_type, database_type, effective_type, semantic_type,"
+ " coercion_strategy, description, nfc_path)"
+ " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"))
+
+(defn- set-int-or-null! [^PreparedStatement ps ^long idx v]
+ (if v
+ (.setInt ps idx (int v))
+ (.setNull ps idx Types/INTEGER)))
+
+(defn- set-string-or-null! [^PreparedStatement ps ^long idx v]
+ (if v
+ (.setString ps idx (str v))
+ (.setNull ps idx Types/VARCHAR)))
+
+(defn drain-tables-batch-jdbc!
+ "Per-batch handler for `:tables` that binds rows onto `ps` and flushes via
+ `executeBatch`. Validates each row first; throws on schema failure.
+
+ `ps` must be a `PreparedStatement` for [[tables-insert-sql]]; the caller
+ owns the connection + statement lifecycle."
+ [^PreparedStatement ps databases-by-source-id batch]
+ (doseq [[ln line] batch]
+ (validate-line! ::schemas/table-info ln line {:source-id (:id line)}))
+ (when (seq batch)
+ (doseq [[_ {:keys [id db_id schema name description]}] batch]
+ (.setInt ps 1 (int id))
+ (.setInt ps 2 (int db_id))
+ (.setString ps 3 (str (get databases-by-source-id db_id)))
+ (set-string-or-null! ps 4 schema)
+ (.setString ps 5 (str name))
+ (set-string-or-null! ps 6 description)
+ (.setString ps 7 (humanization/name->human-readable-name name))
+ (.addBatch ps))
+ (.executeBatch ps)))
+
+(defn drain-fields-batch-jdbc!
+ "Per-batch handler for `:fields` that binds rows onto `ps` and flushes via
+ `executeBatch`. Validates each row first; throws on schema failure.
+
+ `ps` must be a `PreparedStatement` for [[fields-insert-sql]]; the caller
+ owns the connection + statement lifecycle."
+ [^PreparedStatement ps batch]
+ (doseq [[ln line] batch]
+ (validate-line! ::schemas/field-info ln line {:source-id (:id line)}))
+ (when (seq batch)
+ (doseq [[_ {:keys [id table_id parent_id fk_target_field_id
+ name base_type database_type
+ effective_type semantic_type
+ coercion_strategy description nfc_path]}] batch]
+ (.setInt ps 1 (int id))
+ (.setInt ps 2 (int table_id))
+ (set-int-or-null! ps 3 parent_id)
+ (set-int-or-null! ps 4 fk_target_field_id)
+ (.setString ps 5 (str name))
+ (.setString ps 6 (str base_type))
+ (.setString ps 7 (str database_type))
+ (set-string-or-null! ps 8 effective_type)
+ (set-string-or-null! ps 9 semantic_type)
+ (set-string-or-null! ps 10 coercion_strategy)
+ (set-string-or-null! ps 11 description)
+ (set-string-or-null! ps 12 (encode-path-or-nil nfc_path))
+ (.addBatch ps))
+ (.executeBatch ps)))
+
+;;; ============================== PG-specific COPY drain ==============================
+;;;
+;;; Drives staging-table inserts through PostgreSQL's COPY wire protocol via
+;;; `CopyManager` / `CopyIn`. Streams TSV-formatted row bytes directly to the
+;;; server — no per-statement parsing, no JDBC parameter binding overhead.
+;;; Approaches PG's raw bulk-load throughput; in practice ~5× faster than the
+;;; JDBC `executeBatch` path at multi-million-row scale.
+
+(def tables-copy-sql
+ "PostgreSQL COPY statement for `metabase_table_import`. Column order matches
+ the TSV emitted by [[table-tsv-line]]."
+ (str "COPY metabase_table_import"
+ " (source_id, source_db_id, db_name, schema, name, description, display_name)"
+ " FROM STDIN"))
+
+(def fields-copy-sql
+ "PostgreSQL COPY statement for `metabase_field_import`. Column order matches
+ the TSV emitted by [[field-tsv-line]]."
+ (str "COPY metabase_field_import"
+ " (source_id, source_table_id, source_parent_id, source_fk_target_id,"
+ " name, base_type, database_type, effective_type, semantic_type,"
+ " coercion_strategy, description, nfc_path)"
+ " FROM STDIN"))
+
+(defn- tsv-escape ^String [^String s]
+ (-> s
+ (.replace "\\" "\\\\")
+ (.replace "\t" "\\t")
+ (.replace "\n" "\\n")
+ (.replace "\r" "\\r")))
+
+(defn- tsv-cell ^String [v]
+ (cond
+ (nil? v) "\\N"
+ (number? v) (str v)
+ (boolean? v) (str v)
+ :else (tsv-escape (str v))))
+
+(defn- table-tsv-line ^String [row databases-by-source-id]
+ (let [{:keys [id db_id schema name description]} row]
+ (str (tsv-cell id) "\t"
+ (tsv-cell db_id) "\t"
+ (tsv-cell (get databases-by-source-id db_id)) "\t"
+ (tsv-cell schema) "\t"
+ (tsv-cell name) "\t"
+ (tsv-cell description) "\t"
+ (tsv-cell (humanization/name->human-readable-name name)) "\n")))
+
+(defn- field-tsv-line ^String [row]
+ (let [{:keys [id table_id parent_id fk_target_field_id
+ name base_type database_type
+ effective_type semantic_type
+ coercion_strategy description nfc_path]} row]
+ (str (tsv-cell id) "\t"
+ (tsv-cell table_id) "\t"
+ (tsv-cell parent_id) "\t"
+ (tsv-cell fk_target_field_id) "\t"
+ (tsv-cell name) "\t"
+ (tsv-cell base_type) "\t"
+ (tsv-cell database_type) "\t"
+ (tsv-cell effective_type) "\t"
+ (tsv-cell semantic_type) "\t"
+ (tsv-cell coercion_strategy) "\t"
+ (tsv-cell description) "\t"
+ (tsv-cell (encode-path-or-nil nfc_path)) "\n")))
+
+(defn drain-tables-batch-pg-copy!
+ "Per-batch handler for `:tables` that streams TSV rows over PG's COPY wire
+ protocol via the supplied `CopyIn` handle. Validates each row first."
+ [^CopyIn copy-in databases-by-source-id batch]
+ (doseq [[ln line] batch]
+ (validate-line! ::schemas/table-info ln line {:source-id (:id line)}))
+ (doseq [[_ row] batch]
+ (let [^bytes bs (.getBytes (table-tsv-line row databases-by-source-id) StandardCharsets/UTF_8)]
+ (.writeToCopy copy-in bs 0 (alength bs)))))
+
+(defn drain-fields-batch-pg-copy!
+ "Per-batch handler for `:fields` that streams TSV rows over PG's COPY wire
+ protocol via the supplied `CopyIn` handle. Validates each row first."
+ [^CopyIn copy-in batch]
+ (doseq [[ln line] batch]
+ (validate-line! ::schemas/field-info ln line {:source-id (:id line)}))
+ (doseq [[_ row] batch]
+ (let [^bytes bs (.getBytes (field-tsv-line row) StandardCharsets/UTF_8)]
+ (.writeToCopy copy-in bs 0 (alength bs)))))
+
+(defn resolve-target-table-ids-in-staging!
+ "Set `metabase_table_import.target_id` to the int id of the matching
+ `metabase_table` row for every staging row whose match key resolves. The
+ match key is `(db_name, schema, name)` against `(d.name, t.schema, t.name)`,
+ restricted to active, non-defective live rows.
+
+ Rows that do not match keep `target_id` NULL. Inactive matches are
+ deliberately not resolved so a re-import after a deactivation creates a
+ fresh active row.
+
+ Idempotent — running again with the same staging contents produces the
+ same assignments."
+ []
+ (t2/query
+ {:update :metabase_table_import
+ :set {:target_id
+ ;; MIN(t.id): `metabase_database`'s unique constraint is
+ ;; `(router_database_id, name)`, so two NULL-router databases
+ ;; can share a name → two `(db_name, schema, name)` matches
+ ;; (GHY-3549).
+ {:select [[[:min :t.id]]]
+ :from [[:metabase_table :t]]
+ :join [[:metabase_database :d] [:= :d.id :t.db_id]]
+ :where [:and
+ [:= :metabase_table_import.db_name :d.name]
+ [:= [:coalesce :t.schema [:inline ""]]
+ [:coalesce :metabase_table_import.schema [:inline ""]]]
+ [:= :t.name :metabase_table_import.name]
+ [:= :t.is_defective_duplicate [:inline false]]
+ [:= :t.active [:inline true]]]}}}))
+
+;; UPDATE before INSERT: on a clean-schema import the UPDATE matches nothing
+;; (fast no-op) and the INSERT then writes each row exactly once.
+;;
+;; Callers re-run [[resolve-target-table-ids-in-staging!]] and pass the set to
+;; [[new-target-tables-from-staging]] to read back the freshly-created rows.
+(defn merge-tables!
+ "Merge `metabase_table_import` into `metabase_table` atomically: UPDATE the
+ staging rows that matched a live table, INSERT the rest. Returns the set of
+ staging `source_id`s that were INSERTed (snapshot taken before the INSERT).
+
+ Pre-condition: [[resolve-target-table-ids-in-staging!]] must have run, so
+ matched staging rows carry a `target_id`."
+ []
+ (t2/with-transaction [_]
+ ;; t2 joins an outer txn rather than creating a savepoint, so wrapping
+ ;; this call in a larger transaction is safe.
+
+ ;; Capture which staging rows are about to be INSERTed (target_id NULL).
+ ;; After the INSERT + a subsequent resolve round, these source_ids will
+ ;; have their `target_id` populated with the freshly-inserted live id.
+ (let [insert-source-ids
+ (into #{} (map :source_id)
+ (t2/query {:select [:source_id]
+ :from [:metabase_table_import]
+ :where [:= :target_id nil]}))]
+ ;; UPDATE matched rows first (clobber description, bump updated_at).
+ ;; Skip-if-unchanged: the EXISTS subquery additionally requires that
+ ;; description actually differs (NULL-safe via COALESCE-with-empty-
+ ;; string), so a re-import with identical description is a true no-op
+ ;; — no UPDATE fires, no dead tuple. updated_at is deliberately NOT
+ ;; in the predicate (otherwise the UPDATE would always fire,
+ ;; defeating the optimization).
+ (t2/query
+ {:update :metabase_table
+ :set {:description {;; ORDER BY + LIMIT 1: two staging rows can
+ ;; resolve to the same `target_id` (GHY-3549).
+ :select [:it.description]
+ :from [[:metabase_table_import :it]]
+ :where [:= :it.target_id :metabase_table.id]
+ :order-by [[:it.source_id :asc]]
+ :limit 1}
+ :updated_at :%now}
+ :where [:and
+ [:= :metabase_table.is_defective_duplicate [:inline false]]
+ [:= :metabase_table.active [:inline true]]
+ [:exists {:select [[[:inline 1]]]
+ :from [[:metabase_table_import :it]]
+ :where [:and
+ [:= :it.target_id :metabase_table.id]
+ [:!= [:coalesce :metabase_table.description [:inline ""]]
+ [:coalesce :it.description [:inline ""]]]]}]]})
+ ;; INSERT bypasses :model/Table's :after-insert hook — that hook
+ ;; schedules per-DB Quartz triggers we don't want and fires
+ ;; `set-new-table-permissions!` once per row. The JOIN on db_name
+ ;; silently drops staging rows whose source DB has no target appdb.
+ (t2/query
+ {:insert-into
+ [[:metabase_table [:db_id :schema :name :description :display_name :data_layer
+ :active :show_in_getting_started :is_defective_duplicate
+ :created_at :updated_at]]
+ {:select [:d.id :it.schema :it.name :it.description :it.display_name
+ [[:inline "internal"]]
+ [[:inline true]] [[:inline false]] [[:inline false]]
+ :%now :%now]
+ :from [[:metabase_table_import :it]]
+ :join [[:metabase_database :d] [:= :d.name :it.db_name]]
+ :where [:= :it.target_id nil]}]})
+ insert-source-ids)))
+
+(defn new-target-tables-from-staging
+ "Look up the `metabase_table` rows that were just INSERTed by
+ [[merge-tables!]]. `insert-source-ids` is the set returned by
+ [[merge-tables!]]. Pre-condition:
+ [[resolve-target-table-ids-in-staging!]] has run since the INSERT, so
+ staging's `target_id` is populated for these rows."
+ [insert-source-ids]
+ (when (seq insert-source-ids)
+ (t2/query {:select [:t.id :t.db_id :t.schema]
+ :from [[:metabase_table :t]]
+ :join [[:metabase_table_import :it] [:= :it.target_id :t.id]]
+ :where [:in :it.source_id (vec insert-source-ids)]})))
+
+;;; ============================== Pre-flight orphan check ==============================
+
+(def ^:private orphan-sample-cap
+ "How many orphan rows to surface in the error data when bailing out. Visibility
+ without exploding the exception payload."
+ 10)
+
+(defn- orphan-not-exists-predicate
+ "WHERE-clause fragment matching `metabase_field_import` rows whose `column-key` (a `:source_*_id` column)
+ is non-null but references no other staging row's `source_id` — i.e. an orphan reference."
+ [column-key]
+ ;; Correlated NOT EXISTS, not NOT IN: at 9M+ rows PG can't plan
+ ;; `NOT IN (subquery)` (NULL-semantics force full-materialization of the
+ ;; inner set) and it effectively never terminates. NOT EXISTS resolves to a
+ ;; proper anti-join via the PK index on `source_id` — O(N log N), not O(N²).
+ ;;
+ ;; The inner FROM is a `(SELECT source_id FROM ...)` derived table so the
+ ;; predicate stays valid inside an UPDATE on `metabase_field_import`: MySQL
+ ;; forbids referencing the UPDATE's target table directly in a subquery; the
+ ;; wrap defeats that, and PG/H2 inline it away. Aliased `:s` so the inner
+ ;; WHERE's unqualified column-key resolves to the outer table.
+ (let [outer-col (keyword (str "metabase_field_import." (name column-key)))]
+ [:and
+ [:not= column-key nil]
+ [:not [:exists
+ {:select [[[:inline 1]]]
+ :from [[{:select [:source_id]
+ :from [:metabase_field_import]} :s]]
+ :where [:= :s.source_id outer-col]}]]]))
+
+(defn- orphan-count
+ "Number of `metabase_field_import` rows whose `column-key` (a `:source_*_id`
+ column) is non-null but doesn't reference any other row's `source_id`."
+ [column-key]
+ (t2/count :metabase_field_import {:where (orphan-not-exists-predicate column-key)}))
+
+(defn- orphan-sample
+ "Up to `orphan-sample-cap` `[source_id, <column-key>]` pairs for orphan rows."
+ [column-key]
+ (t2/query
+ {:select [:source_id column-key]
+ :from [:metabase_field_import]
+ :where (orphan-not-exists-predicate column-key)
+ :limit orphan-sample-cap}))
+
+(defn assert-no-orphan-refs!
+ "Pre-flight check after drain: every staging field row's `source_parent_id`
+ must reference another staging row's `source_id`. Throws `ex-info` with
+ `:kind :file-incomplete` and a sample of orphan rows in the error data
+ when any orphan parent ref exists.
+
+ Parent refs are structural — a field claiming to be a child of a missing
+ field can't be positioned, so the file is genuinely incomplete. Orphan
+ `source_fk_target_id` refs are not fatal here; see
+ [[null-orphan-fk-target-refs!]]."
+ []
+ (let [parent-count (orphan-count :source_parent_id)]
+ (when (pos? parent-count)
+ (throw (ex-info (format "metadata-file-import: file is incomplete — %d orphan parent ref(s)"
+ parent-count)
+ {:kind :file-incomplete
+ :orphan-parent-count parent-count
+ :orphan-parent-sample (orphan-sample :source_parent_id)})))))
+
+(defn null-orphan-fk-target-refs!
+ "Find staging rows whose `source_fk_target_id` points at a `source_id`
+ not present in staging, and NULL the `source_fk_target_id` column on
+ those rows. Returns `{:count N :sample [...]}` when at least one row was
+ scrubbed, `{:count 0}` otherwise."
+ []
+ ;; Why NULL instead of abort: fk_target refs that cross a hidden/archived
+ ;; table boundary on the source side (the table is `active=false` or
+ ;; `visibility_type='hidden'`) survive in `metabase_field` but get filtered
+ ;; out by the export's table-visibility join. The exported file then contains
+ ;; fk_target refs whose targets aren't emitted — a real-data shape we observe
+ ;; in production appdbs. Treating these as fatal would block legitimate
+ ;; imports; treating them as no-fk is the lossless choice for the importer.
+ (let [n (orphan-count :source_fk_target_id)]
+ (if (zero? n)
+ {:count 0}
+ (let [sample (orphan-sample :source_fk_target_id)]
+ (t2/query
+ {:update :metabase_field_import
+ :set {:source_fk_target_id nil}
+ :where (orphan-not-exists-predicate :source_fk_target_id)})
+ {:count n :sample sample}))))
+
+;;; ============================== Depth tagging ==============================
+
+(def depth-iteration-cap
+ "Iteration cap on the depth-tagging fixpoint loop; exceeding it throws
+ `:cycle-in-field-graph` with the un-tagged sample."
+ ;; Field parent chains are shallow in practice (a few levels at most), so 50
+ ;; is a comfortable safety belt — exceeding it means the algorithm is the
+ ;; bug, not the data.
+ 50)
+
+(defn- mark-roots-at-depth-zero!
+ "Tag every staging row with no parent and no fk_target ref as `depth = 0`.
+ Bootstraps the iteration."
+ []
+ (t2/query
+ {:update :metabase_field_import
+ :set {:depth 0}
+ :where [:and
+ [:= :source_parent_id nil]
+ [:= :source_fk_target_id nil]]}))
+
+(defn- mark-rows-at-depth!
+ "Tag every still-untagged staging row whose `source_parent_id` and
+ `source_fk_target_id` (when non-NULL) reference rows that already have
+ `depth < d`."
+ [d]
+ ;; MySQL forbids referencing the UPDATE's target table directly inside a
+ ;; subquery; wrap in a derived table there. PG/H2 accept the direct form.
+ (let [staging-source (case (mdb/db-type)
+ :mysql {:select [:source_id :depth]
+ :from [:metabase_field_import]}
+ :metabase_field_import)]
+ (t2/query
+ {:update :metabase_field_import
+ :set {:depth d}
+ :where [:and
+ [:= :metabase_field_import.depth nil]
+ [:or
+ [:= :metabase_field_import.source_parent_id nil]
+ [:exists {:select [[[:inline 1]]]
+ :from [[staging-source :p]]
+ :where [:and
+ [:= :p.source_id :metabase_field_import.source_parent_id]
+ [:not= :p.depth nil]
+ [:< :p.depth d]]}]]
+ [:or
+ [:= :metabase_field_import.source_fk_target_id nil]
+ [:exists {:select [[[:inline 1]]]
+ :from [[staging-source :f]]
+ :where [:and
+ [:= :f.source_id :metabase_field_import.source_fk_target_id]
+ [:not= :f.depth nil]
+ [:< :f.depth d]]}]]]})))
+
+(defn- untagged-staging-row-count
+ "Number of `metabase_field_import` rows still at `depth IS NULL`. Used as
+ the convergence signal in `compute-staging-depth!`."
+ []
+ (t2/count :metabase_field_import :depth nil))
+
+(defn- untagged-staging-row-sample
+ "Up to `orphan-sample-cap` un-tagged rows for cycle-error diagnostics."
+ []
+ (t2/query
+ {:select [:source_id :source_parent_id :source_fk_target_id]
+ :from [:metabase_field_import]
+ :where [:= :depth nil]
+ :limit orphan-sample-cap}))
+
+;; The fixpoint loop terminates on convergence (no untagged rows) or a
+;; no-progress round. The `assert-no-orphan-refs!` pre-condition is what lets
+;; us read no-progress as a cycle: with orphans already ruled out, a row that
+;; can never be tagged must be part of one.
+(defn compute-staging-depth!
+ "Tag every `metabase_field_import` row with a non-NULL `depth`: 0 for roots
+ (no parent, no fk_target ref), `d` for rows whose deps are all at depth
+ < `d`. Returns the maximum depth in staging.
+
+ Pre-condition: [[assert-no-orphan-refs!]] must have run successfully.
+
+ Throws `:cycle-in-field-graph` if rows can't be tagged (a reference cycle)
+ or `:depth-tagging-cap-exceeded` if the loop runs past [[depth-iteration-cap]]."
+ []
+ (mark-roots-at-depth-zero!)
+ (loop [d 1]
+ (let [before (untagged-staging-row-count)]
+ (cond
+ (zero? before)
+ nil
+
+ (>= d depth-iteration-cap)
+ (throw (ex-info (format "metadata-file-import: depth-tagging exceeded cap of %d iterations" depth-iteration-cap)
+ {:kind :depth-tagging-cap-exceeded
+ :iterations d
+ :remaining-rows-count before
+ :remaining-rows-sample (untagged-staging-row-sample)}))
+
+ :else
+ (do
+ (mark-rows-at-depth! d)
+ (let [after (untagged-staging-row-count)]
+ (if (= before after)
+ (throw (ex-info (format "metadata-file-import: %d staging row(s) could not be tagged with depth — cycle in file's parent/fk_target reference graph"
+ after)
+ {:kind :cycle-in-field-graph
+ :remaining-rows-count after
+ :iterations d
+ :remaining-rows-sample (untagged-staging-row-sample)}))
+ (recur (inc d))))))))
+ (or (:max (first (t2/query {:select [[[:max :depth] :max]]
+ :from [:metabase_field_import]})))
+ 0))
+
+;;; ============================== Per-depth field merge ==============================
+
+(def ^:private field-clobber-cols
+ "The field payload columns the importer owns: on a matched row they are
+ overwritten from the file's values (the import is an alternate sync)."
+ ;; Each is also its staging column name, so [[update-matched-fields-at-depth!]]
+ ;; can build the SET from identically-named correlated subqueries.
+ [:base_type :database_type :description
+ :effective_type :semantic_type :coercion_strategy :nfc_path])
+
+(defn- target-id-clobber-subquery
+ "Correlated subquery yielding `staging-col`'s value from the staging row
+ that resolved to this `metabase_field`."
+ [staging-col]
+ {:select [(keyword (str "fi." (name staging-col)))]
+ :from [[:metabase_field_import :fi]]
+ :where [:= :fi.target_id :metabase_field.id]
+ ;; ORDER BY + LIMIT 1: two staging rows can resolve to the same target_id
+ ;; (GHY-3549); pick one deterministically rather than erroring.
+ :order-by [[:fi.source_id :asc]]
+ :limit 1})
+
+(def ^:private field-payload-changed-predicate
+ "True when a matched `metabase_field` row's observable payload differs from
+ its staging row — the predicate behind the merge UPDATE's skip-if-unchanged
+ behavior."
+ ;; Coalesce-with-sentinel is the portable NULL-safe `IS DISTINCT FROM`:
+ ;; empty string for text columns, -1 (never a real id) for FK ids.
+ [[:!= [:coalesce :metabase_field.base_type [:inline ""]] [:coalesce :fi.base_type [:inline ""]]]
+ [:!= [:coalesce :metabase_field.database_type [:inline ""]] [:coalesce :fi.database_type [:inline ""]]]
+ [:!= [:coalesce :metabase_field.description [:inline ""]] [:coalesce :fi.description [:inline ""]]]
+ ;; `effective_type` coalesces against `base_type`: the export omits it when
+ ;; the two are equal, and the app reads NULL effective_type as base_type.
+ [:!= [:coalesce :metabase_field.effective_type :metabase_field.base_type]
+ [:coalesce :fi.effective_type :fi.base_type]]
+ [:!= [:coalesce :metabase_field.semantic_type [:inline ""]] [:coalesce :fi.semantic_type [:inline ""]]]
+ [:!= [:coalesce :metabase_field.coercion_strategy [:inline ""]] [:coalesce :fi.coercion_strategy [:inline ""]]]
+ [:!= [:coalesce :metabase_field.nfc_path [:inline ""]] [:coalesce :fi.nfc_path [:inline ""]]]
+ ;; staging's FK column is `target_fk_target_id`, not `fk_target_field_id`.
+ [:!= [:coalesce :metabase_field.fk_target_field_id [:inline -1]] [:coalesce :fi.target_fk_target_id [:inline -1]]]
+ ;; `active` compares against TRUE — the merge SET always sets it TRUE.
+ [:!= :metabase_field.active [:inline true]]])
+
+(defn resolve-target-table-ids-for-fields-in-staging!
+ "Populate `metabase_field_import.target_table_id` by joining through
+ `metabase_table_import.source_id → metabase_table_import.target_id`.
+
+ Pre-condition: `metabase_table_import.target_id` must be populated
+ (via [[resolve-target-table-ids-in-staging!]]) for both matched and
+ newly-inserted target rows.
+
+ Field rows whose source table has no target (orphan-source-database
+ case) keep `target_table_id NULL`."
+ []
+ (t2/query
+ {:update :metabase_field_import
+ :set {:target_table_id
+ {:select [:ti.target_id]
+ :from [[:metabase_table_import :ti]]
+ :where [:= :ti.source_id :metabase_field_import.source_table_id]}}}))
+
+(defn fill-target-parent-ids-at-depth!
+ "Populate `target_parent_id` for depth-`d` staging rows by self-joining
+ staging on `source_parent_id → source_id` and reading the parent's
+ `target_id`. Pre-condition: all prior-depth staging rows have `target_id`
+ resolved (the depth walk processes lower depths first, so this is
+ guaranteed when called in order)."
+ [d]
+ ;; Same MySQL same-table-in-UPDATE workaround as [[mark-rows-at-depth!]].
+ (let [parent-source (case (mdb/db-type)
+ :mysql {:select [:source_id :target_id]
+ :from [:metabase_field_import]}
+ :metabase_field_import)]
+ (t2/query
+ {:update :metabase_field_import
+ :set {:target_parent_id
+ {:select [:p.target_id]
+ :from [[parent-source :p]]
+ :where [:= :p.source_id :metabase_field_import.source_parent_id]}}
+ :where [:and
+ [:= :metabase_field_import.depth d]
+ [:not= :metabase_field_import.source_parent_id nil]]})))
+
+(defn fill-target-fk-target-ids-at-depth!
+ "Populate `target_fk_target_id` for depth-`d` staging rows by self-joining
+ staging on `source_fk_target_id → source_id`."
+ [d]
+ ;; Same MySQL same-table-in-UPDATE workaround as [[mark-rows-at-depth!]].
+ (let [fk-source (case (mdb/db-type)
+ :mysql {:select [:source_id :target_id]
+ :from [:metabase_field_import]}
+ :metabase_field_import)]
+ (t2/query
+ {:update :metabase_field_import
+ :set {:target_fk_target_id
+ {:select [:f.target_id]
+ :from [[fk-source :f]]
+ :where [:= :f.source_id :metabase_field_import.source_fk_target_id]}}
+ :where [:and
+ [:= :metabase_field_import.depth d]
+ [:not= :metabase_field_import.source_fk_target_id nil]]})))
+
+(defn resolve-target-field-ids-at-depth!
+ "Set `target_id` for depth-`d` staging rows by natural-key match against `metabase_field`. Matches
+ by `(target_table_id, name, target_parent_id)` with NULL-safe parent comparison (a staging row with
+ no parent matches only against `metabase_field.parent_id IS NULL`). Skips defective duplicates on
+ the target side.
+
+ Run twice per depth — once before the UPDATE/INSERT step (to identify matches for the clobber path)
+ and once after (to capture INSERT ids so higher-depth rows can find them as parents/FK targets)."
+ [d]
+ (t2/query
+ {:update :metabase_field_import
+ :set {:target_id
+ ;; MIN(f.id) tiebreaker — `metabase_field` has no uniqueness
+ ;; constraint on `(table_id, name, parent_id)` (GHY-3549).
+ {:select [[[:min :f.id]]]
+ :from [[:metabase_field :f]]
+ :where [:and
+ [:= :f.table_id :metabase_field_import.target_table_id]
+ [:= :f.name :metabase_field_import.name]
+ [:or
+ [:and [:= :metabase_field_import.target_parent_id nil]
+ [:= :f.parent_id nil]]
+ [:= :f.parent_id :metabase_field_import.target_parent_id]]
+ [:= :f.is_defective_duplicate [:inline false]]]}}
+ :where [:and
+ [:= :metabase_field_import.depth d]
+ [:= :metabase_field_import.target_id nil]]}))
+
+(defn update-matched-fields-at-depth!
+ "Bring the live `metabase_field` rows that depth-`d` staging matched into line with the file. Rows
+ whose payload already matches staging are left untouched."
+ [d]
+ (t2/query
+ {:update :metabase_field
+ ;; `fk_target_field_id` reads staging's `target_fk_target_id` — it isn't
+ ;; part of the natural-key match, so a matched field's FK can drift.
+ :set (-> (into {} (map (fn [c] [c (target-id-clobber-subquery c)])) field-clobber-cols)
+ (assoc :fk_target_field_id (target-id-clobber-subquery :target_fk_target_id)
+ :active [:inline true]
+ :updated_at :%now))
+ :where [:and
+ [:= :metabase_field.is_defective_duplicate [:inline false]]
+ ;; Scope the outer walk to the ~staging-row-count IDs at depth `d`.
+ ;; Without this, PG starts from `metabase_field` and probes staging
+ ;; per row — a full scan on an appdb with millions of field rows.
+ [:in :metabase_field.id {:select [:target_id]
+ :from [:metabase_field_import]
+ :where [:and
+ [:= :depth d]
+ [:not= :target_id nil]]}]
+ ;; Skip-if-unchanged: only touch rows whose payload actually differs.
+ [:exists {:select [[[:inline 1]]]
+ :from [[:metabase_field_import :fi]]
+ :where [:and
+ [:= :fi.target_id :metabase_field.id]
+ [:= :fi.depth d]
+ (into [:or] field-payload-changed-predicate)]}]]}))
+
+(defn insert-new-fields-at-depth!
+ "INSERT `metabase_field` for depth-`d` staging rows that didn't match a
+ live row (`target_id IS NULL`). The new row's `parent_id` and
+ `fk_target_field_id` come from staging's `target_parent_id` /
+ `target_fk_target_id` — populated earlier in this depth's iteration.
+
+ Rows whose `target_table_id IS NULL` (source table didn't match target)
+ are silently dropped. Rows whose `source_parent_id IS NOT NULL` but
+ `target_parent_id IS NULL` are also dropped — that's the case where the
+ parent field's table didn't match target, cascading the orphan downward."
+ [d]
+ (t2/query
+ {:insert-into
+ [[:metabase_field [:table_id :name :base_type :database_type :description
+ :effective_type :semantic_type :coercion_strategy
+ :nfc_path :parent_id :fk_target_field_id
+ :is_defective_duplicate :active :created_at :updated_at]]
+ {:select [:fi.target_table_id :fi.name :fi.base_type :fi.database_type :fi.description
+ :fi.effective_type :fi.semantic_type :fi.coercion_strategy
+ :fi.nfc_path :fi.target_parent_id :fi.target_fk_target_id
+ [[:inline false]] [[:inline true]] :%now :%now]
+ :from [[:metabase_field_import :fi]]
+ :where [:and
+ [:= :fi.depth d]
+ [:= :fi.target_id nil]
+ [:not= :fi.target_table_id nil]
+ [:or
+ [:= :fi.source_parent_id nil]
+ [:not= :fi.target_parent_id nil]]]}]}))
+
+(defn merge-fields-by-depth!
+ "Merge `metabase_field_import` into `metabase_field`, walking depth 0 to
+ max-depth so a row's parents and FK targets are resolved before the row
+ itself. All depths run inside one transaction.
+
+ Pre-conditions: [[compute-staging-depth!]] has populated `depth` and
+ [[resolve-target-table-ids-for-fields-in-staging!]] has populated
+ `target_table_id`."
+ []
+ (let [max-d (or (:max (first (t2/query {:select [[[:max :depth] :max]]
+ :from [:metabase_field_import]})))
+ 0)]
+ ;; One transaction across all depths — t2 joins the import's outer txn.
+ (t2/with-transaction [_]
+ (doseq [d (range (inc max-d))]
+ (fill-target-parent-ids-at-depth! d)
+ (fill-target-fk-target-ids-at-depth! d)
+ (resolve-target-field-ids-at-depth! d)
+ (update-matched-fields-at-depth! d)
+ (insert-new-fields-at-depth! d)
+ ;; Re-resolve to capture INSERT ids, so higher depths can find these
+ ;; rows as parents / FK targets.
+ (resolve-target-field-ids-at-depth! d)))))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/schemas.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/schemas.clj
new file mode 100644
index 00000000000..6b5ab8968ae
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/schemas.clj
@@ -0,0 +1,64 @@
+(ns metabase-enterprise.serialization.metadata-file-import.schemas
+ "Malli schemas describing the wire-format rows streamed by
+ `GET /api/ee/serialization/metadata/export` and consumed by
+ `POST /api/ee/serialization/metadata/import`.
+
+ IDs are **integers from the source appdb**. They are internally consistent
+ within one file (so `:fields[].parent_id` references a `:fields[].id` in the
+ same file) but mean nothing across files.
+
+ Cross-row references:
+ - `:tables[].db_id` → `:databases[].id`
+ - `:fields[].table_id` → `:tables[].id`
+ - `:fields[].parent_id` → another `:fields[].id` in the same file
+ - `:fields[].fk_target_field_id` → another `:fields[].id` in the same file
+
+ Optional columns (everything outside the always-present set per row type) are
+ *omitted from the row* by the export's `remove-nils` step rather than emitted
+ as `null`. Schemas mark these as `{:optional true} [:maybe T]` — absent and
+ `null` are accepted equivalently.
+
+ `:effective_type` is special-cased: emitted only when `≠ :base_type`. Importer
+ treats absent as `≡ :base_type`."
+ (:require
+ [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::database-info
+ "Database row. All three keys are required."
+ [:map
+ [:id :int]
+ [:name :string]
+ [:engine :string]])
+
+(mr/def ::table-info
+ "Table row. `:id`, `:db_id`, and `:name` always present. `:schema` and
+ `:description` are optional and may be nil."
+ [:map
+ [:id :int]
+ [:db_id :int]
+ [:name :string]
+ [:schema {:optional true} [:maybe :string]]
+ [:description {:optional true} [:maybe :string]]])
+
+(mr/def ::field-info
+ "Field row. Five required keys + several optional, per the export's
+ `metadata-query-format :model/Field`."
+ [:map
+ [:id :int]
+ [:table_id :int]
+ [:name :string]
+ [:base_type :string]
+ [:database_type :string]
+ [:parent_id {:optional true} [:maybe :int]]
+ [:fk_target_field_id {:optional true} [:maybe :int]]
+ [:description {:optional true} [:maybe :string]]
+ [:effective_type {:optional true} [:maybe :string]]
+ [:semantic_type {:optional true} [:maybe :string]]
+ [:coercion_strategy {:optional true} [:maybe :string]]
+ [:nfc_path {:optional true}
+ [:maybe
+ [:fn {:error/message "must be a sequence of strings"}
+ (fn [x] (and (instance? java.util.List x)
+ (every? string? x)))]]]])
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj b/enterprise/backend/src/metabase_enterprise/serialization/schema.clj
index d05dfa1c6ec..815adb38d2a 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/schema.clj
@@ -67,3 +67,16 @@
[:databases {:optional true} [:sequential ::exported-database]]
[:tables {:optional true} [:sequential ::exported-table]]
[:fields {:optional true} [:sequential ::exported-field]]])
+
+(mr/def ::import-status-response
+ "Wire shape of `GET /metadata/import/:id`. Timestamps are ISO-8601 strings.
+ `:started-at`/`:finished-at`/`:wall-ms` are nil until the relevant lifecycle
+ stage is reached; `:error` is nil unless the import failed."
+ [:map {:closed true}
+ [:id :string]
+ [:status [:enum "queued" "running" "ok" "error"]]
+ [:enqueued-at :string]
+ [:started-at [:maybe :string]]
+ [:finished-at [:maybe :string]]
+ [:wall-ms [:maybe number?]]
+ [:error [:maybe :string]]])
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index ee08740fee9..c28b539add8 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -6,6 +6,7 @@
[clojure.walk :as walk]
[medley.core :as m]
[metabase-enterprise.serialization.api :as api.serialization]
+ [metabase-enterprise.serialization.metadata-file-import :as metadata-file-import]
[metabase-enterprise.serialization.v2.ingest :as v2.ingest]
[metabase.analytics.snowplow-test :as snowplow-test]
[metabase.models.serialization :as serdes]
@@ -14,10 +15,11 @@
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
[metabase.util.compress :as u.compress]
+ [metabase.util.json :as json]
[metabase.util.random :as u.random]
[toucan2.core :as t2])
(:import
- (java.io File)
+ (java.io ByteArrayInputStream File)
(org.apache.commons.compress.archivers.tar TarArchiveEntry TarArchiveInputStream)
(org.apache.commons.compress.compressors.gzip GzipCompressorInputStream)))
@@ -668,6 +670,151 @@
"Serialization"
(mt/user-http-request :crowberto :post 402 "ee/serialization/metadata/export")))))
+;;; --------------------------------------- /metadata/import ---------------------------------------
+
+(defn- import-metadata!
+ "Helper: JSON-encode `payload` and POST it as an octet-stream body to
+ `/api/ee/serialization/metadata/import`."
+ ([payload] (import-metadata! :crowberto 202 payload))
+ ([user expected-status payload]
+ (mt/user-http-request
+ user :post expected-status "ee/serialization/metadata/import"
+ {:request-options
+ {:headers {"content-type" "application/octet-stream"}
+ :body (ByteArrayInputStream.
+ (.getBytes ^String (json/encode payload) "UTF-8"))}})))
+
+(defn- import-status!
+ "Helper: GET `/api/ee/serialization/metadata/import/:id`."
+ ([id] (import-status! :crowberto 200 id))
+ ([user expected-status id]
+ (mt/user-http-request
+ user :get expected-status (str "ee/serialization/metadata/import/" id))))
+
+(deftest metadata-import-roundtrip-test
+ (testing "POST /api/ee/serialization/metadata/import is the inverse of POST /metadata/export"
+ ;; The agent runs the import on a separate connection from the test thread;
+ ;; mt/with-temp's default rollback-only wrapper would hide the temp rows
+ ;; from that connection. Real commits + auto-sync disabled lets the agent
+ ;; see what the test set up.
+ ;; Warm test-data before disabling sync: the :model/Field with-temp default resolves
+ ;; `(data/id :checkins)`, which needs a synced test-data DB.
+ (mt/db)
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"
+ :description "round trip"}
+ :model/Field _ {:table_id t-id
+ :base_type :type/Integer
+ :database_type "BIGINT"
+ :semantic_type :type/PK}]
+ (let [exported (mt/user-http-request :crowberto :post 202
+ "ee/serialization/metadata/export"
+ :with-databases true
+ :with-tables true
+ :with-fields true)
+ before-tables (t2/count :model/Table :db_id db-id)
+ before-fields (t2/count :model/Field :table_id t-id)
+ resp (import-metadata! exported)]
+ (is (true? (:queued resp)) "POST acknowledges the import was queued")
+ (is (string? (:import-id resp)) "POST returns a string import-id")
+ ;; The import endpoint hands the file off to an agent and returns
+ ;; immediately. Block until the agent has drained it, then assert
+ ;; the agent didn't silently swallow an exception (without this
+ ;; check, the row-count assertions below would falsely pass on a
+ ;; failed re-import since nothing would have changed).
+ (await @#'metadata-file-import/import-agent)
+ (is (= :ok (:status (metadata-file-import/import-status (:import-id resp))))
+ "agent finished the import successfully")
+ (testing "no rows are added by re-importing the same payload"
+ (is (= before-tables (t2/count :model/Table :db_id db-id)))
+ (is (= before-fields (t2/count :model/Field :table_id t-id))))
+ (testing "the table description is preserved"
+ (is (= "round trip"
+ (t2/select-one-fn :description :model/Table t-id)))))))))))
+
+(deftest metadata-import-post-returns-202-and-import-id-test
+ (testing "POST /api/ee/serialization/metadata/import returns 202 with :queued true and a string :import-id"
+ (mt/with-premium-features #{:serialization}
+ (let [resp (import-metadata! :crowberto 202 {:databases [] :tables [] :fields []})]
+ (is (true? (:queued resp)))
+ (is (string? (:import-id resp)) ":import-id is a string")
+ (is (uuid? (java.util.UUID/fromString (:import-id resp)))
+ ":import-id parses as a UUID")
+ (await @#'metadata-file-import/import-agent)))))
+
+(deftest metadata-import-status-endpoint-lifecycle-test
+ (testing "GET /metadata/import/:id reports the import status across its lifecycle"
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{:serialization}
+ (let [resp (import-metadata! :crowberto 202 {:databases [] :tables [] :fields []})
+ id (:import-id resp)]
+ ;; settle the agent, then the GET endpoint should report a terminal status
+ (await @#'metadata-file-import/import-agent)
+ (let [status (import-status! :crowberto 200 id)]
+ (is (= id (:id status)) "the status response echoes the id")
+ (is (string? (:status status)) ":status is serialized as a string")
+ (is (contains? #{"ok" "error"} (:status status))
+ "after the agent settles the status is terminal")
+ (is (contains? status :enqueued-at))
+ (is (contains? status :started-at))
+ (is (contains? status :finished-at))
+ (is (contains? status :wall-ms)))))))))
+
+(deftest metadata-import-status-omits-file-path-test
+ (testing "GET /metadata/import/:id never leaks the server-side temp file path"
+ (mt/with-premium-features #{:serialization}
+ (let [resp (import-metadata! :crowberto 202 {:databases [] :tables [] :fields []})
+ id (:import-id resp)]
+ (await @#'metadata-file-import/import-agent)
+ (let [status (import-status! :crowberto 200 id)]
+ (is (not (contains? status :file))
+ "the wire shape must not include :file")
+ (is (= #{:id :status :enqueued-at :started-at :finished-at :wall-ms :error}
+ (set (keys status)))
+ "the wire shape is exactly the documented key set"))))))
+
+(deftest metadata-import-status-unknown-id-404-test
+ (testing "GET /metadata/import/:id returns 404 for an unknown or evicted id"
+ (mt/with-premium-features #{:serialization}
+ (import-status! :crowberto 404 (str (java.util.UUID/randomUUID))))))
+
+(deftest metadata-import-status-superuser-test
+ (testing "GET /metadata/import/:id — non-admins get a 403"
+ (mt/with-premium-features #{:serialization}
+ (is (= "You don't have permissions to do that."
+ (import-status! :rasta 403 (str (java.util.UUID/randomUUID))))))))
+
+(deftest metadata-import-status-token-feature-test
+ (testing "GET /metadata/import/:id requires the :serialization premium feature"
+ (mt/with-premium-features #{}
+ (mt/assert-has-premium-feature-error
+ "Serialization"
+ (import-status! :crowberto 402 (str (java.util.UUID/randomUUID)))))))
+
+(deftest metadata-import-wrong-content-type-test
+ (testing "POST /api/ee/serialization/metadata/import — JSON content-type is rejected with 415"
+ (mt/with-premium-features #{:serialization}
+ (mt/user-http-request :crowberto :post 415
+ "ee/serialization/metadata/import"
+ {:tables [] :fields []}))))
+
+(deftest metadata-import-superuser-test
+ (testing "POST /api/ee/serialization/metadata/import — non-admins get a 403"
+ (mt/with-premium-features #{:serialization}
+ (is (= "You don't have permissions to do that."
+ (import-metadata! :rasta 403 {:tables [] :fields []}))))))
+
+(deftest metadata-import-token-feature-test
+ (testing "POST /api/ee/serialization/metadata/import requires the :serialization premium feature"
+ (mt/with-premium-features #{}
+ (mt/assert-has-premium-feature-error
+ "Serialization"
+ (import-metadata! :crowberto 402 {:tables [] :fields []})))))
+
(deftest serialization-cleanup-test
(testing "No temp files are left behind after export/import operations"
(let [known-files (set (.list (io/file api.serialization/parent-dir)))]
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/concurrency_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/concurrency_test.clj
new file mode 100644
index 00000000000..9a2e81e3f9f
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/concurrency_test.clj
@@ -0,0 +1,259 @@
+(ns metabase-enterprise.serialization.metadata-file-import.concurrency-test
+ "Tests for the in-JVM concurrency guard on metadata-file-import. The guard
+ is a single Clojure agent serializing all imports plus a busy-predicate
+ registered with `metabase.sync.util` so that warehouse-sync skips with a
+ WARN log line whenever an import is in flight."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.metadata-file-import :as mfi]
+ [metabase.sync.util :as sync.util]
+ [metabase.test :as mt]
+ [metabase.util.json :as json])
+ (:import
+ (java.io File)))
+
+(set! *warn-on-reflection* true)
+
+;; Kondo flags `test-helpers-set-global-values!` inside any fixture as
+;; "destructive in parallel context". Here it is *intentionally* the opt-out
+;; switch — without it the agent runs on a separate connection that can't see
+;; the test's mt/with-temp rollback-only state.
+#_{:clj-kondo/ignore [:metabase/validate-deftest]}
+(use-fixtures :once
+ (fn [thunk]
+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with
+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.
+ (mt/db)
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (mt/test-helpers-set-global-values!
+ (thunk)))))
+
+(defn- tiny-meta-file ^File [data]
+ (let [f (File/createTempFile "mfi-concurrency-" ".json")]
+ (.deleteOnExit f)
+ (spit f (json/encode data))
+ f))
+
+(defn- tiny-payload [db-name]
+ {:databases [{:id 7 :name db-name :engine "postgres"}]
+ :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]
+ :fields [{:id 1000 :table_id 100 :name "x"
+ :base_type "type/Integer" :database_type "integer"}]})
+
+;;; ============================== import-running? ==============================
+
+(deftest import-running-false-when-idle-test
+ (testing "import-running? returns false when no import is in flight"
+ (await @#'mfi/import-agent)
+ (is (false? (mfi/import-running?)))))
+
+(deftest import-running-toggles-around-enqueue-test
+ (testing "import-running? is true mid-flight, false after the agent settles"
+ (mt/with-temp [:model/Database {} {:name "mfi-running-toggles" :engine :postgres}]
+ (let [file (tiny-meta-file (tiny-payload "mfi-running-toggles"))
+ observed (atom nil)
+ real-import mfi/import-metadata-file!]
+ (with-redefs [mfi/import-metadata-file!
+ (fn [f] (reset! observed (mfi/import-running?))
+ (real-import f))]
+ (mfi/enqueue-import! file)
+ (await @#'mfi/import-agent))
+ (is (true? @observed)
+ "import-running? is true while the agent is inside the import fn")
+ (is (false? (mfi/import-running?))
+ "import-running? is false again once the agent finishes")))))
+
+;;; ============================== Serialization ==============================
+
+(deftest two-enqueues-run-sequentially-test
+ (testing "two enqueued imports execute one-at-a-time, in arrival order"
+ (mt/with-temp [:model/Database {} {:name "mfi-seq-1" :engine :postgres}
+ :model/Database {} {:name "mfi-seq-2" :engine :postgres}]
+ (let [f1 (tiny-meta-file (tiny-payload "mfi-seq-1"))
+ f2 (tiny-meta-file (tiny-payload "mfi-seq-2"))
+ events (atom [])
+ real-import mfi/import-metadata-file!]
+ (with-redefs [mfi/import-metadata-file!
+ (fn [^java.io.File f]
+ (swap! events conj [:start (.getName f) (System/nanoTime)])
+ (real-import f)
+ (swap! events conj [:end (.getName f) (System/nanoTime)]))]
+ (mfi/enqueue-import! f1)
+ (mfi/enqueue-import! f2)
+ (await @#'mfi/import-agent))
+ (let [evs @events]
+ (is (= 4 (count evs)) "two starts + two ends")
+ (is (= :start (first (nth evs 0))))
+ (is (= :end (first (nth evs 1))))
+ (is (= :start (first (nth evs 2))))
+ (is (= :end (first (nth evs 3))))
+ (is (= (second (nth evs 0)) (second (nth evs 1)))
+ "first import's start and end are for the same file")
+ (is (= (second (nth evs 2)) (second (nth evs 3)))
+ "second import's start and end are for the same file")
+ (is (<= (nth (nth evs 1) 2) (nth (nth evs 2) 2))
+ "second import does not start until first has finished"))))))
+
+;;; ============================== Agent does not poison ==============================
+
+(deftest agent-survives-failing-import-test
+ (testing "an exception inside the import body does not leave the agent in a failed state"
+ (await @#'mfi/import-agent)
+ (with-redefs [mfi/import-metadata-file!
+ (fn [_f] (throw (ex-info "synthetic" {:kind :test-failure})))]
+ (mfi/enqueue-import! (tiny-meta-file (tiny-payload "mfi-fail")))
+ (await @#'mfi/import-agent))
+ (is (nil? (agent-error @#'mfi/import-agent))
+ "agent did not enter failed state")
+ (is (false? (mfi/import-running?))
+ "state reset to :idle after the failure")))
+
+;;; ============================== Registry ==============================
+
+(deftest enqueue-import-returns-string-id-and-creates-queued-record-test
+ (testing "enqueue-import! returns a string UUID id and synchronously inserts a :queued record"
+ (await @#'mfi/import-agent)
+ ;; Block the agent so we can observe the :queued record before it transitions.
+ (let [release (promise)]
+ (with-redefs [mfi/import-metadata-file! (fn [_f] @release :ok)]
+ (let [id (mfi/enqueue-import! (tiny-meta-file (tiny-payload "mfi-queued")))]
+ (is (string? id) "enqueue-import! returns a string id, not :queued")
+ (is (uuid? (java.util.UUID/fromString id))
+ "the returned id parses as a UUID")
+ (let [record (mfi/import-status id)]
+ (is (some? record) "a registry record exists for the returned id")
+ (is (= id (:id record)))
+ (is (contains? #{:queued :running} (:status record))
+ "record is :queued (or already :running if the agent picked it up)")
+ (is (some? (:enqueued-at record)) ":enqueued-at is set on insert"))
+ (deliver release :go)
+ (await @#'mfi/import-agent))))))
+
+(deftest record-transitions-queued-running-ok-test
+ (testing "the registry record moves :queued -> :running -> :ok through the agent"
+ (mt/with-temp [:model/Database {} {:name "mfi-lifecycle-ok" :engine :postgres}]
+ (await @#'mfi/import-agent)
+ (let [file (tiny-meta-file (tiny-payload "mfi-lifecycle-ok"))
+ id (mfi/enqueue-import! file)]
+ ;; Capturing the record mid-flight is racy; settle the agent and assert
+ ;; the terminal record — the recorded timestamps prove it passed
+ ;; through :running.
+ (await @#'mfi/import-agent)
+ (let [record (mfi/import-status id)]
+ (is (some? record))
+ (is (= :ok (:status record)) "terminal status is :ok on success")
+ (is (some? (:started-at record)) ":started-at set => passed through :running")
+ (is (some? (:finished-at record)) ":finished-at set on completion")
+ (is (number? (:wall-ms record)) ":wall-ms recorded on completion")
+ (is (nil? (:error record)) "no :error on a successful import"))))))
+
+(deftest record-transitions-to-error-on-failing-import-test
+ (testing "a failing import yields an :error record and does not poison the agent"
+ (await @#'mfi/import-agent)
+ (let [id (with-redefs [mfi/import-metadata-file!
+ (fn [_f] (throw (ex-info "synthetic failure" {:kind :test-failure})))]
+ (let [id (mfi/enqueue-import! (tiny-meta-file (tiny-payload "mfi-lifecycle-err")))]
+ (await @#'mfi/import-agent)
+ id))]
+ (is (nil? (agent-error @#'mfi/import-agent))
+ "agent did not enter a failed state")
+ (let [record (mfi/import-status id)]
+ (is (some? record))
+ (is (= :error (:status record)) "terminal status is :error on failure")
+ (is (some? (:started-at record)))
+ (is (some? (:finished-at record)))
+ (is (number? (:wall-ms record)))
+ (is (string? (:error record)) ":error holds a stringified exception")))))
+
+(deftest import-status-nil-for-unknown-id-test
+ (testing "import-status returns nil for an id that was never enqueued"
+ (is (nil? (mfi/import-status (str (java.util.UUID/randomUUID))))
+ "unknown id => nil")
+ (is (nil? (mfi/import-status "not-even-a-uuid"))
+ "garbage id => nil")))
+
+;;; ============================== Eviction ==============================
+
+(defn- terminal-record [i]
+ [(str "id-" i) {:id (str "id-" i) :status :ok
+ :finished-at (java.time.Instant/ofEpochSecond i)}])
+
+(deftest prune-terminal-records-noop-under-bound-test
+ (testing "prune-terminal-records leaves the registry untouched when terminal count is within the bound"
+ (let [limit @#'mfi/terminal-record-limit
+ reg (into {} (map terminal-record) (range (dec limit)))]
+ (is (= reg (#'mfi/prune-terminal-records reg))))))
+
+(deftest prune-terminal-records-drops-oldest-terminal-test
+ (testing "prune-terminal-records caps terminal records at the bound, dropping the oldest by :finished-at"
+ (let [limit @#'mfi/terminal-record-limit
+ extra 5
+ reg (into {} (map terminal-record) (range (+ limit extra)))
+ pruned (#'mfi/prune-terminal-records reg)]
+ (is (= limit (count pruned))
+ "terminal records are capped at the bound")
+ (is (every? #(contains? pruned (str "id-" %)) (range extra (+ limit extra)))
+ "the most-recent `limit` records are retained")
+ (is (not-any? #(contains? pruned (str "id-" %)) (range extra))
+ "the oldest records are evicted"))))
+
+(deftest prune-terminal-records-never-drops-in-flight-test
+ (testing ":queued and :running records are never pruned, regardless of count"
+ (let [limit @#'mfi/terminal-record-limit
+ in-flight (into {} (for [i (range (* 3 limit))]
+ [(str "q-" i) {:id (str "q-" i)
+ :status (if (even? i) :queued :running)}]))
+ terminal (into {} (map terminal-record) (range (* 2 limit)))
+ pruned (#'mfi/prune-terminal-records (merge in-flight terminal))]
+ (is (every? #(contains? pruned (str "q-" %)) (range (* 3 limit)))
+ "every in-flight record survives")
+ (is (= limit (count (filter (comp #{:ok} :status val) pruned)))
+ "terminal records are still capped at the bound"))))
+
+;;; ============================== Sync-side skip ==============================
+
+(deftest sync-skips-when-import-is-running-test
+ (testing "sync.util/do-sync-operation skips its body when an import is in flight"
+ (mt/with-temp [:model/Database db {:name "mfi-sync-skip" :engine :postgres}]
+ (await @#'mfi/import-agent)
+ ;; Drive a real :running record into the registry by blocking the agent
+ ;; inside a redef'd slow import, then exercise the busy-check while it sits.
+ (let [release (promise)]
+ (with-redefs [mfi/import-metadata-file! (fn [_f] @release :ok)]
+ (mfi/enqueue-import! (tiny-meta-file (tiny-payload "mfi-sync-skip")))
+ ;; Wait until the agent has actually flipped a record to :running.
+ (let [deadline (+ (System/currentTimeMillis) 5000)]
+ (while (and (not (mfi/import-running?))
+ (< (System/currentTimeMillis) deadline))
+ (Thread/sleep 10)))
+ (try
+ (is (true? (mfi/import-running?))
+ "an import is in flight before we exercise the busy-check")
+ (let [sync-ran? (atom false)]
+ (sync.util/do-sync-operation
+ :sync-metadata
+ db
+ "test-sync"
+ (fn [] (reset! sync-ran? true) :result-not-checked))
+ (is (false? @sync-ran?)
+ "sync body did NOT run because import-busy short-circuited it"))
+ (finally
+ (deliver release :go)
+ (await @#'mfi/import-agent))))))))
+
+(deftest sync-runs-normally-when-no-import-in-flight-test
+ (testing "control: sync body executes normally when no import is in flight"
+ ;; The namespace fixture sets `disable-auto-sync true`, which itself makes
+ ;; `should-sync?` short-circuit before the busy-check runs. Re-enable it
+ ;; locally so we're actually exercising the busy-check path.
+ (mt/with-temporary-setting-values [disable-auto-sync false]
+ (mt/with-temp [:model/Database db {:name "mfi-sync-control" :engine :postgres}]
+ (await @#'mfi/import-agent)
+ (let [sync-ran? (atom false)]
+ (sync.util/do-sync-operation
+ :sync-metadata
+ db
+ "test-sync-control"
+ (fn [] (reset! sync-ran? true) :result-not-checked))
+ (is (true? @sync-ran?)
+ "sync body DID run because no import is busy"))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/depth_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/depth_test.clj
new file mode 100644
index 00000000000..805ad21befe
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/depth_test.clj
@@ -0,0 +1,205 @@
+(ns metabase-enterprise.serialization.metadata-file-import.depth-test
+ "Tests for `compute-staging-depth!` — the post-drain step that tags every
+ `metabase_field_import` row with a `depth` value, where 0 = root and depth
+ d = max(parent depth, fk_target depth) + 1.
+
+ Pre-condition for the tagging step is that orphan refs have already been
+ rejected by `assert-no-orphan-refs!`. These tests assume that condition
+ holds (every fixture below is consistent)."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.metadata-file-import.processors :as p]
+ [toucan2.core :as t2]))
+
+(defn- insert-staging-fields!
+ "Insert a sequence of partial field rows into `metabase_field_import`,
+ filling in required NOT NULL columns with sensible defaults."
+ [rows]
+ (let [defaults {:name "x" :base_type "type/Integer" :database_type "int"}]
+ (t2/insert! :metabase_field_import
+ (mapv (fn [row] (merge defaults {:source_table_id 1} row)) rows))))
+
+(defn- depth-by-source-id
+ "Return `{source_id → depth}` after tagging."
+ []
+ (into {}
+ (map (juxt :source_id :depth))
+ (t2/query "SELECT source_id, depth FROM metabase_field_import ORDER BY source_id")))
+
+;;; ============================== empty + roots ==============================
+
+(deftest empty-staging-is-noop-test
+ (try
+ (p/clear-staging-tables!)
+ (is (= 0 (p/compute-staging-depth!)))
+ (finally (p/clear-staging-tables!))))
+
+(deftest all-roots-get-depth-zero-test
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 1}
+ {:source_id 2}
+ {:source_id 3}])
+ (is (= 0 (p/compute-staging-depth!))
+ "max depth assigned is 0 when all rows are roots")
+ (is (= {1 0, 2 0, 3 0} (depth-by-source-id)))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== parent chains ==============================
+
+(deftest one-level-parent-chain-test
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 10}
+ {:source_id 11 :source_parent_id 10}])
+ (is (= 1 (p/compute-staging-depth!)))
+ (is (= {10 0, 11 1} (depth-by-source-id)))
+ (finally (p/clear-staging-tables!))))
+
+(deftest three-level-parent-chain-test
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 100}
+ {:source_id 101 :source_parent_id 100}
+ {:source_id 102 :source_parent_id 101}
+ {:source_id 103 :source_parent_id 102}])
+ (is (= 3 (p/compute-staging-depth!)))
+ (is (= {100 0, 101 1, 102 2, 103 3} (depth-by-source-id)))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== fk_target chains ==============================
+
+(deftest fk-target-chain-test
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 200}
+ {:source_id 201 :source_fk_target_id 200}
+ {:source_id 202 :source_fk_target_id 201}])
+ (is (= 2 (p/compute-staging-depth!)))
+ (is (= {200 0, 201 1, 202 2} (depth-by-source-id)))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== mixed (max-depth) semantics ==============================
+
+(deftest depth-is-max-of-parent-and-fk-target-depths-test
+ (try
+ (p/clear-staging-tables!)
+ ;; Build:
+ ;; A (root) — depth 0
+ ;; B → A (parent) — depth 1
+ ;; C → B (parent) — depth 2
+ ;; D → A (parent, depth 1) AND fk_target → C (depth 2)
+ ;; → D should be depth 3, not 2 (max of 1, 2 + 1)
+ (insert-staging-fields! [{:source_id 10}
+ {:source_id 20 :source_parent_id 10}
+ {:source_id 30 :source_parent_id 20}
+ {:source_id 40 :source_parent_id 10 :source_fk_target_id 30}])
+ (is (= 3 (p/compute-staging-depth!)))
+ (is (= {10 0, 20 1, 30 2, 40 3} (depth-by-source-id))
+ "row with parent at depth 1 and fk_target at depth 2 lands at depth 3")
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== unfolded leaves ==============================
+
+(deftest unfolded-leaves-are-roots-test
+ (try
+ (p/clear-staging-tables!)
+ ;; Leaves with no source_parent_id (the leaf isn't materialized as a
+ ;; separate parent storage row) are roots.
+ (insert-staging-fields! [{:source_id 50}
+ {:source_id 51}])
+ (is (= 0 (p/compute-staging-depth!)))
+ (is (= {50 0, 51 0} (depth-by-source-id)))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== disconnected components ==============================
+
+(deftest multiple-disconnected-chains-tag-in-parallel-test
+ (try
+ (p/clear-staging-tables!)
+ ;; Two unrelated chains.
+ (insert-staging-fields! [{:source_id 100} ; root in chain A
+ {:source_id 101 :source_parent_id 100} ; depth 1 chain A
+ {:source_id 200} ; root in chain B
+ {:source_id 201 :source_parent_id 200} ; depth 1 chain B
+ {:source_id 202 :source_parent_id 201}]) ; depth 2 chain B
+ (is (= 2 (p/compute-staging-depth!)))
+ (is (= {100 0, 101 1, 200 0, 201 1, 202 2} (depth-by-source-id)))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== cycle detection ==============================
+
+(deftest two-row-parent-cycle-throws-test
+ (try
+ (p/clear-staging-tables!)
+ ;; X.parent = Y, Y.parent = X — neither is a root, neither can ever
+ ;; be tagged.
+ (insert-staging-fields! [{:source_id 1 :source_parent_id 2}
+ {:source_id 2 :source_parent_id 1}])
+ (let [thrown (try (p/compute-staging-depth!) nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (some? thrown) "should have thrown")
+ (is (= :cycle-in-field-graph (:kind (ex-data thrown))))
+ (is (= 2 (:remaining-rows-count (ex-data thrown))))
+ (is (= 2 (count (:remaining-rows-sample (ex-data thrown))))))
+ (finally (p/clear-staging-tables!))))
+
+(deftest self-referential-parent-id-throws-test
+ ;; A row pointing at itself can never be tagged: it would need depth d
+ ;; where its own depth is < d.
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 5 :source_parent_id 5}])
+ (let [thrown (try (p/compute-staging-depth!) nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (some? thrown))
+ (is (= :cycle-in-field-graph (:kind (ex-data thrown))))
+ (is (= 1 (:remaining-rows-count (ex-data thrown)))))
+ (finally (p/clear-staging-tables!))))
+
+(deftest cycle-via-fk-target-throws-test
+ ;; X.fk_target = Y, Y.fk_target = X — same problem via the FK relationship
+ ;; rather than parent.
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 1 :source_fk_target_id 2}
+ {:source_id 2 :source_fk_target_id 1}])
+ (let [thrown (try (p/compute-staging-depth!) nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (some? thrown))
+ (is (= :cycle-in-field-graph (:kind (ex-data thrown)))))
+ (finally (p/clear-staging-tables!))))
+
+(deftest cycle-with-some-tagged-rows-test
+ ;; Two roots and a 2-cycle. The roots get depth 0; the cycle never
+ ;; resolves. The error data should reflect just the cycle rows.
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 1} ; root
+ {:source_id 2} ; root
+ {:source_id 100 :source_parent_id 101} ; cycle
+ {:source_id 101 :source_parent_id 100}]) ; cycle
+ (let [thrown (try (p/compute-staging-depth!) nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (= :cycle-in-field-graph (:kind (ex-data thrown))))
+ (is (= 2 (:remaining-rows-count (ex-data thrown)))))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== idempotence ==============================
+
+(deftest depth-tagging-is-idempotent-test
+ ;; Running twice on the same data should produce the same result.
+ ;; Useful contract: the merge phase shouldn't have to worry about whether
+ ;; depths are stale.
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 10}
+ {:source_id 11 :source_parent_id 10}
+ {:source_id 12 :source_parent_id 11}])
+ (let [first-result (p/compute-staging-depth!)
+ first-depths (depth-by-source-id)
+ second-result (p/compute-staging-depth!)
+ second-depths (depth-by-source-id)]
+ (is (= first-result second-result))
+ (is (= first-depths second-depths)))
+ (finally (p/clear-staging-tables!))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/drain_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/drain_test.clj
new file mode 100644
index 00000000000..e3632eb74e4
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/drain_test.clj
@@ -0,0 +1,308 @@
+(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.drain-test
+ "Tests for the drain phase of the metadata file importer. Exercises
+ `process-databases!` and the per-batch drain handlers directly against
+ hand-built batches — bypassing the parser and orchestrator — and asserts
+ the resulting staging-table contents.
+
+ The production drain has two implementations: JDBC `executeBatch` for any
+ appdb (the portable path) and PG `CopyManager` for PostgreSQL (the perf
+ path). Per-batch tests below use the JDBC variant via the helpers
+ [[drain-tables!]] / [[drain-fields!]] (each opens its own connection +
+ PreparedStatement). The drain-paths-equivalence test below additionally
+ asserts the PG-COPY variant produces the same staging-table state."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.metadata-file-import.processors :as p]
+ [metabase.app-db.core :as mdb]
+ [metabase.test :as mt]
+ [metabase.util.json :as json]
+ [toucan2.core :as t2])
+ (:import
+ (java.sql Connection PreparedStatement)
+ (org.postgresql PGConnection)
+ (org.postgresql.copy CopyIn)))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once
+ (fn [thunk]
+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with
+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.
+ (mt/db)
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (thunk))))
+
+(defn- staging-tables
+ "Return the contents of `metabase_table_import` ordered by `source_id`.
+ HoneySQL (not raw SQL) so the `schema` column is dialect-quoted correctly
+ on MySQL/MariaDB, where `SCHEMA` is a reserved word."
+ []
+ (t2/query {:select [:source_id :source_db_id :db_name :schema :name
+ :description :display_name :target_id]
+ :from [:metabase_table_import]
+ :order-by [[:source_id :asc]]}))
+
+(defn- staging-fields
+ "Return the contents of `metabase_field_import` ordered by `source_id`."
+ []
+ (t2/query {:select [:source_id :source_table_id :source_parent_id :source_fk_target_id
+ :name :base_type :database_type :effective_type :semantic_type
+ :coercion_strategy :description :nfc_path
+ :depth :target_id :target_table_id :target_parent_id :target_fk_target_id]
+ :from [:metabase_field_import]
+ :order-by [[:source_id :asc]]}))
+
+(defn- batch
+ "Wrap rows in `[line-num row]` tuples as the parser would emit."
+ [rows]
+ (mapv (fn [i row] [(inc i) row]) (range) rows))
+
+(defn- drain-tables!
+ "Open a connection + PreparedStatement and drain `tbl-batch` via the JDBC
+ variant. For tests that don't care which production path runs."
+ [databases-by-source-id tbl-batch]
+ (with-open [^Connection conn (.getConnection (mdb/data-source))
+ ^PreparedStatement ps (.prepareStatement conn (p/tables-insert-sql))]
+ (p/drain-tables-batch-jdbc! ps databases-by-source-id tbl-batch)))
+
+(defn- drain-fields!
+ "Open a connection + PreparedStatement and drain `fld-batch` via the JDBC
+ variant. For tests that don't care which production path runs."
+ [fld-batch]
+ (with-open [^Connection conn (.getConnection (mdb/data-source))
+ ^PreparedStatement ps (.prepareStatement conn (p/fields-insert-sql))]
+ (p/drain-fields-batch-jdbc! ps fld-batch)))
+
+;;; ============================== process-databases! ==============================
+
+(deftest process-databases!-matches-by-name-and-engine-test
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}]
+ (let [results (vec (p/process-databases!
+ (batch [{:id 100 :name db-name :engine "h2"}])))]
+ (testing "matched result carries source-id (int), name, target-id"
+ (is (= [{:source-id 100 :name db-name :target-id db-id :status :matched}]
+ results))))))
+
+(deftest process-databases!-no-match-test
+ (let [results (vec (p/process-databases!
+ (batch [{:id 200 :name "nonexistent-db" :engine "h2"}])))]
+ (testing "missing target produces :no-match (non-fatal)"
+ (is (= 1 (count results)))
+ (is (= :no-match (:status (first results))))
+ (is (= 200 (:source-id (first results))))
+ (is (= "nonexistent-db" (:name (first results)))))))
+
+;;; ============================== tables drain ==============================
+
+(deftest drain-tables-writes-source-and-target-columns-test
+ (try
+ (p/clear-staging-tables!)
+ (let [databases-by-source-id {7 "warehouse-a" 9 "warehouse-b"}]
+ (drain-tables!
+ databases-by-source-id
+ (batch [{:id 100 :db_id 7 :schema "public" :name "users"
+ :description "user table"}
+ {:id 101 :db_id 7 :schema nil :name "events"}
+ {:id 200 :db_id 9 :schema "analytics" :name "purchases"}]))
+ (let [rows (staging-tables)]
+ (testing "three rows landed"
+ (is (= 3 (count rows))))
+ (testing "source ids and natural-key columns populated"
+ (is (= [100 7 "warehouse-a" "public" "users" "user table"]
+ ((juxt :source_id :source_db_id :db_name :schema :name :description) (nth rows 0))))
+ (is (= [101 7 "warehouse-a" nil "events" nil]
+ ((juxt :source_id :source_db_id :db_name :schema :name :description) (nth rows 1))))
+ (is (= [200 9 "warehouse-b" "analytics" "purchases" nil]
+ ((juxt :source_id :source_db_id :db_name :schema :name :description) (nth rows 2)))))
+ (testing "display_name pre-humanized at drain time"
+ (is (= "Users" (:display_name (nth rows 0))))
+ (is (= "Events" (:display_name (nth rows 1))))
+ (is (= "Purchases" (:display_name (nth rows 2)))))
+ (testing "target_id starts NULL — populated by the resolve step"
+ (is (every? nil? (map :target_id rows))))))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== fields drain ==============================
+
+(deftest drain-fields-writes-all-shapes-test
+ (try
+ (p/clear-staging-tables!)
+ (drain-fields!
+ (batch
+ ;; Flat field — no parent, no FK, no nfc_path
+ [{:id 1000 :table_id 100 :name "id"
+ :base_type "type/Integer" :database_type "int"}
+ ;; Dictionary parent
+ {:id 1001 :table_id 100 :name "data"
+ :base_type "type/Dictionary" :database_type "json"}
+ ;; nested leaf — parent_id + nfc_path
+ {:id 1002 :table_id 100 :name "value"
+ :base_type "type/Text" :database_type "text"
+ :parent_id 1001
+ :nfc_path ["data" "value"]}
+ ;; unfolded leaf — nfc_path only, no parent_id
+ {:id 1003 :table_id 100 :name "embedded → leaf"
+ :base_type "type/Text" :database_type "text"
+ :nfc_path ["embedded" "leaf"]}
+ ;; FK source
+ {:id 1004 :table_id 100 :name "account_fk"
+ :base_type "type/Integer" :database_type "int"
+ :fk_target_field_id 2000}
+ ;; Field with effective_type set explicitly (≠ base_type case)
+ {:id 1005 :table_id 100 :name "casted_value"
+ :base_type "type/Text" :database_type "varchar"
+ :effective_type "type/Text"
+ :semantic_type "type/Email"
+ :coercion_strategy "Coercion/UNIXSeconds->DateTime"
+ :description "an email field"}]))
+ (let [rows (staging-fields)
+ by-id (into {} (map (juxt :source_id identity) rows))]
+ (testing "six rows landed"
+ (is (= 6 (count rows))))
+ (testing "flat field has no parent or FK source ids"
+ (let [r (by-id 1000)]
+ (is (nil? (:source_parent_id r)))
+ (is (nil? (:source_fk_target_id r)))
+ (is (nil? (:nfc_path r)))))
+ (testing "Dictionary parent row has no parent_id and no nfc_path"
+ (let [r (by-id 1001)]
+ (is (nil? (:source_parent_id r)))
+ (is (nil? (:nfc_path r)))))
+ (testing "nested leaf carries source_parent_id pointing at the parent's source_id"
+ (let [r (by-id 1002)]
+ (is (= 1001 (:source_parent_id r)))
+ (is (= ["data" "value"] (json/decode (:nfc_path r))))))
+ (testing "unfolded leaf has nfc_path but no source_parent_id"
+ (let [r (by-id 1003)]
+ (is (nil? (:source_parent_id r)))
+ (is (= ["embedded" "leaf"] (json/decode (:nfc_path r))))))
+ (testing "FK source carries source_fk_target_id"
+ (let [r (by-id 1004)]
+ (is (= 2000 (:source_fk_target_id r)))))
+ (testing "optional payload columns roundtrip through staging"
+ (let [r (by-id 1005)]
+ (is (= "type/Text" (:effective_type r)))
+ (is (= "type/Email" (:semantic_type r)))
+ (is (= "Coercion/UNIXSeconds->DateTime" (:coercion_strategy r)))
+ (is (= "an email field" (:description r)))))
+ (testing "resolve outputs all start NULL"
+ (doseq [{:keys [depth target_id target_table_id target_parent_id target_fk_target_id]} rows]
+ (is (nil? depth))
+ (is (nil? target_id))
+ (is (nil? target_table_id))
+ (is (nil? target_parent_id))
+ (is (nil? target_fk_target_id)))))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== empty-batch behavior ==============================
+
+(deftest drain-tables-empty-batch-is-noop-test
+ (try
+ (p/clear-staging-tables!)
+ (drain-tables! {} (batch []))
+ (is (zero? (count (staging-tables))))
+ (finally (p/clear-staging-tables!))))
+
+(deftest drain-fields-empty-batch-is-noop-test
+ (try
+ (p/clear-staging-tables!)
+ (drain-fields! (batch []))
+ (is (zero? (count (staging-fields))))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== validation failures ==============================
+
+(deftest drain-tables-rejects-rows-missing-required-keys-test
+ (try
+ (p/clear-staging-tables!)
+ (testing "missing :id throws with :kind :invalid-input"
+ (is (= :invalid-input
+ (try
+ (drain-tables! {} (batch [{:db_id 7 :name "no-id" :schema "public"}]))
+ nil
+ (catch clojure.lang.ExceptionInfo e (:kind (ex-data e)))))))
+ (testing "no rows landed in staging on validation failure"
+ (is (zero? (count (staging-tables)))))
+ (finally (p/clear-staging-tables!))))
+
+(deftest drain-fields-rejects-rows-missing-required-keys-test
+ (try
+ (p/clear-staging-tables!)
+ (testing "missing :base_type throws with :kind :invalid-input"
+ (is (= :invalid-input
+ (try
+ (drain-fields! (batch [{:id 1 :table_id 100 :name "x" :database_type "text"}]))
+ nil
+ (catch clojure.lang.ExceptionInfo e (:kind (ex-data e)))))))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== Drain-path equivalence ==============================
+
+(def ^:private equivalence-databases-by-source-id
+ {7 "warehouse-a" 9 "warehouse-b"})
+
+(def ^:private equivalence-table-batch-rows
+ [{:id 100 :db_id 7 :schema "public" :name "users" :description "user table"}
+ {:id 101 :db_id 7 :schema nil :name "events"}
+ {:id 200 :db_id 9 :schema "analytics" :name "purchases"}])
+
+(def ^:private equivalence-field-batch-rows
+ [{:id 1000 :table_id 100 :name "id"
+ :base_type "type/Integer" :database_type "int"}
+ {:id 1001 :table_id 100 :name "data"
+ :base_type "type/Dictionary" :database_type "json"}
+ {:id 1002 :table_id 100 :name "value"
+ :base_type "type/Text" :database_type "text"
+ :parent_id 1001 :nfc_path ["data" "value"]}
+ {:id 1003 :table_id 100 :name "embedded → leaf"
+ :base_type "type/Text" :database_type "text"
+ :nfc_path ["embedded" "leaf"]}
+ {:id 1004 :table_id 100 :name "account_fk"
+ :base_type "type/Integer" :database_type "int"
+ :fk_target_field_id 2000}
+ {:id 1005 :table_id 100 :name "casted_value"
+ :base_type "type/Text" :database_type "varchar"
+ :effective_type "type/Text" :semantic_type "type/Email"
+ :coercion_strategy "Coercion/UNIXSeconds->DateTime"
+ :description "an email field"}])
+
+(defn- snapshot-after! [drain-fn]
+ (try
+ (p/clear-staging-tables!)
+ (drain-fn)
+ {:tables (staging-tables) :fields (staging-fields)}
+ (finally (p/clear-staging-tables!))))
+
+(deftest drain-paths-produce-identical-staging-test
+ (testing "the JDBC and PG-COPY drain paths produce the same staging-table
+ contents for the same batch — guards against silent drift if
+ one path's row-shaping diverges from the other. Only meaningful
+ on a PG appdb (the JDBC variant runs on every appdb; the
+ PG-COPY variant runs only on PG)."
+ (when (= :postgres (mdb/db-type))
+ (let [tbl-batch (batch equivalence-table-batch-rows)
+ fld-batch (batch equivalence-field-batch-rows)
+ jdbc (snapshot-after!
+ (fn []
+ (with-open [^Connection conn (.getConnection (mdb/data-source))
+ ^PreparedStatement tps (.prepareStatement conn (p/tables-insert-sql))
+ ^PreparedStatement fps (.prepareStatement conn (p/fields-insert-sql))]
+ (p/drain-tables-batch-jdbc! tps equivalence-databases-by-source-id tbl-batch)
+ (p/drain-fields-batch-jdbc! fps fld-batch))))
+ pg-copy (snapshot-after!
+ (fn []
+ (with-open [^Connection conn (.getConnection (mdb/data-source))]
+ (let [^PGConnection pg-conn (.unwrap conn PGConnection)
+ copy-mgr (.getCopyAPI pg-conn)
+ t-ci (.copyIn copy-mgr ^String p/tables-copy-sql)]
+ (try
+ (p/drain-tables-batch-pg-copy! t-ci equivalence-databases-by-source-id tbl-batch)
+ (finally (.endCopy ^CopyIn t-ci))))
+ (let [^PGConnection pg-conn (.unwrap conn PGConnection)
+ copy-mgr (.getCopyAPI pg-conn)
+ f-ci (.copyIn copy-mgr ^String p/fields-copy-sql)]
+ (try
+ (p/drain-fields-batch-pg-copy! f-ci fld-batch)
+ (finally (.endCopy ^CopyIn f-ci)))))))]
+ (is (= jdbc pg-copy)
+ "JDBC drain produces the same staging rows as the PG-COPY drain")))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/fields_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/fields_test.clj
new file mode 100644
index 00000000000..cde6b147089
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/fields_test.clj
@@ -0,0 +1,497 @@
+(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.fields-test
+ "Tests for the per-depth field-merge functions: the three resolves
+ (`resolve-target-table-ids-for-fields-in-staging!`,
+ `fill-target-parent-ids-at-depth!`, `fill-target-fk-target-ids-at-depth!`,
+ `resolve-target-field-ids-at-depth!`), the UPDATE/INSERT pair
+ (`update-matched-fields-at-depth!`, `insert-new-fields-at-depth!`), and
+ the orchestration loop (`merge-fields-by-depth!`)."
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.metadata-file-import.processors :as p]
+ [metabase.test :as mt]
+ [metabase.util.json :as json]
+ [toucan2.core :as t2]))
+
+(use-fixtures :once
+ (fn [thunk]
+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with
+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.
+ (mt/db)
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (thunk))))
+
+(defn- insert-staging-table-row!
+ "Helper: drop one row into metabase_table_import. `overrides` can include
+ `:target_id` to pre-resolve the row."
+ [source-id db-name name & {:as overrides}]
+ (t2/insert! :metabase_table_import
+ (merge {:source_id source-id
+ :source_db_id 1
+ :db_name db-name
+ :schema "PUBLIC"
+ :name name
+ :display_name (str/capitalize name)}
+ overrides)))
+
+(defn- insert-staging-field-row!
+ "Helper: drop one row into metabase_field_import. Fills in NOT NULL
+ defaults; `overrides` is everything else."
+ [source-id name & {:as overrides}]
+ (t2/insert! :metabase_field_import
+ (merge {:source_id source-id
+ :source_table_id 1
+ :name name
+ :base_type "type/Integer"
+ :database_type "int"}
+ overrides)))
+
+(defn- field-staging-row
+ "Read a staging row by source_id for assertion."
+ [source-id]
+ (t2/select-one
+ [:metabase_field_import
+ :source_id :source_table_id :source_parent_id :source_fk_target_id
+ :name :base_type :database_type :nfc_path :depth
+ :target_id :target_table_id :target_parent_id :target_fk_target_id]
+ :source_id source-id))
+
+;;; ============================== resolve-target-table-ids-for-fields ==============================
+
+(deftest resolve-target-table-ids-for-fields-copies-from-table-staging-test
+ (try
+ (p/clear-staging-tables!)
+ ;; Two staging tables, one pre-resolved (target_id = 42), one not.
+ (insert-staging-table-row! 1 "warehouse" "users" :target_id 42)
+ (insert-staging-table-row! 2 "warehouse" "ghost")
+ (insert-staging-field-row! 100 "id" :source_table_id 1)
+ (insert-staging-field-row! 200 "x" :source_table_id 2)
+ (p/resolve-target-table-ids-for-fields-in-staging!)
+ (is (= 42 (:target_table_id (field-staging-row 100))))
+ (is (nil? (:target_table_id (field-staging-row 200)))
+ "field whose table has no target_id stays NULL")
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== fill-target-parent-ids-at-depth ==============================
+
+(deftest fill-target-parent-ids-only-touches-the-given-depth-test
+ (try
+ (p/clear-staging-tables!)
+ ;; root at depth 0 (target_id = 1000 — pretend it's already been merged)
+ (insert-staging-field-row! 10 "root" :depth 0 :target_id 1000)
+ ;; child at depth 1 with parent pointing at the root
+ (insert-staging-field-row! 11 "child" :depth 1 :source_parent_id 10)
+ ;; another row at depth 2 — should NOT be touched by depth=1 call
+ (insert-staging-field-row! 12 "grandchild" :depth 2 :source_parent_id 11)
+ (p/fill-target-parent-ids-at-depth! 1)
+ (is (= 1000 (:target_parent_id (field-staging-row 11)))
+ "depth-1 row's target_parent_id picked up from parent at depth 0")
+ (is (nil? (:target_parent_id (field-staging-row 12)))
+ "depth-2 row not touched by depth=1 call")
+ (finally (p/clear-staging-tables!))))
+
+(deftest fill-target-parent-ids-skips-rows-without-source-parent-test
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 10 "root" :depth 0 :target_id 1000)
+ (insert-staging-field-row! 11 "no-parent" :depth 1)
+ (p/fill-target-parent-ids-at-depth! 1)
+ (is (nil? (:target_parent_id (field-staging-row 11))))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== fill-target-fk-target-ids-at-depth ==============================
+
+(deftest fill-target-fk-target-ids-at-depth-test
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 50 "id" :depth 0 :target_id 5000)
+ (insert-staging-field-row! 51 "account_fk" :depth 1 :source_fk_target_id 50)
+ (p/fill-target-fk-target-ids-at-depth! 1)
+ (is (= 5000 (:target_fk_target_id (field-staging-row 51))))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== resolve-target-field-ids-at-depth ==============================
+
+(deftest resolve-target-field-ids-matches-by-natural-key-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}
+ :model/Field {target-id :id} {:table_id t-id :name "user_id"
+ :base_type :type/Integer}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 100 "user_id" :depth 0 :target_table_id t-id)
+ (p/resolve-target-field-ids-at-depth! 0)
+ (is (= target-id (:target_id (field-staging-row 100))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest resolve-target-field-ids-honors-parent-id-in-match-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}
+ :model/Field {parent-id :id} {:table_id t-id :name "data" :base_type :type/Dictionary}
+ :model/Field {child-id :id} {:table_id t-id :name "value"
+ :base_type :type/Text :parent_id parent-id}
+ ;; Sibling at the root level with the same name as the nested child
+ :model/Field {root-id :id} {:table_id t-id :name "value" :base_type :type/Text}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 100 "value" :depth 1 :target_table_id t-id
+ :target_parent_id parent-id)
+ (insert-staging-field-row! 101 "value" :depth 0 :target_table_id t-id)
+ (p/resolve-target-field-ids-at-depth! 1)
+ (p/resolve-target-field-ids-at-depth! 0)
+ (is (= child-id (:target_id (field-staging-row 100)))
+ "nested 'value' (parent_id set) matches the nested target")
+ (is (= root-id (:target_id (field-staging-row 101)))
+ "flat 'value' (parent_id NULL) matches the root target")
+ (finally (p/clear-staging-tables!)))))
+
+(deftest resolve-target-field-ids-leaves-null-when-no-match-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 100 "no_such_field" :depth 0 :target_table_id t-id)
+ (p/resolve-target-field-ids-at-depth! 0)
+ (is (nil? (:target_id (field-staging-row 100))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest resolve-target-field-ids-skips-defective-duplicate-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}
+ :model/Field {} {:table_id t-id :name "user_id"
+ :base_type :type/Integer
+ :is_defective_duplicate true}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 100 "user_id" :depth 0 :target_table_id t-id)
+ (p/resolve-target-field-ids-at-depth! 0)
+ (is (nil? (:target_id (field-staging-row 100))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest resolve-target-field-ids-is-idempotent-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}
+ :model/Field {target-id :id} {:table_id t-id :name "x" :base_type :type/Text}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id)
+ (p/resolve-target-field-ids-at-depth! 0)
+ (p/resolve-target-field-ids-at-depth! 0)
+ (is (= target-id (:target_id (field-staging-row 100))))
+ (finally (p/clear-staging-tables!)))))
+
+;;; ============================== update-matched-fields-at-depth ==============================
+
+(deftest update-matched-clobbers-payload-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}
+ :model/Field {target-id :id} {:table_id t-id :name "x"
+ :base_type :type/Integer
+ :description "old"
+ :database_type "int"
+ :semantic_type nil}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id :target_id target-id
+ :base_type "type/Integer" :database_type "bigint"
+ :description "new" :semantic_type "type/PK")
+ (p/update-matched-fields-at-depth! 0)
+ (let [after (t2/select-one :model/Field :id target-id)]
+ (is (= "new" (:description after)))
+ (is (= "bigint" (:database_type after)))
+ (is (= :type/PK (:semantic_type after))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest update-matched-skip-if-unchanged-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}
+ :model/Field {target-id :id} {:table_id t-id :name "x"
+ :base_type :type/Integer
+ :description "same"
+ :database_type "int"
+ :active true}]
+ (try
+ (t2/query {:update :metabase_field
+ :set {:updated_at #t "2020-01-01T00:00:00Z"}
+ :where [:= :id target-id]})
+ (let [orig (:updated_at (t2/select-one :model/Field :id target-id))]
+ (p/clear-staging-tables!)
+ ;; Staging carries identical payload — UPDATE must NOT fire.
+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id :target_id target-id
+ :base_type "type/Integer" :database_type "int"
+ :description "same")
+ (p/update-matched-fields-at-depth! 0)
+ (is (= orig (:updated_at (t2/select-one :model/Field :id target-id)))
+ "identical payload → no UPDATE → updated_at unchanged"))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest update-matched-fires-when-payload-differs-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}
+ :model/Field {target-id :id} {:table_id t-id :name "x"
+ :base_type :type/Integer
+ :description "old"
+ :database_type "int"}]
+ (try
+ (t2/query {:update :metabase_field
+ :set {:updated_at #t "2020-01-01T00:00:00Z"}
+ :where [:= :id target-id]})
+ (let [orig (:updated_at (t2/select-one :model/Field :id target-id))]
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id :target_id target-id
+ :base_type "type/Integer" :database_type "int"
+ :description "new")
+ (p/update-matched-fields-at-depth! 0)
+ (is (not= orig (:updated_at (t2/select-one :model/Field :id target-id)))
+ "different payload → UPDATE fires → updated_at bumped"))
+ (finally (p/clear-staging-tables!)))))
+
+;;; ============================== insert-new-fields-at-depth ==============================
+
+(deftest insert-new-inserts-root-field-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 100 "user_id" :depth 0 :target_table_id t-id
+ :base_type "type/Integer" :database_type "int")
+ (p/insert-new-fields-at-depth! 0)
+ (let [inserted (t2/select-one :model/Field :table_id t-id :name "user_id")]
+ (is (some? inserted))
+ (is (= :type/Integer (:base_type inserted)))
+ (is (= "int" (:database_type inserted)))
+ (is (true? (:active inserted)))
+ (is (nil? (:parent_id inserted)))
+ (is (nil? (:fk_target_field_id inserted))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest insert-new-uses-target-parent-id-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}
+ :model/Field {parent-id :id} {:table_id t-id :name "data" :base_type :type/Dictionary}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 100 "value" :depth 1 :target_table_id t-id
+ :base_type "type/Text" :database_type "text"
+ :target_parent_id parent-id
+ :nfc_path (json/encode ["data" "value"]))
+ (p/insert-new-fields-at-depth! 1)
+ (let [inserted (t2/select-one :model/Field :table_id t-id :name "value" :parent_id parent-id)]
+ (is (some? inserted))
+ (is (= parent-id (:parent_id inserted)))
+ (is (= ["data" "value"] (:nfc_path inserted))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest insert-new-uses-target-fk-target-id-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}
+ :model/Field {target-id :id} {:table_id t-id :name "id" :base_type :type/Integer}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 100 "account_fk" :depth 1 :target_table_id t-id
+ :base_type "type/Integer" :database_type "int"
+ :target_fk_target_id target-id)
+ (p/insert-new-fields-at-depth! 1)
+ (let [inserted (t2/select-one :model/Field :table_id t-id :name "account_fk")]
+ (is (= target-id (:fk_target_field_id inserted))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest insert-new-preserves-null-effective-type-test
+ ;; The application reads NULL effective_type as "equal to base_type" at
+ ;; use sites, so the stored NULL is correct.
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id
+ :base_type "type/Integer" :database_type "int"
+ :effective_type nil)
+ (p/insert-new-fields-at-depth! 0)
+ (let [inserted (t2/select-one :model/Field :table_id t-id :name "x")]
+ (is (some? inserted))
+ (is (nil? (:effective_type inserted))
+ "NULL staging effective_type → NULL target effective_type"))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest insert-new-preserves-explicit-effective-type-test
+ ;; When source had effective_type ≠ base_type, the wire row carries the
+ ;; explicit value. Staging carries it. INSERT writes it.
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id
+ :base_type "type/Text" :database_type "varchar"
+ :effective_type "type/PostgresEnum")
+ (p/insert-new-fields-at-depth! 0)
+ (let [inserted (t2/select-one :model/Field :table_id t-id :name "x")]
+ (is (= :type/PostgresEnum (:effective_type inserted))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest update-matched-skip-if-unchanged-handles-effective-type-coalesce-test
+ ;; Cross-checks the COALESCE branch in field-payload-changed-predicate:
+ ;; target.effective_type explicitly == base_type, staging.effective_type
+ ;; NULL → COALESCED values match → skip-if-unchanged fires.
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}
+ :model/Field {target-id :id} {:table_id t-id :name "x"
+ :base_type :type/Integer
+ :effective_type :type/Integer
+ :database_type "int"}]
+ (try
+ (t2/query {:update :metabase_field
+ :set {:updated_at #t "2020-01-01T00:00:00Z"}
+ :where [:= :id target-id]})
+ (let [orig (:updated_at (t2/select-one :model/Field :id target-id))]
+ (p/clear-staging-tables!)
+ ;; Staging matches except effective_type is NULL (the export-omitted case).
+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id :target_id target-id
+ :base_type "type/Integer" :database_type "int"
+ :effective_type nil)
+ (p/update-matched-fields-at-depth! 0)
+ (is (= orig (:updated_at (t2/select-one :model/Field :id target-id)))
+ "COALESCE(target.effective_type, target.base_type) == COALESCE(NULL, staging.base_type) — no UPDATE"))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest insert-new-skips-row-with-no-target-table-test
+ (try
+ (p/clear-staging-tables!)
+ (let [count-before (t2/count :metabase_field)]
+ (insert-staging-field-row! 100 "orphan" :depth 0 :target_table_id nil)
+ (p/insert-new-fields-at-depth! 0)
+ (is (= count-before (t2/count :metabase_field))))
+ (finally (p/clear-staging-tables!))))
+
+(deftest insert-new-skips-row-whose-parent-was-orphaned-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]
+ (try
+ (p/clear-staging-tables!)
+ ;; Row has a source_parent_id but target_parent_id wasn't resolved
+ ;; (would happen if the parent's table didn't match target).
+ (insert-staging-field-row! 100 "orphaned_child" :depth 1
+ :target_table_id t-id
+ :source_parent_id 999)
+ (p/insert-new-fields-at-depth! 1)
+ (is (zero? (t2/count :model/Field :table_id t-id :name "orphaned_child")))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest insert-new-keeps-row-whose-fk-target-was-dropped-test
+ (testing "a field whose FK target was dropped during merge (source_fk_target_id
+ set, target_fk_target_id unresolved) is still inserted with
+ fk_target_field_id NULL — the FK is a semantic annotation, not structural."
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]
+ (try
+ (p/clear-staging-tables!)
+ ;; Field sits in a matched table (target_table_id set) and has an FK whose
+ ;; target was dropped: source_fk_target_id is set, target_fk_target_id NULL.
+ (insert-staging-field-row! 100 "user_id" :depth 1
+ :target_table_id t-id
+ :source_fk_target_id 999)
+ (p/insert-new-fields-at-depth! 1)
+ (let [inserted (t2/select-one :model/Field :table_id t-id :name "user_id")]
+ (is (some? inserted)
+ "field in a matched table is inserted even though its FK target didn't resolve")
+ (is (nil? (:fk_target_field_id inserted))
+ "the unresolved FK is stored as NULL, not the field dropped"))
+ (finally (p/clear-staging-tables!))))))
+
+;;; ============================== merge-fields-by-depth (integration) ==============================
+
+(deftest merge-fields-by-depth-handles-flat-fields-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)
+ (insert-staging-field-row! 100 "a" :depth 0 :target_table_id t-id)
+ (insert-staging-field-row! 101 "b" :depth 0 :target_table_id t-id)
+ (p/merge-fields-by-depth!)
+ (is (= #{"a" "b"} (set (map :name (t2/select :model/Field :table_id t-id)))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest merge-fields-by-depth-handles-parent-chain-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)
+ ;; root → parent → grandchild
+ (insert-staging-field-row! 10 "data" :depth 0 :target_table_id t-id
+ :base_type "type/Dictionary" :database_type "json")
+ (insert-staging-field-row! 11 "inner" :depth 1 :target_table_id t-id
+ :base_type "type/Dictionary" :database_type "json"
+ :source_parent_id 10
+ :nfc_path (json/encode ["data"]))
+ (insert-staging-field-row! 12 "leaf" :depth 2 :target_table_id t-id
+ :base_type "type/Text" :database_type "text"
+ :source_parent_id 11
+ :nfc_path (json/encode ["data" "inner"]))
+ (p/merge-fields-by-depth!)
+ (let [root (t2/select-one :model/Field :table_id t-id :name "data")
+ inner (t2/select-one :model/Field :table_id t-id :name "inner")
+ leaf (t2/select-one :model/Field :table_id t-id :name "leaf")]
+ (is (some? root))
+ (is (= (:id root) (:parent_id inner)))
+ (is (= (:id inner) (:parent_id leaf)))
+ (is (= ["data" "inner"] (:nfc_path leaf))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest merge-fields-by-depth-handles-fk-chain-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)
+ ;; A is non-FK, B points at A via fk_target.
+ (insert-staging-field-row! 10 "a" :depth 0 :target_table_id t-id)
+ (insert-staging-field-row! 11 "b" :depth 1 :target_table_id t-id
+ :source_fk_target_id 10)
+ (p/merge-fields-by-depth!)
+ (let [a (t2/select-one :model/Field :table_id t-id :name "a")
+ b (t2/select-one :model/Field :table_id t-id :name "b")]
+ (is (some? a))
+ (is (some? b))
+ (is (= (:id a) (:fk_target_field_id b))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest merge-fields-by-depth-reconciles-fk-on-matched-field-test
+ (testing "a matched field's fk_target_field_id is reconciled from the import
+ file, even when no other payload column differs — the FK isn't part
+ of the natural-key match, so it can drift independently of row identity."
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}
+ :model/Field {id-field :id} {:table_id t-id :name "id"
+ :base_type :type/Integer :database_type "int"}
+ :model/Field {acct-field :id} {:table_id t-id :name "account_id"
+ :base_type :type/Integer :database_type "int"}]
+ (try
+ (is (nil? (:fk_target_field_id (t2/select-one :model/Field :id acct-field)))
+ "fixture starts with account_id un-FK'd")
+ (p/clear-staging-tables!)
+ (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)
+ ;; Staging payloads are identical to the live rows except account_id now
+ ;; carries an FK pointing at id — so only the FK has drifted.
+ (insert-staging-field-row! 10 "id" :depth 0 :target_table_id t-id)
+ (insert-staging-field-row! 11 "account_id" :depth 1 :target_table_id t-id
+ :source_fk_target_id 10)
+ (p/merge-fields-by-depth!)
+ (is (= id-field (:fk_target_field_id (t2/select-one :model/Field :id acct-field)))
+ "matched field's fk_target_field_id is reconciled from the file")
+ (finally (p/clear-staging-tables!))))))
+
+(deftest merge-fields-by-depth-is-idempotent-test
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)
+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id)
+ (p/merge-fields-by-depth!)
+ (let [first-id (:id (t2/select-one :model/Field :table_id t-id :name "x"))
+ _ (p/merge-fields-by-depth!)
+ second-id (:id (t2/select-one :model/Field :table_id t-id :name "x"))]
+ (is (= first-id second-id))
+ (is (= 1 (t2/count :model/Field :table_id t-id :name "x"))))
+ (finally (p/clear-staging-tables!)))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/orphan_check_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/orphan_check_test.clj
new file mode 100644
index 00000000000..45bf495c0ed
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/orphan_check_test.clj
@@ -0,0 +1,167 @@
+(ns metabase-enterprise.serialization.metadata-file-import.orphan-check-test
+ "Tests for the pre-flight orphan handling that runs after drain and before
+ the depth-walk merge. Asymmetric by ref kind:
+
+ - `source_parent_id` orphans → fatal (`assert-no-orphan-refs!` throws
+ `:file-incomplete`). A field claiming a non-present parent has no
+ structurally valid position.
+ - `source_fk_target_id` orphans → lossy (`null-orphan-fk-target-refs!`
+ NULLs the ref and returns the count for the caller to WARN-log).
+ A missing fk target degrades cleanly to 'no fk relationship known'."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.metadata-file-import.processors :as p]
+ [toucan2.core :as t2]))
+
+(defn- insert-staging-fields!
+ "Insert a sequence of partial field rows directly into `metabase_field_import`,
+ filling in required columns with sensible defaults so we can focus the test
+ on the source_parent_id / source_fk_target_id graph."
+ [rows]
+ (let [defaults {:name "x" :base_type "type/Integer" :database_type "int"}]
+ (t2/insert! :metabase_field_import
+ (mapv (fn [row] (merge defaults {:source_table_id 1} row)) rows))))
+
+;;; ============================== happy path ==============================
+
+(deftest empty-staging-passes-test
+ (try
+ (p/clear-staging-tables!)
+ (is (nil? (p/assert-no-orphan-refs!)))
+ (finally (p/clear-staging-tables!))))
+
+(deftest staging-with-only-roots-passes-test
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 1}
+ {:source_id 2}
+ {:source_id 3}])
+ (is (nil? (p/assert-no-orphan-refs!)))
+ (finally (p/clear-staging-tables!))))
+
+(deftest staging-with-resolvable-parent-and-fk-passes-test
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 10}
+ {:source_id 11 :source_parent_id 10}
+ {:source_id 12 :source_fk_target_id 11}
+ {:source_id 13 :source_parent_id 11 :source_fk_target_id 10}])
+ (is (nil? (p/assert-no-orphan-refs!)))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== orphan parent ref ==============================
+
+(deftest single-orphan-parent-ref-throws-test
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 100}
+ {:source_id 101 :source_parent_id 999}])
+ (let [thrown (try (p/assert-no-orphan-refs!) nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (some? thrown) "should have thrown")
+ (is (= :file-incomplete (:kind (ex-data thrown))))
+ (is (= 1 (:orphan-parent-count (ex-data thrown))))
+ (is (= [{:source_id 101 :source_parent_id 999}]
+ (:orphan-parent-sample (ex-data thrown))))
+ (testing "error message includes the count"
+ (is (re-find #"1 orphan parent" (ex-message thrown)))))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== orphan fk-target ref ==============================
+
+(deftest orphan-fk-target-ref-is-not-fatal-for-assert-test
+ ;; Handled separately by null-orphan-fk-target-refs!.
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 200}
+ {:source_id 201 :source_fk_target_id 9999}])
+ (is (nil? (p/assert-no-orphan-refs!))
+ "orphan fk-target alone should not throw")
+ (finally (p/clear-staging-tables!))))
+
+(deftest orphan-fk-target-ref-gets-nulled-test
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 200}
+ {:source_id 201 :source_fk_target_id 9999}])
+ (let [result (p/null-orphan-fk-target-refs!)]
+ (is (= 1 (:count result)))
+ (is (= [{:source_id 201 :source_fk_target_id 9999}]
+ (:sample result))))
+ (testing "the orphan ref is NULL'd in staging"
+ (is (= [{:source_id 201 :source_fk_target_id nil}]
+ (t2/query
+ {:select [:source_id :source_fk_target_id]
+ :from [:metabase_field_import]
+ :where [:= :source_id 201]})))
+ (testing "the non-orphan row is untouched"
+ (is (= [{:source_id 200 :source_fk_target_id nil}]
+ (t2/query
+ {:select [:source_id :source_fk_target_id]
+ :from [:metabase_field_import]
+ :where [:= :source_id 200]})))))
+ (finally (p/clear-staging-tables!))))
+
+(deftest valid-fk-target-ref-survives-null-pass-test
+ ;; null-orphan-fk-target-refs! must not touch rows whose fk target IS
+ ;; present in staging.
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 210}
+ {:source_id 211 :source_fk_target_id 210}])
+ (let [result (p/null-orphan-fk-target-refs!)]
+ (is (= 0 (:count result)))
+ (is (nil? (:sample result))))
+ (is (= [{:source_id 211 :source_fk_target_id 210}]
+ (t2/query
+ {:select [:source_id :source_fk_target_id]
+ :from [:metabase_field_import]
+ :where [:= :source_id 211]})))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== both kinds of orphan ==============================
+
+(deftest both-orphan-types-handled-asymmetrically-test
+ ;; Orphan parent still throws; orphan fk-target is handled by the separate
+ ;; null-pass. The throw happens before the null-pass would run.
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 300}
+ {:source_id 301 :source_parent_id 9001}
+ {:source_id 302 :source_fk_target_id 9002}])
+ (let [thrown (try (p/assert-no-orphan-refs!) nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (some? thrown) "assert-no-orphan-refs! should still throw on parent orphan")
+ (is (= :file-incomplete (:kind (ex-data thrown))))
+ (is (= 1 (:orphan-parent-count (ex-data thrown))))
+ (is (seq (:orphan-parent-sample (ex-data thrown)))))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== sample truncation ==============================
+
+(deftest orphan-sample-caps-at-ten-rows-test
+ (try
+ (p/clear-staging-tables!)
+ ;; 25 rows all pointing at a non-existent parent
+ (insert-staging-fields!
+ (cons {:source_id 1}
+ (for [i (range 1000 1025)] {:source_id i :source_parent_id 999})))
+ (let [thrown (try (p/assert-no-orphan-refs!) nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (testing "count reflects all 25 orphans"
+ (is (= 25 (:orphan-parent-count (ex-data thrown)))))
+ (testing "sample is capped at 10"
+ (is (= 10 (count (:orphan-parent-sample (ex-data thrown)))))))
+ (finally (p/clear-staging-tables!))))
+
+;;; ============================== self-reference is not an orphan ==============================
+
+(deftest self-referential-parent-id-is-not-an-orphan-test
+ ;; A row whose source_parent_id equals its own source_id is a cycle —
+ ;; that's a different problem (caught by depth tagging) but it's NOT an
+ ;; orphan since the referenced source_id IS in the file.
+ (try
+ (p/clear-staging-tables!)
+ (insert-staging-fields! [{:source_id 500 :source_parent_id 500}])
+ (is (nil? (p/assert-no-orphan-refs!)))
+ (finally (p/clear-staging-tables!))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/parsers_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/parsers_test.clj
new file mode 100644
index 00000000000..4fc46cb6149
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/parsers_test.clj
@@ -0,0 +1,125 @@
+(ns ^:parallel metabase-enterprise.serialization.metadata-file-import.parsers-test
+ "Tests for the streaming JSON parser. Each test writes a temp file, then
+ asserts on the batches passed to `process-batch!`."
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.metadata-file-import.parsers :as parsers])
+ (:import
+ (java.io File)))
+
+(set! *warn-on-reflection* true)
+
+(defn- temp-json-file ^File [^String content]
+ (let [f (File/createTempFile "parsers-test-" ".json")]
+ (.deleteOnExit f)
+ (spit f content)
+ f))
+
+(defn- collect-batches!
+ "Stream `json-string` through the parser and return a vector of all batches that
+ the callback received, in order. Each batch is a vector of `[line-num row]` tuples."
+ [json-string array-key batch-size]
+ (let [file (temp-json-file json-string)
+ batches (atom [])]
+ (try
+ (parsers/stream-array-batches! file array-key batch-size
+ (fn [batch] (swap! batches conj (vec batch))))
+ @batches
+ (finally (.delete file)))))
+
+(deftest streams-rows-as-keyword-keyed-clojure-maps-test
+ (testing "each item in the named array is parsed into a Clojure map with keyword keys
+ (not LinkedHashMap with string keys), wrapped in a [line-num row] tuple"
+ (let [batches (collect-batches!
+ "{\"databases\":[{\"id\":17,\"name\":\"pg\",\"engine\":\"postgres\"},
+ {\"id\":18,\"name\":\"h2\",\"engine\":\"h2\"}]}"
+ :databases 100)]
+ (is (= 1 (count batches)))
+ (is (= [[1 {:id 17 :name "pg" :engine "postgres"}]
+ [2 {:id 18 :name "h2" :engine "h2"}]]
+ (first batches))))))
+
+(deftest line-numbers-are-1-indexed-across-batches-test
+ (testing "line-num is 1-indexed and continues across batch boundaries — so the third
+ item in the source array is `line-num 3`, regardless of which batch it lands in"
+ (let [json (str "{\"xs\":["
+ (str/join "," (for [i (range 1 6)] (format "{\"v\":%d}" i)))
+ "]}")
+ batches (collect-batches! json :xs 2)]
+ (is (= 3 (count batches))) ;; 2 + 2 + 1
+ (is (= [[1 {:v 1}] [2 {:v 2}]] (nth batches 0)))
+ (is (= [[3 {:v 3}] [4 {:v 4}]] (nth batches 1)))
+ (is (= [[5 {:v 5}]] (nth batches 2))))))
+
+(deftest empty-array-invokes-callback-zero-times-test
+ (testing "if the named array is empty, the callback is not invoked at all — caller
+ sees no batches, but the parser does not throw"
+ (is (= [] (collect-batches! "{\"xs\":[]}" :xs 100)))))
+
+(deftest accepts-keyword-or-string-array-key-test
+ (testing "array-key may be passed as a Clojure keyword or as a string; both forms
+ normalize to the same JSON key lookup"
+ (let [json "{\"databases\":[{\"id\":1}]}"]
+ (is (= [[[1 {:id 1}]]] (collect-batches! json :databases 10)))
+ (is (= [[[1 {:id 1}]]] (collect-batches! json "databases" 10))))))
+
+(deftest missing-array-key-throws-shape-error-test
+ (testing "a top-level object that doesn't contain the requested key throws ex-info with
+ :kind :missing-key"
+ (try
+ (collect-batches! "{\"other\":[]}" :databases 10)
+ (is false "should have thrown")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :missing-key (:kind (ex-data e))))
+ (is (= "databases" (:key (ex-data e))))))))
+
+(deftest non-array-value-at-key-throws-shape-error-test
+ (testing "a top-level key whose value is not an array throws ex-info with :kind :bad-shape"
+ (try
+ (collect-batches! "{\"databases\":{\"not\":\"an array\"}}" :databases 10)
+ (is false "should have thrown")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :bad-shape (:kind (ex-data e))))))))
+
+(deftest non-object-top-level-throws-shape-error-test
+ (testing "a document that doesn't begin with a top-level object throws"
+ (try
+ (collect-batches! "[1, 2, 3]" :anything 10)
+ (is false "should have thrown")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :bad-shape (:kind (ex-data e))))))))
+
+(deftest skips-other-top-level-keys-before-target-test
+ (testing "the parser advances past unrelated top-level keys (and their values, including
+ nested structures) before reaching the target array"
+ (let [json (str "{\"version\":1,"
+ "\"unrelated_object\":{\"a\":1,\"b\":[2,3]},"
+ "\"unrelated_array\":[4,5,6],"
+ "\"databases\":[{\"id\":99}]}")
+ batches (collect-batches! json :databases 10)]
+ (is (= [[[1 {:id 99}]]] batches)))))
+
+(deftest exact-batch-size-emits-single-batch-test
+ (testing "boundary: when item count exactly equals batch size, callback fires once"
+ (let [json "{\"xs\":[{\"v\":1},{\"v\":2}]}"
+ batches (collect-batches! json :xs 2)]
+ (is (= 1 (count batches)))
+ (is (= 2 (count (first batches)))))))
+
+(deftest single-item-with-batch-size-1-test
+ (testing "boundary: batch size 1 fires the callback once per item"
+ (let [json "{\"xs\":[{\"v\":1},{\"v\":2},{\"v\":3}]}"
+ batches (collect-batches! json :xs 1)]
+ (is (= 3 (count batches)))
+ (is (= [[1 {:v 1}]] (nth batches 0)))
+ (is (= [[2 {:v 2}]] (nth batches 1)))
+ (is (= [[3 {:v 3}]] (nth batches 2))))))
+
+(deftest file-handle-is-closed-after-streaming-test
+ (testing "the parser manages the file lifecycle: the InputStream and Reader are
+ closed after streaming completes (or throws). On Windows a sharing
+ violation would surface if the FD were leaked."
+ (let [file (temp-json-file "{\"xs\":[]}")]
+ (parsers/stream-array-batches! file :xs 10 (fn [_]))
+ (is (.delete file)))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/processors_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/processors_test.clj
new file mode 100644
index 00000000000..2e315c6c229
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/processors_test.clj
@@ -0,0 +1,151 @@
+(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.processors-test
+ "Tests for the pure batch processors. Each processor is exercised against an
+ appdb populated via `mt/with-temp` — no HTTP, no streaming parser. Tests
+ verify the processor's behavior contract: input shape, return shape, batch
+ ordering, error attribution, and observable side-effects on the appdb."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.metadata-file-import.processors :as processors]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(use-fixtures :once
+ (fn [thunk]
+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with
+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.
+ (mt/db)
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (thunk))))
+
+;;; ============================== with-staging-tables ==============================
+
+(defn- count-staging
+ "Total row count across both staging tables."
+ []
+ (+ (t2/count :metabase_table_import)
+ (t2/count :metabase_field_import)))
+
+(deftest clear-staging-tables-empties-both-tables-test
+ (testing "clear-staging-tables! deletes every row from both staging tables"
+ (t2/insert! :metabase_table_import {:source_id 1 :source_db_id 1 :db_name "x" :name "t"})
+ (t2/insert! :metabase_field_import {:source_id 1 :source_table_id 1
+ :name "f" :base_type "type/Integer" :database_type "int"})
+ (processors/clear-staging-tables!)
+ (is (zero? (count-staging))
+ "both staging tables empty after clear-staging-tables!")))
+
+(deftest with-staging-tables-returns-body-value-test
+ (testing "with-staging-tables yields the value of the body's last form"
+ (is (= ::sentinel
+ (processors/with-staging-tables ::sentinel)))))
+
+(deftest with-staging-tables-clears-on-entry-test
+ (testing "rows present before the macro are gone by the time the body runs"
+ (t2/insert! :metabase_table_import {:source_id 1 :source_db_id 1 :db_name "leftover" :name "t"})
+ (t2/insert! :metabase_field_import {:source_id 1 :source_table_id 1
+ :name "f" :base_type "type/Integer" :database_type "int"})
+ (let [observed (processors/with-staging-tables (count-staging))]
+ (is (zero? observed)
+ "macro entry wipes pre-existing rows so the body sees an empty staging area"))))
+
+(deftest with-staging-tables-clears-on-successful-exit-test
+ (testing "rows inserted inside the body are gone after the macro returns normally"
+ (processors/with-staging-tables
+ (t2/insert! :metabase_table_import {:source_id 1 :source_db_id 1 :db_name "x" :name "t"})
+ (t2/insert! :metabase_field_import {:source_id 1 :source_table_id 1
+ :name "f" :base_type "type/Integer" :database_type "int"}))
+ (is (zero? (count-staging))
+ "macro exit wipes rows the body added")))
+
+(deftest with-staging-tables-clears-on-thrown-exit-and-propagates-test
+ (testing "if the body throws, the exception bubbles AND staging tables are still wiped (try/finally)"
+ (let [thrown (atom nil)]
+ (try
+ (processors/with-staging-tables
+ (t2/insert! :metabase_table_import {:source_id 1 :source_db_id 1 :db_name "x" :name "t"})
+ (t2/insert! :metabase_field_import {:source_id 1 :source_table_id 1
+ :name "f" :base_type "type/Integer" :database_type "int"})
+ (throw (ex-info "boom" {:kind ::test-error})))
+ (catch clojure.lang.ExceptionInfo e
+ (reset! thrown e)))
+ (is (= ::test-error (:kind (ex-data @thrown)))
+ "exception propagates with its ex-data intact")
+ (is (zero? (count-staging))
+ "staging tables wiped even though body threw"))))
+
+;;; ============================== process-databases ==============================
+
+(deftest process-databases-matches-by-name-and-engine-test
+ (testing "a source row whose (name, engine) pair matches an existing target Database
+ produces a :matched result whose :target-id is the existing row's id.
+ :source-id carries the wire :id (source appdb's integer database id);
+ :name is also surfaced for the orchestrator's db_name denormalization map."
+ (mt/with-temp [:model/Database {target-id :id} {:name "imported-db" :engine :postgres}]
+ (is (= [{:source-id 100 :name "imported-db" :target-id target-id :status :matched}]
+ (into [] (processors/process-databases!
+ [[1 {:id 100 :name "imported-db" :engine "postgres"}]])))))))
+
+(deftest process-databases-emits-no-match-when-name-or-engine-differs-test
+ (testing "an unmatched row emits a :no-match result with line attribution and a
+ human-readable detail string. Mismatches are non-fatal — the loader logs and
+ skips dependents rather than aborting boot."
+ (let [[result] (into [] (processors/process-databases!
+ [[7 {:id 200 :name "no-such-db-zzz" :engine "h2"}]]))]
+ (is (= 200 (:source-id result)))
+ (is (= "no-such-db-zzz" (:name result)))
+ (is (= :no-match (:status result)))
+ (is (= 7 (:line result)))
+ (is (string? (:detail result)))
+ (is (not (contains? result :target-id))
+ "no :target-id key on no-match (so callers can use (:target-id r) as a presence check)"))))
+
+(deftest process-databases-disambiguates-by-engine-pair-test
+ (testing "two databases share a name but differ by engine; matching uses the
+ (name, engine) pair, not just name"
+ (mt/with-temp [:model/Database {pg-id :id} {:name "shared-name-test" :engine :postgres}
+ :model/Database {h2-id :id} {:name "shared-name-test" :engine :h2}]
+ (let [[r] (into [] (processors/process-databases!
+ [[1 {:id 300 :name "shared-name-test" :engine "h2"}]]))]
+ (is (= h2-id (:target-id r)))
+ (is (not= pg-id (:target-id r)))))))
+
+(deftest process-databases-validation-failure-throws-with-attribution-test
+ (testing "a malformed row (missing required key) throws ex-info carrying
+ :kind :invalid-input, the file line number, and the row's wire :id
+ as the :source-id (for attribution in the boot-time error message)."
+ (let [e (is (thrown? clojure.lang.ExceptionInfo
+ (into [] (processors/process-databases!
+ [[42 {:id 400 :name "missing-engine"}]])))) ;; no :engine
+ data (ex-data e)]
+ (is (= :invalid-input (:kind data)))
+ (is (= 42 (:line data)))
+ (is (= 400 (:source-id data))))))
+
+(deftest process-databases-preserves-input-order-test
+ (testing "results are in input order regardless of internal SELECT ordering or
+ which entries match"
+ (mt/with-temp [:model/Database {a-id :id} {:name "order-a" :engine :postgres}
+ :model/Database {b-id :id} {:name "order-b" :engine :postgres}]
+ (let [results (into [] (processors/process-databases!
+ [[1 {:id 501 :name "order-b" :engine "postgres"}]
+ [2 {:id 502 :name "no-such-name-q" :engine "h2"}]
+ [3 {:id 503 :name "order-a" :engine "postgres"}]]))]
+ (is (= [501 502 503] (mapv :source-id results)))
+ (is (= ["order-b" "no-such-name-q" "order-a"] (mapv :name results)))
+ (is (= [b-id nil a-id] (mapv :target-id results)))
+ (is (= [:matched :no-match :matched] (mapv :status results)))))))
+
+(deftest process-databases-empty-batch-test
+ (testing "empty input → empty output, no SQL, no exception"
+ (is (= [] (into [] (processors/process-databases! []))))))
+
+(deftest process-databases-result-is-streamable-test
+ (testing "the return value supports both `reduce` (the loader's fold path) and
+ seq/iteration (the test path) without re-running the eager batch work"
+ (mt/with-temp [:model/Database {target-id :id} {:name "stream-probe" :engine :postgres}]
+ (let [result (processors/process-databases!
+ [[1 {:id 700 :name "stream-probe" :engine "postgres"}]])
+ via-reduce (reduce (fn [acc r] (assoc acc (:source-id r) (:target-id r))) {} result)
+ via-seq (vec (seq result))]
+ (is (= {700 target-id} via-reduce))
+ (is (= 1 (count via-seq)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/schemas_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/schemas_test.clj
new file mode 100644
index 00000000000..bbd98ab9bfd
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/schemas_test.clj
@@ -0,0 +1,145 @@
+(ns ^:parallel metabase-enterprise.serialization.metadata-file-import.schemas-test
+ "Tests for the Malli schemas validating per-line shapes streamed by the
+ metadata file importer. The schemas describe the wire format
+ (source-side integer IDs); each row's `:id` is an integer, and cross-row
+ references (`:db_id`, `:table_id`, `:parent_id`, `:fk_target_field_id`)
+ are integers pointing at another row's `:id` in the same file.
+
+ The wire-format contract against actual export output is anchored by
+ `wire_format_test.clj`; these tests focus on schema-level
+ acceptance/rejection semantics in isolation."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.metadata-file-import.schemas :as schemas]
+ [metabase.util.malli.registry :as mr]))
+
+(defn- al
+ "Build a `java.util.ArrayList` from `xs` — mimics what Jackson hands the
+ importer for JSON arrays."
+ [xs]
+ (java.util.ArrayList. ^java.util.List xs))
+
+;;; ============================== ::database-info =====================================
+
+(deftest database-info-accepts-valid-shape-test
+ (is (mr/validate ::schemas/database-info {:id 7 :name "warehouse" :engine "postgres"})))
+
+(deftest database-info-rejects-missing-required-keys-test
+ (is (not (mr/validate ::schemas/database-info {:name "warehouse" :engine "postgres"}))
+ "missing :id")
+ (is (not (mr/validate ::schemas/database-info {:id 7 :name "warehouse"}))
+ "missing :engine")
+ (is (not (mr/validate ::schemas/database-info {:id 7 :engine "postgres"}))
+ "missing :name")
+ (is (not (mr/validate ::schemas/database-info {}))))
+
+(deftest database-info-rejects-wrong-types-test
+ (is (not (mr/validate ::schemas/database-info {:id "not-an-int" :name "warehouse" :engine "postgres"}))
+ ":id must be an integer")
+ (is (not (mr/validate ::schemas/database-info {:id 7 :name 5 :engine "postgres"}))
+ ":name must be a string")
+ (is (not (mr/validate ::schemas/database-info {:id 7 :name "warehouse" :engine 42}))
+ ":engine must be a string"))
+
+;;; ============================== ::table-info ========================================
+
+(deftest table-info-accepts-valid-shape-test
+ (testing "required keys only"
+ (is (mr/validate ::schemas/table-info
+ {:id 100 :db_id 7 :name "orders"})))
+ (testing "with optional :schema and :description"
+ (is (mr/validate ::schemas/table-info
+ {:id 100 :db_id 7 :name "orders"
+ :schema "public" :description "customer orders"})))
+ (testing ":schema present-and-null is accepted"
+ (is (mr/validate ::schemas/table-info
+ {:id 100 :db_id 7 :name "orders" :schema nil}))))
+
+(deftest table-info-rejects-missing-required-keys-test
+ (is (not (mr/validate ::schemas/table-info {:db_id 7 :name "orders"}))
+ "missing :id")
+ (is (not (mr/validate ::schemas/table-info {:id 100 :name "orders"}))
+ "missing :db_id")
+ (is (not (mr/validate ::schemas/table-info {:id 100 :db_id 7}))
+ "missing :name"))
+
+(deftest table-info-rejects-wrong-types-test
+ (is (not (mr/validate ::schemas/table-info {:id "wh" :db_id 7 :name "orders"}))
+ ":id must be an integer")
+ (is (not (mr/validate ::schemas/table-info {:id 100 :db_id "wh" :name "orders"}))
+ ":db_id must be an integer"))
+
+;;; ============================== ::field-info ========================================
+
+(deftest field-info-accepts-flat-root-field-test
+ (testing "flat root field — required keys only, no parent or nfc_path"
+ (is (mr/validate ::schemas/field-info
+ {:id 1000 :table_id 100 :name "id"
+ :base_type "type/Integer" :database_type "integer"}))))
+
+(deftest field-info-accepts-row-with-parent-id-and-nfc-path-test
+ (testing "nested leaf: both :parent_id and :nfc_path set"
+ (is (mr/validate ::schemas/field-info
+ {:id 1002 :table_id 100 :name "zip"
+ :base_type "type/Text" :database_type "text"
+ :parent_id 1001
+ :nfc_path ["address"]}))))
+
+(deftest field-info-accepts-fk-target-test
+ (is (mr/validate ::schemas/field-info
+ {:id 1003 :table_id 100 :name "user_id"
+ :base_type "type/Integer" :database_type "integer"
+ :fk_target_field_id 2000})))
+
+(deftest field-info-accepts-row-with-nfc-path-and-no-parent-id-test
+ (testing "unfolded leaf: :nfc_path set, no :parent_id (no parent storage row exists)"
+ (is (mr/validate ::schemas/field-info
+ {:id 1004 :table_id 100 :name "payload → address → zip"
+ :base_type "type/Text" :database_type "text"
+ :nfc_path ["payload" "address" "zip"]})))
+ (testing "Jackson ArrayList for :nfc_path also validates"
+ (is (mr/validate ::schemas/field-info
+ {:id 1004 :table_id 100 :name "payload → address → zip"
+ :base_type "type/Text" :database_type "text"
+ :nfc_path (al ["payload" "address" "zip"])}))))
+
+(deftest field-info-accepts-all-optional-keys-test
+ (is (mr/validate ::schemas/field-info
+ {:id 1005 :table_id 100 :name "amount"
+ :base_type "type/Float" :database_type "DECIMAL"
+ :description "order total"
+ :effective_type "type/Float"
+ :semantic_type "type/Currency"
+ :coercion_strategy "Coercion/UNIXSeconds->DateTime"
+ :parent_id 1001
+ :fk_target_field_id 2000
+ :nfc_path ["amount"]})))
+
+(deftest field-info-rejects-missing-required-keys-test
+ (let [base {:id 1000 :table_id 100 :name "x"
+ :base_type "type/Integer" :database_type "integer"}]
+ (is (not (mr/validate ::schemas/field-info (dissoc base :id))) "missing :id")
+ (is (not (mr/validate ::schemas/field-info (dissoc base :table_id))) "missing :table_id")
+ (is (not (mr/validate ::schemas/field-info (dissoc base :name))) "missing :name")
+ (is (not (mr/validate ::schemas/field-info (dissoc base :base_type))) "missing :base_type")
+ (is (not (mr/validate ::schemas/field-info (dissoc base :database_type))) "missing :database_type")))
+
+(deftest field-info-rejects-wrong-id-types-test
+ (let [base {:id 1000 :table_id 100 :name "x"
+ :base_type "type/Integer" :database_type "integer"}]
+ (is (not (mr/validate ::schemas/field-info (assoc base :id "not-int")))
+ ":id must be an integer")
+ (is (not (mr/validate ::schemas/field-info (assoc base :table_id "not-int")))
+ ":table_id must be an integer")
+ (is (not (mr/validate ::schemas/field-info (assoc base :parent_id "not-int")))
+ ":parent_id (when present) must be an integer")
+ (is (not (mr/validate ::schemas/field-info (assoc base :fk_target_field_id "not-int")))
+ ":fk_target_field_id (when present) must be an integer")))
+
+(deftest field-info-rejects-malformed-nfc-path-test
+ (let [base {:id 1000 :table_id 100 :name "x"
+ :base_type "type/Integer" :database_type "integer"}]
+ (is (not (mr/validate ::schemas/field-info (assoc base :nfc_path "not-a-list")))
+ ":nfc_path must be a list/vector")
+ (is (not (mr/validate ::schemas/field-info (assoc base :nfc_path [1 2 3])))
+ ":nfc_path elements must be strings")))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/tables_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/tables_test.clj
new file mode 100644
index 00000000000..633bac0e132
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/tables_test.clj
@@ -0,0 +1,258 @@
+(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.tables-test
+ "Tests for `resolve-target-table-ids-in-staging!` and `merge-tables!` —
+ the table-merge half of the pipeline. These functions key on the
+ natural-key tuple (db_name, schema, name) against `metabase_table`,
+ filtered to active non-defective rows."
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.metadata-file-import.processors :as p]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(use-fixtures :once
+ (fn [thunk]
+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with
+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.
+ (mt/db)
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (thunk))))
+
+(defn- staging-row [source-id db-name name & {:as overrides}]
+ (merge {:source_id source-id
+ :source_db_id 1
+ :db_name db-name
+ :schema "PUBLIC"
+ :name name
+ :display_name (str/capitalize name)}
+ overrides))
+
+(defn- staging-rows-by-source-id []
+ ;; HoneySQL (not raw SQL) so `schema` is dialect-quoted correctly on MySQL.
+ (into {} (map (juxt :source_id identity))
+ (t2/query {:select [:source_id :db_name :schema :name :description :target_id]
+ :from [:metabase_table_import]})))
+
+;;; ============================== resolve-target-table-ids ==============================
+
+(deftest resolve-populates-target-id-when-match-exists-test
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"}]
+ (try
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])
+ (p/resolve-target-table-ids-in-staging!)
+ (let [row (get (staging-rows-by-source-id) 100)]
+ (is (= target-id (:target_id row))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest resolve-leaves-target-id-null-when-no-match-test
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {} {:db_id db-id :schema "PUBLIC" :name "users"}]
+ (try
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "no_such_table")])
+ (p/resolve-target-table-ids-in-staging!)
+ (let [row (get (staging-rows-by-source-id) 100)]
+ (is (nil? (:target_id row))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest resolve-handles-nil-schema-test
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {target-id :id} {:db_id db-id :schema nil :name "raw_table"}]
+ (try
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "raw_table" :schema nil)])
+ (p/resolve-target-table-ids-in-staging!)
+ (let [row (get (staging-rows-by-source-id) 100)]
+ (is (= target-id (:target_id row))
+ "NULL-safe match: staging schema=NULL ⇔ target schema=NULL"))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest resolve-skips-inactive-target-test
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {} {:db_id db-id :schema "PUBLIC" :name "users" :active false}]
+ (try
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])
+ (p/resolve-target-table-ids-in-staging!)
+ (let [row (get (staging-rows-by-source-id) 100)]
+ (is (nil? (:target_id row))
+ "inactive target rows are intentionally not resolved — re-import creates a fresh active row"))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest resolve-skips-defective-duplicate-target-test
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {} {:db_id db-id :schema "PUBLIC" :name "users" :is_defective_duplicate true}]
+ (try
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])
+ (p/resolve-target-table-ids-in-staging!)
+ (let [row (get (staging-rows-by-source-id) 100)]
+ (is (nil? (:target_id row))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest resolve-is-idempotent-test
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"}]
+ (try
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])
+ (p/resolve-target-table-ids-in-staging!)
+ (p/resolve-target-table-ids-in-staging!)
+ (let [row (get (staging-rows-by-source-id) 100)]
+ (is (= target-id (:target_id row))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest resolve-tolerates-duplicate-natural-key-target-test
+ (testing "If two live (db_name, schema, name) candidates exist — possible
+ because `metabase_database`'s unique constraint is
+ `(router_database_id, name)` and NULL routers don't collide —
+ the resolve subquery picks a stable tiebreaker (MIN(id)) rather
+ than throwing `subquery returned more than one row`. Regression
+ for GHY-3549."
+ (mt/with-temp [:model/Database {db-id-1 :id db-name :name} {:engine :h2}
+ :model/Database {db-id-2 :id} {:engine :h2}]
+ ;; Make the second Database share the first's name; mt/with-temp gives
+ ;; each a unique name but PG's unique constraint is `(router_database_id,
+ ;; name)`, so an UPDATE that sets a duplicate (NULL, name) is permitted.
+ (t2/query {:update :metabase_database :set {:name db-name} :where [:= :id db-id-2]})
+ (let [t1-id (:id (t2/insert-returning-instance!
+ :model/Table {:db_id db-id-1 :schema "PUBLIC" :name "users"
+ :display_name "Users" :active true}))
+ t2-id (:id (t2/insert-returning-instance!
+ :model/Table {:db_id db-id-2 :schema "PUBLIC" :name "users"
+ :display_name "Users" :active true}))]
+ (try
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])
+ ;; The bug pre-fix: resolve throws "subquery returned more than one row".
+ ;; With the fix: resolve completes and target_id is set to MIN(id).
+ (p/resolve-target-table-ids-in-staging!)
+ (let [row (get (staging-rows-by-source-id) 100)]
+ (is (= (min t1-id t2-id) (:target_id row))
+ "MIN(id) tiebreaker: target_id is the smaller of the two candidates"))
+ (finally
+ (p/clear-staging-tables!)))))))
+
+;;; ============================== merge-tables ==============================
+
+(deftest merge-empty-staging-is-noop-test
+ (try
+ (p/clear-staging-tables!)
+ (let [before (t2/count :metabase_table)]
+ (p/merge-tables!)
+ (is (= before (t2/count :metabase_table))))
+ (finally (p/clear-staging-tables!))))
+
+(deftest merge-inserts-unmatched-table-test
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}]
+ (try
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "new_table"
+ :description "first import")])
+ (p/resolve-target-table-ids-in-staging!)
+ (p/merge-tables!)
+ (let [inserted (t2/select-one :model/Table :db_id db-id :name "new_table")]
+ (is (some? inserted))
+ (is (= "PUBLIC" (:schema inserted)))
+ (is (= "first import" (:description inserted)))
+ (is (true? (:active inserted)))
+ (is (= "internal" (name (:data_layer inserted)))))
+ (finally
+ (t2/delete! :model/Table :db_id db-id)
+ (p/clear-staging-tables!)))))
+
+(deftest merge-updates-description-on-match-test
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"
+ :description "old description"}]
+ (try
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users"
+ :description "new description")])
+ (p/resolve-target-table-ids-in-staging!)
+ (p/merge-tables!)
+ (is (= "new description" (:description (t2/select-one :model/Table :id target-id))))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest merge-skips-row-whose-db-has-no-target-test
+ (try
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 "nonexistent_db" "ghost_table")])
+ (p/resolve-target-table-ids-in-staging!)
+ (let [before (t2/count :model/Table :name "ghost_table")]
+ (p/merge-tables!)
+ (is (= before (t2/count :model/Table :name "ghost_table"))
+ "INSERT JOINs metabase_database on db_name; no match → row silently skipped"))
+ (finally (p/clear-staging-tables!))))
+
+(deftest merge-handles-nil-schema-test
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}]
+ (try
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "raw_table" :schema nil)])
+ (p/resolve-target-table-ids-in-staging!)
+ (p/merge-tables!)
+ (let [inserted (t2/select-one :model/Table :db_id db-id :name "raw_table")]
+ (is (some? inserted))
+ (is (nil? (:schema inserted))))
+ (finally
+ (t2/delete! :model/Table :db_id db-id)
+ (p/clear-staging-tables!)))))
+
+(deftest merge-is-idempotent-test
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}]
+ (try
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users" :description "v1")])
+ (p/resolve-target-table-ids-in-staging!)
+ (p/merge-tables!)
+ (let [first-id (:id (t2/select-one :model/Table :db_id db-id :name "users"))]
+ ;; second pass — re-resolve and re-merge against the same staging contents
+ (p/resolve-target-table-ids-in-staging!)
+ (p/merge-tables!)
+ (let [second-id (:id (t2/select-one :model/Table :db_id db-id :name "users"))]
+ (is (= first-id second-id) "re-import doesn't insert a duplicate")
+ (is (= 1 (t2/count :model/Table :db_id db-id :name "users")))))
+ (finally
+ (t2/delete! :model/Table :db_id db-id)
+ (p/clear-staging-tables!)))))
+
+(deftest merge-skip-if-unchanged-test
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"
+ :description "same"}]
+ (try
+ ;; Stamp updated_at to a known-old value so we can detect if the UPDATE fires.
+ (t2/query {:update :metabase_table
+ :set {:updated_at #t "2020-01-01T00:00:00Z"}
+ :where [:= :id target-id]})
+ (let [original-updated-at (:updated_at (t2/select-one :model/Table :id target-id))]
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users" :description "same")])
+ (p/resolve-target-table-ids-in-staging!)
+ (p/merge-tables!)
+ (let [after-updated-at (:updated_at (t2/select-one :model/Table :id target-id))]
+ (is (= original-updated-at after-updated-at)
+ "skip-if-unchanged: identical description means no UPDATE fires, updated_at unchanged")))
+ (finally (p/clear-staging-tables!)))))
+
+(deftest merge-fires-when-description-differs-test
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"
+ :description "v1"}]
+ (try
+ (t2/query {:update :metabase_table
+ :set {:updated_at #t "2020-01-01T00:00:00Z"}
+ :where [:= :id target-id]})
+ (let [original-updated-at (:updated_at (t2/select-one :model/Table :id target-id))]
+ (p/clear-staging-tables!)
+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users" :description "v2")])
+ (p/resolve-target-table-ids-in-staging!)
+ (p/merge-tables!)
+ (let [after (t2/select-one :model/Table :id target-id)]
+ (is (= "v2" (:description after)))
+ (is (not= original-updated-at (:updated_at after))
+ "description differs → UPDATE fires → updated_at bumped")))
+ (finally (p/clear-staging-tables!)))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/wire_format_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/wire_format_test.clj
new file mode 100644
index 00000000000..661b97ef9b5
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/wire_format_test.clj
@@ -0,0 +1,132 @@
+(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.wire-format-test
+ "Contract test between the metadata export and import sides. The importer's
+ Malli schemas describe what the importer expects on the wire; this test
+ exercises the export pipeline against a small hand-built fixture and
+ validates every emitted row against the corresponding import schema. If the
+ export starts emitting a column the import schema doesn't know about — or
+ stops emitting a required one — the test fails.
+
+ NOT `^:parallel`: each test does an `mt/with-temp [:model/Database :model/Table
+ :model/Field …]` whose `:after-insert` chain hits `metabase_data_permissions`
+ + the cluster lock. Concurrent tests in this ns reliably deadlock on MariaDB."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.export :as export]
+ [metabase-enterprise.serialization.metadata-file-import.schemas :as schemas]
+ [metabase.test :as mt]
+ [metabase.util.json :as json]
+ [metabase.util.malli.registry :as mr])
+ (:import
+ (java.io ByteArrayOutputStream)))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once
+ (fn [thunk]
+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with
+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.
+ (mt/db)
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (thunk))))
+
+(defn- export-as-json!
+ "Run the export against `opts`, decode JSON to keyword-keyed maps."
+ [opts]
+ (let [bos (ByteArrayOutputStream.)]
+ (export/export-metadata! bos opts)
+ (json/decode+kw (.toString bos "UTF-8"))))
+
+(defmacro ^:private with-tiny-fixture!
+ "Bind a hand-built fixture covering each wire-format field shape:
+ - flat root field (no parent, no nfc_path, no FK)
+ - Dictionary parent (has children pointing at it)
+ - nested leaf (parent_id + nfc_path)
+ - unfolded leaf (nfc_path only, no parent_id)
+ - FK source (fk_target_field_id pointing at another table's field)
+ - FK target (referenced from another field via fk_target_field_id)
+
+ Yields `[db-id table-1-id table-2-id]` in scope of `body`."
+ [bindings & body]
+ (let [[db table1 table2] bindings]
+ `(mt/with-temp
+ [:model/Database {~db :id} {:engine :h2}
+ :model/Table {~table1 :id} {:db_id ~db :schema "PUBLIC" :name "users"}
+ :model/Table {~table2 :id} {:db_id ~db :schema "PUBLIC" :name "accounts"}
+ :model/Field _flat# {:table_id ~table1 :name "user_id" :base_type :type/Integer}
+ :model/Field {parent-id# :id} {:table_id ~table1 :name "data" :base_type :type/Dictionary}
+ :model/Field _nested-leaf# {:table_id ~table1 :name "value"
+ :base_type :type/Text
+ :parent_id parent-id#
+ :nfc_path ["data" "value"]}
+ :model/Field _unfolded-leaf# {:table_id ~table1 :name "embedded → leaf"
+ :base_type :type/Text
+ :nfc_path ["embedded" "leaf"]}
+ :model/Field {target-id# :id} {:table_id ~table2 :name "id" :base_type :type/Integer}
+ :model/Field _fk-source# {:table_id ~table1
+ :name "account_fk"
+ :base_type :type/Integer
+ :fk_target_field_id target-id#}]
+ ~@body)))
+
+(defn- export-fixture!
+ "Run the export scoped to the fixture's database id."
+ [db-id]
+ (export-as-json! {:user-info {:user-id (mt/user->id :crowberto) :is-superuser? true}
+ :with-databases true
+ :with-tables true
+ :with-fields true
+ :database-ids [db-id]}))
+
+;;; ============================== Per-section tests ==============================
+
+(deftest database-rows-validate-against-import-schema-test
+ (mt/with-premium-features #{:serialization}
+ (with-tiny-fixture! [db-id _t1 _t2]
+ (let [{:keys [databases]} (export-fixture! db-id)]
+ (testing "fixture surfaces at least one database row"
+ (is (= 1 (count databases))))
+ (doseq [row databases]
+ (testing (str "row validates against ::schemas/database-info — " (pr-str row))
+ (is (mr/validate ::schemas/database-info row))))))))
+
+(deftest table-rows-validate-against-import-schema-test
+ (mt/with-premium-features #{:serialization}
+ (with-tiny-fixture! [db-id _t1 _t2]
+ (let [{:keys [tables]} (export-fixture! db-id)]
+ (testing "fixture surfaces both tables"
+ (is (= 2 (count tables))))
+ (doseq [row tables]
+ (testing (str "row validates against ::schemas/table-info — " (pr-str row))
+ (is (mr/validate ::schemas/table-info row))))))))
+
+(deftest field-rows-validate-against-import-schema-test
+ (mt/with-premium-features #{:serialization}
+ (with-tiny-fixture! [db-id _t1 _t2]
+ (let [{:keys [fields]} (export-fixture! db-id)]
+ (testing "fixture surfaces all six field shapes"
+ (is (= 6 (count fields))))
+ (doseq [row fields]
+ (testing (str "row validates against ::schemas/field-info — " (pr-str row))
+ (is (mr/validate ::schemas/field-info row))))))))
+
+;;; ============================== Per-shape coverage assertions ==============================
+;;; Belt-and-suspenders: assert the fixture actually exercised each variant.
+;;; If the fixture stops producing a particular shape (e.g., a refactor of
+;;; mt/with-temp's defaults), the per-section tests still pass, but these
+;;; coverage tests catch the gap.
+
+(deftest field-shape-coverage-test
+ (mt/with-premium-features #{:serialization}
+ (with-tiny-fixture! [db-id _t1 _t2]
+ (let [{:keys [fields]} (export-fixture! db-id)]
+ (testing "exactly one nested leaf (parent_id + nfc_path)"
+ (is (= 1 (count (filter #(and (:parent_id %) (:nfc_path %)) fields)))))
+ (testing "exactly one unfolded leaf (nfc_path without parent_id)"
+ (is (= 1 (count (filter #(and (:nfc_path %) (not (:parent_id %))) fields)))))
+ (testing "exactly one FK source (fk_target_field_id present)"
+ (is (= 1 (count (filter :fk_target_field_id fields)))))
+ (testing "at least one flat field (no parent_id, no nfc_path, no FK)"
+ (is (pos? (count (filter #(and (not (:parent_id %))
+ (not (:nfc_path %))
+ (not (:fk_target_field_id %)))
+ fields)))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import_test.clj
new file mode 100644
index 00000000000..e7ce33b978e
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import_test.clj
@@ -0,0 +1,256 @@
+(ns metabase-enterprise.serialization.metadata-file-import-test
+ "End-to-end orchestrator tests for [[metabase-enterprise.serialization.metadata-file-import/import-metadata-file!]].
+ Each test writes a temp JSON file in the wire format, sets up matching
+ `mt/with-temp` Database/Table/Field rows where needed, runs the loader, and
+ asserts on the appdb state. See
+ [[metabase-enterprise.serialization.metadata-file-import.schemas]] for the
+ wire-format primer."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.metadata-file-import :as loader]
+ [metabase-enterprise.serialization.metadata-file-import.processors :as processors]
+ [metabase.test :as mt]
+ [metabase.util.json :as json]
+ [metabase.warehouse-schema.models.table :as schema.table]
+ [toucan2.core :as t2])
+ (:import
+ (java.io File)))
+
+(set! *warn-on-reflection* true)
+
+;; NOT ^:parallel — test-helpers-set-global-values! asserts non-parallel.
+;; See note in concurrency_test.clj fixture — intentional opt-out from the
+;; parallel-safe contract that kondo's `validate-deftest` enforces.
+#_{:clj-kondo/ignore [:metabase/validate-deftest]}
+(use-fixtures :once
+ (fn [thunk]
+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with
+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.
+ (mt/db)
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (mt/test-helpers-set-global-values!
+ (thunk)))))
+
+;;; ============================== Test helpers ==============================
+
+(defn- temp-file ^File [suffix content]
+ (let [f (File/createTempFile "loader-test-" suffix)]
+ (.deleteOnExit f)
+ (spit f content)
+ f))
+
+(defn- json-file ^File [data]
+ (temp-file ".json" (json/encode data)))
+
+;;; ============================== Happy paths ==============================
+
+(deftest end-to-end-happy-path-with-json-file-test
+ (testing "given a target Database that matches by (name, engine) and a metadata file
+ describing one table with one root field and one nested field, the loader
+ inserts the missing rows and updates initial_sync_status on the matched DB"
+ (mt/with-temp [:model/Database {tgt-db :id} {:name "happy-path-db" :engine :postgres
+ :initial_sync_status "incomplete"}]
+ (let [meta-file (json-file
+ {:databases [{:id 7 :name "happy-path-db" :engine "postgres"}]
+ :tables [{:id 100 :db_id 7 :schema "public" :name "orders"}]
+ :fields [{:id 1000 :table_id 100 :name "id"
+ :base_type "type/Integer" :database_type "integer"}
+ {:id 1001 :table_id 100 :name "address"
+ :base_type "type/Structured" :database_type "json"}
+ {:id 1002 :table_id 100 :name "zip"
+ :parent_id 1001 :nfc_path ["address"]
+ :base_type "type/Text" :database_type "text"}]})]
+ (loader/import-metadata-file! meta-file)
+ (testing "the matched Database's initial_sync_status was flipped to complete"
+ (is (= "complete" (:initial_sync_status (t2/select-one :model/Database :id tgt-db)))))
+ (testing "the table was inserted, attached to the target DB"
+ (let [tbl (t2/select-one :model/Table :db_id tgt-db :name "orders")]
+ (is (some? tbl))
+ (is (= "public" (:schema tbl)))))
+ (testing "the root and nested fields were inserted with correct parent linkage"
+ (let [tbl-id (:id (t2/select-one :model/Table :db_id tgt-db :name "orders"))
+ root (t2/select-one :model/Field :table_id tbl-id :name "id")
+ addr (t2/select-one :model/Field :table_id tbl-id :name "address")
+ zip (t2/select-one :model/Field :table_id tbl-id :name "zip")]
+ (is (some? root))
+ (is (some? addr))
+ (is (some? zip))
+ (is (nil? (:parent_id root)))
+ (is (nil? (:parent_id addr)))
+ (is (= (:id addr) (:parent_id zip))
+ "the nested field's parent_id is the address field's target id")
+ (is (= ["address"] (:nfc_path zip)))))))))
+
+;;; ============================== No-match skipping ==============================
+
+(deftest unmatched-source-database-skips-its-tables-and-fields-test
+ (testing "a source database with no (name, engine) match in target: WARN logged,
+ its tables/fields are silently dropped during merge"
+ (let [meta-file (json-file
+ {:databases [{:id 7 :name "no-such-db" :engine "postgres"}]
+ :tables [{:id 100 :db_id 7 :schema "public" :name "ghost"}]
+ :fields [{:id 1000 :table_id 100 :name "x"
+ :base_type "type/Integer" :database_type "int"}]})]
+ (loader/import-metadata-file! meta-file)
+ (is (zero? (t2/count :model/Table :name "ghost"))
+ "no metabase_table row was created for the unmatched-DB's table")
+ (is (zero? (t2/count :model/Field :name "x"))
+ "no metabase_field row either"))))
+
+;;; ============================== FK resolution ==============================
+
+(deftest fk-target-field-id-resolved-after-import-test
+ (testing "two tables, one with a field that has fk_target_field_id pointing at the
+ other table's id field — after import, target appdb's fk_target_field_id
+ points at the correctly-resolved target field id"
+ (mt/with-temp [:model/Database {tgt-db :id} {:name "fk-test-db" :engine :postgres}]
+ (let [meta-file (json-file
+ {:databases [{:id 7 :name "fk-test-db" :engine "postgres"}]
+ :tables [{:id 100 :db_id 7 :schema "public" :name "users"}
+ {:id 101 :db_id 7 :schema "public" :name "orders"}]
+ :fields [{:id 1000 :table_id 100 :name "id"
+ :base_type "type/Integer" :database_type "integer"}
+ {:id 1001 :table_id 101 :name "user_id"
+ :base_type "type/Integer" :database_type "integer"
+ :fk_target_field_id 1000}]})]
+ (loader/import-metadata-file! meta-file)
+ (let [users-tbl (:id (t2/select-one :model/Table :db_id tgt-db :name "users"))
+ orders-tbl (:id (t2/select-one :model/Table :db_id tgt-db :name "orders"))
+ users-id (:id (t2/select-one :model/Field :table_id users-tbl :name "id"))
+ orders-fk (t2/select-one :model/Field :table_id orders-tbl :name "user_id")]
+ (is (= users-id (:fk_target_field_id orders-fk))
+ "orders.user_id's fk_target_field_id is users.id's resolved target id"))))))
+
+;;; ============================== Per-table default permissions ==============================
+
+(deftest newly-imported-tables-get-default-permissions-test
+ (testing "Tables inserted by the importer flow through `set-new-tables-permissions!`
+ in batch, mirroring what the `:after-insert` hook on `:model/Table`
+ would do for single-row inserts (just in one call instead of N).
+ Verified via a spy on the bulk grant function — asserting which
+ rows actually land in `data_permissions` would couple the test to
+ permission-graph semantics (db-level perms cover new tables ⇒ no
+ table-level rows), which isn't what we're checking here."
+ (mt/with-temp [:model/Database {tgt-db :id} {:name "perms-bulk-db" :engine :postgres}]
+ (let [meta-file (json-file
+ {:databases [{:id 7 :name "perms-bulk-db" :engine "postgres"}]
+ :tables [{:id 100 :db_id 7 :schema "public" :name "alpha"}
+ {:id 101 :db_id 7 :schema "public" :name "beta"}]
+ :fields [{:id 1000 :table_id 100 :name "id"
+ :base_type "type/Integer" :database_type "integer"}
+ {:id 1001 :table_id 101 :name "id"
+ :base_type "type/Integer" :database_type "integer"}]})
+ spy (atom [])
+ real-bulk @#'schema.table/set-new-tables-permissions!]
+ (with-redefs [schema.table/set-new-tables-permissions!
+ (fn [db-id rows]
+ (swap! spy conj {:db-id db-id :rows rows})
+ (real-bulk db-id rows))]
+ (loader/import-metadata-file! meta-file))
+ (let [calls @spy]
+ (is (= 1 (count calls)) "set-new-tables-permissions! fired exactly once (per db_id)")
+ (let [{:keys [db-id rows]} (first calls)]
+ (is (= tgt-db db-id) "called with the target db_id")
+ (is (= 2 (count rows)) "passed both newly-inserted tables")
+ (is (= #{"alpha" "beta"}
+ (set (map :name
+ (t2/select [:model/Table :name] :id [:in (map :id rows)]))))
+ "rows correspond to alpha and beta")))))))
+
+;;; ============================== Idempotence ==============================
+
+(deftest re-import-is-idempotent-test
+ (testing "running the same file twice produces the same final appdb state
+ — no duplicate rows, no error"
+ (mt/with-temp [:model/Database {tgt-db :id} {:name "idem-db" :engine :postgres}]
+ (let [meta-file (json-file
+ {:databases [{:id 7 :name "idem-db" :engine "postgres"}]
+ :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]
+ :fields [{:id 1000 :table_id 100 :name "x"
+ :base_type "type/Integer" :database_type "int"
+ :description "v1"}]})]
+ (loader/import-metadata-file! meta-file)
+ (loader/import-metadata-file! meta-file)
+ (let [tbl-id (:id (t2/select-one :model/Table :db_id tgt-db :name "t"))]
+ (is (= 1 (t2/count :model/Table :db_id tgt-db :name "t")))
+ (is (= 1 (t2/count :model/Field :table_id tbl-id :name "x")))
+ (is (= "v1" (:description (t2/select-one :model/Field :table_id tbl-id :name "x")))))))))
+
+;;; ============================== Pre-flight orphan bail ==============================
+
+(deftest pre-flight-orphan-bail-test
+ (testing "a file whose field references a parent_id not in the same file is rejected
+ with :file-incomplete; no live writes happen"
+ (mt/with-temp [:model/Database {} {:name "orphan-bail-db" :engine :postgres}]
+ (let [meta-file (json-file
+ {:databases [{:id 7 :name "orphan-bail-db" :engine "postgres"}]
+ :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]
+ :fields [{:id 1000 :table_id 100 :name "x"
+ :base_type "type/Integer" :database_type "int"
+ :parent_id 9999}]}) ; bad ref — no such source_id in file
+ thrown (try (loader/import-metadata-file! meta-file) nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (some? thrown))
+ (is (= :file-incomplete (:kind (ex-data thrown))))
+ (is (zero? (t2/count :model/Table :name "t"))
+ "no live writes happened — pre-flight ran before any txn")))))
+
+(deftest cycle-bail-test
+ (testing "a file with a cycle in source_parent_id is rejected with
+ :cycle-in-field-graph; no live writes happen"
+ (mt/with-temp [:model/Database {} {:name "cycle-db" :engine :postgres}]
+ (let [meta-file (json-file
+ {:databases [{:id 7 :name "cycle-db" :engine "postgres"}]
+ :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]
+ :fields [{:id 1000 :table_id 100 :name "a"
+ :base_type "type/Integer" :database_type "int"
+ :parent_id 1001}
+ {:id 1001 :table_id 100 :name "b"
+ :base_type "type/Integer" :database_type "int"
+ :parent_id 1000}]})
+ thrown (try (loader/import-metadata-file! meta-file) nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (some? thrown))
+ (is (= :cycle-in-field-graph (:kind (ex-data thrown))))
+ (is (zero? (t2/count :model/Table :name "t")))))))
+
+;;; ============================== Atomic rollback ==============================
+
+(deftest atomic-rollback-on-mid-merge-throw-test
+ (testing "if anything inside the merge transaction throws, all live writes are rolled
+ back — neither tables nor fields land in the appdb"
+ (mt/with-temp [:model/Database {tgt-db :id} {:name "rollback-db" :engine :postgres}]
+ (let [meta-file (json-file
+ {:databases [{:id 7 :name "rollback-db" :engine "postgres"}]
+ :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]
+ :fields [{:id 1000 :table_id 100 :name "x"
+ :base_type "type/Integer" :database_type "int"}]})]
+ ;; Inject a throw into merge-fields-by-depth! — runs after merge-tables! has
+ ;; written to metabase_table.
+ (with-redefs [processors/merge-fields-by-depth!
+ (fn [] (throw (ex-info "boom mid-merge" {:kind ::injected})))]
+ (let [thrown (try (loader/import-metadata-file! meta-file) nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (some? thrown))
+ (is (= ::injected (:kind (ex-data thrown))))))
+ (is (zero? (t2/count :model/Table :db_id tgt-db :name "t"))
+ "the table that merge-tables! 'inserted' was rolled back")))))
+
+;;; ============================== field-shape coverage ==============================
+
+(deftest unfolded-leaf-imports-correctly-test
+ (testing "an unfolded leaf (nfc_path set, no parent_id) imports as a leaf field
+ with nfc_path preserved and parent_id NULL"
+ (mt/with-temp [:model/Database {tgt-db :id} {:name "unfolded-leaf-db" :engine :postgres}]
+ (let [meta-file (json-file
+ {:databases [{:id 7 :name "unfolded-leaf-db" :engine "postgres"}]
+ :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]
+ :fields [{:id 1000 :table_id 100 :name "data.user.zip"
+ :base_type "type/Text" :database_type "text"
+ :nfc_path ["data" "user" "zip"]}]})]
+ (loader/import-metadata-file! meta-file)
+ (let [tbl-id (:id (t2/select-one :model/Table :db_id tgt-db :name "t"))
+ leaf (t2/select-one :model/Field :table_id tbl-id :name "data.user.zip")]
+ (is (some? leaf))
+ (is (nil? (:parent_id leaf)))
+ (is (= ["data" "user" "zip"] (:nfc_path leaf))))))))
diff --git a/resources/migrations/062/20260506_metadata_import_staging_tables.yaml b/resources/migrations/062/20260506_metadata_import_staging_tables.yaml
new file mode 100644
index 00000000000..72f606c7ebc
--- /dev/null
+++ b/resources/migrations/062/20260506_metadata_import_staging_tables.yaml
@@ -0,0 +1,235 @@
+## Staging tables for the metadata file importer.
+##
+## Two scratch tables drained into during an import, then merged into
+## metabase_table / metabase_field inside a single short transaction so
+## live-data writes are atomic. Both tables live permanently in the
+## schema; the importer wipes them on entry and exit (try/finally) so a
+## crashed prior attempt cannot leak rows into the next run.
+##
+## Wire format uses source-side integer IDs (the source appdb's
+## metabase_database / metabase_table / metabase_field surrogate keys).
+## They are internally consistent within one file and meaningless across
+## files. Staging carries `source_*_id` columns for the wire identity
+## and `target_*_id` columns populated by the resolve step against the
+## live appdb.
+##
+## Indexes:
+## - PK on `source_id` — probed by every depth-walk EXISTS subquery
+## - `target_id` — keys the merge UPDATE's clobber subqueries
+## - `depth` — per-level scoping for the depth-walk merge
+
+databaseChangeLog:
+ - logicalFilePath: migrations/062_update_migrations.yaml
+
+ - changeSet:
+ id: v62.2026-05-08T00:00:00
+ author: galdre
+ comment: Create metabase_table_import staging table
+ changes:
+ - createTable:
+ tableName: metabase_table_import
+ remarks: Staging table for the metadata file importer's tables phase. Drained outside the merge txn, merged into metabase_table inside it.
+ columns:
+ - column:
+ name: source_id
+ type: int
+ constraints:
+ nullable: false
+ primaryKey: true
+ primaryKeyName: pk_metabase_table_import
+ remarks: Wire :id — source appdb's metabase_table.id. Unique within one file.
+ - column:
+ name: source_db_id
+ type: int
+ constraints:
+ nullable: false
+ remarks: Wire :db_id — references the source-side database row in :databases[].
+ - column:
+ name: db_name
+ type: varchar(254)
+ constraints:
+ nullable: false
+ remarks: Denormalized from :databases[] at drain time, used for natural-key match against target.
+ - column:
+ name: schema
+ type: varchar(254)
+ constraints:
+ nullable: true
+ remarks: Mirrors metabase_table.schema nullability.
+ - column:
+ name: name
+ type: varchar(254)
+ constraints:
+ nullable: false
+ remarks: Wire :name — table's natural-key name component.
+ - column:
+ name: description
+ type: ${text.type}
+ constraints:
+ nullable: true
+ remarks: Wire :description — clobbered onto metabase_table.description on merge.
+ - column:
+ name: display_name
+ type: varchar(254)
+ constraints:
+ nullable: true
+ remarks: Pre-humanized at drain time so the merge SQL doesn't replicate the algorithm in three dialects.
+ - column:
+ name: target_id
+ type: int
+ constraints:
+ nullable: true
+ remarks: Resolved by table-resolve step; → metabase_table.id. NULL means no live row matches; row is an insert candidate.
+
+ - changeSet:
+ id: v62.2026-05-08T00:00:00.1
+ author: galdre
+ comment: Index metabase_table_import.target_id for the merge-tables clobber subqueries
+ changes:
+ - createIndex:
+ tableName: metabase_table_import
+ indexName: idx_mti_target_id
+ columns:
+ - column:
+ name: target_id
+
+ - changeSet:
+ id: v62.2026-05-08T00:00:01
+ author: galdre
+ comment: Create metabase_field_import staging table
+ changes:
+ - createTable:
+ tableName: metabase_field_import
+ remarks: Staging table for the metadata file importer's fields phase. Drained outside the merge txn, depth-tagged, then merged into metabase_field level by level inside the txn.
+ columns:
+ # --- wire identity ---
+ - column:
+ name: source_id
+ type: int
+ constraints:
+ nullable: false
+ primaryKey: true
+ primaryKeyName: pk_metabase_field_import
+ remarks: Wire :id — source appdb's metabase_field.id. Unique within one file.
+ - column:
+ name: source_table_id
+ type: int
+ constraints:
+ nullable: false
+ remarks: Wire :table_id — references metabase_table_import.source_id within the same file.
+ - column:
+ name: source_parent_id
+ type: int
+ constraints:
+ nullable: true
+ remarks: Wire :parent_id — references another metabase_field_import.source_id within the same file. NULL for top-level fields.
+ - column:
+ name: source_fk_target_id
+ type: int
+ constraints:
+ nullable: true
+ remarks: Wire :fk_target_field_id — references another metabase_field_import.source_id. NULL for non-FK fields.
+ # --- clobber payload ---
+ - column:
+ name: name
+ type: varchar(254)
+ constraints:
+ nullable: false
+ remarks: Wire :name — field's natural-key name component (within its parent + table).
+ - column:
+ name: base_type
+ type: varchar(254)
+ constraints:
+ nullable: false
+ remarks: Wire :base_type — clobbered onto metabase_field.base_type on merge.
+ - column:
+ name: database_type
+ type: varchar(254)
+ constraints:
+ nullable: false
+ remarks: Wire :database_type — clobbered onto metabase_field.database_type on merge.
+ - column:
+ name: effective_type
+ type: varchar(254)
+ constraints:
+ nullable: true
+ remarks: NULL when wire row omitted it (≡ base_type). Importer treats NULL as base_type on INSERT; leaves target alone on UPDATE.
+ - column:
+ name: semantic_type
+ type: varchar(254)
+ constraints:
+ nullable: true
+ remarks: Wire :semantic_type — clobbered onto metabase_field.semantic_type on merge.
+ - column:
+ name: coercion_strategy
+ type: varchar(254)
+ constraints:
+ nullable: true
+ remarks: Wire :coercion_strategy — clobbered onto metabase_field.coercion_strategy on merge.
+ - column:
+ name: description
+ type: ${text.type}
+ constraints:
+ nullable: true
+ remarks: Wire :description — clobbered onto metabase_field.description on merge.
+ - column:
+ name: nfc_path
+ type: ${text.type}
+ constraints:
+ nullable: true
+ remarks: NULL for empty/absent paths, JSON-encoded array string otherwise — same encoding as metabase_field.nfc_path.
+ # --- resolve outputs ---
+ - column:
+ name: depth
+ type: int
+ constraints:
+ nullable: true
+ remarks: Populated once after drain by the depth-tagging step. 0 = root (no parent, no fk_target). Higher = max(parent depth, fk_target depth) + 1. NULL after depth-tagging means a cycle in the file's reference graph.
+ - column:
+ name: target_id
+ type: int
+ constraints:
+ nullable: true
+ remarks: Resolved per-depth-level by natural-key match against metabase_field. NULL means no live row matches; row is an insert candidate at this depth.
+ - column:
+ name: target_table_id
+ type: int
+ constraints:
+ nullable: true
+ remarks: Populated from the table-resolve step before the depth walk starts. Joined-through from metabase_table_import.target_id.
+ - column:
+ name: target_parent_id
+ type: int
+ constraints:
+ nullable: true
+ remarks: Populated per-depth-level by self-join on source_parent_id → another row's target_id.
+ - column:
+ name: target_fk_target_id
+ type: int
+ constraints:
+ nullable: true
+ remarks: Populated per-depth-level by self-join on source_fk_target_id → another row's target_id.
+
+ - changeSet:
+ id: v62.2026-05-08T00:00:01.1
+ author: galdre
+ comment: Index metabase_field_import.depth for per-level scoping in the depth-walk merge
+ changes:
+ - createIndex:
+ tableName: metabase_field_import
+ indexName: idx_mfi_depth
+ columns:
+ - column:
+ name: depth
+
+ - changeSet:
+ id: v62.2026-05-08T00:00:01.2
+ author: galdre
+ comment: Index metabase_field_import.target_id for the merge-fields clobber subqueries
+ changes:
+ - createIndex:
+ tableName: metabase_field_import
+ indexName: idx_mfi_target_id
+ columns:
+ - column:
+ name: target_id
diff --git a/src/metabase/permissions/core.clj b/src/metabase/permissions/core.clj
index 6952ac06c48..1b770465944 100644
--- a/src/metabase/permissions/core.clj
+++ b/src/metabase/permissions/core.clj
@@ -70,6 +70,7 @@
set-default-group-permissions!
set-default-database-permissions!
set-default-table-permissions!
+ set-default-table-permissions-bulk!
with-global-permissions-lock
with-db-scoped-permissions-lock]
[metabase.permissions.models.data-permissions.sql
diff --git a/src/metabase/permissions/models/data_permissions.clj b/src/metabase/permissions/models/data_permissions.clj
index 7d3b9ad30a1..f1ef476939b 100644
--- a/src/metabase/permissions/models/data_permissions.clj
+++ b/src/metabase/permissions/models/data_permissions.clj
@@ -1123,109 +1123,112 @@
groups)]
(batch-insert-permissions! perm-rows))))
-(defn set-default-table-permissions!
- "Bulk-sets default permissions for a newly-created table across all relevant groups.
- Handles three cases per (group, perm-type):
- - Group has DB-level perm matching default → no-op (DB-level covers it)
- - Group has DB-level perm but new table needs :blocked → going-granular (expand to per-table rows)
- - Group has no DB-level perm (already table-granular) → simple insert for the new table
+(defn- mk-perm-row [group-id perm-type perm-value db-id table-id schema]
+ {:perm_type perm-type :group_id group-id :perm_value perm-value
+ :db_id db-id :table_id table-id :schema_name schema})
+
+(defn- load-perm-context
+ "Load the introspection state needed to classify each `(group, perm-type)`
+ for new tables on `db-id`."
+ [db-id group-ids perm-types]
+ (let [qn (mapv u/qualified-name perm-types)
+ db-level (t2/select :model/DataPermissions
+ {:where [:and [:= :db_id db-id] [:= :table_id nil]
+ [:in :group_id group-ids] [:in :perm_type qn]]})
+ table-level (t2/select :model/DataPermissions
+ {:where [:and [:= :db_id db-id] [:not= :table_id nil]
+ [:in :group_id group-ids] [:in :perm_type qn]]})]
+ {:db-id db-id
+ :db-level-idx (into {} (map (juxt (juxt :group_id :perm_type) identity)) db-level)
+ :schema-vals-idx (reduce (fn [acc {:keys [group_id perm_type schema_name perm_value]}]
+ (update-in acc [group_id perm_type schema_name] (fnil conj #{}) perm_value))
+ {} table-level)
+ :all-db-tables (t2/select [:model/Table :id :db_id :schema] :db_id db-id :active true)
+ :view-data-levels (new-table-view-data-permission-levels db-id group-ids)}))
+
+(defn- compute-actual-value
+ "Per-entry resolution: enterprise view-data override, then schema-consistency
+ if all existing tables in the schema agree, else the caller's default."
+ [{:keys [view-data-levels schema-vals-idx]}
+ {:keys [group-id perm-type default-value table]}]
+ (or (when (= perm-type :perms/view-data)
+ (get view-data-levels group-id))
+ (let [sv (get-in schema-vals-idx [group-id perm-type (:schema table)])]
+ (when (and (seq sv) (= (count sv) 1))
+ (first sv)))
+ default-value))
+
+(defn- classify-key
+ "For one `(group-id, perm-type)`, return `{:deletes [id?] :rows [perm-row...]}`.
+ Three branches mirror the original cond: going-granular, simple-insert, no-op."
+ [{:keys [db-id db-level-idx all-db-tables batch-table-ids]}
+ [[group-id perm-type] entries]]
+ (let [db-perm (get db-level-idx [group-id perm-type])
+ any-blocked? (and db-perm (some #(= :blocked (:actual-value %)) entries))
+ mk (fn [v table-id schema]
+ (mk-perm-row group-id perm-type v db-id table-id schema))]
+ (cond
+ ;; Going-granular fires once for the whole `(group, perm-type)`:
+ ;; delete the DB-level row, expand non-batch tables to the old value,
+ ;; then write each batch table at its actual-value.
+ any-blocked?
+ (let [expansion-value (case (:perm_value db-perm)
+ :query-builder-and-native :query-builder
+ (:perm_value db-perm))]
+ {:deletes [(:id db-perm)]
+ :rows (concat
+ (for [t all-db-tables :when (not (contains? batch-table-ids (:id t)))]
+ (mk expansion-value (:id t) (:schema t)))
+ (for [{:keys [actual-value table]} entries]
+ (mk actual-value (u/the-id table) (:schema table))))})
+ (nil? db-perm)
+ {:rows (for [{:keys [actual-value table]} entries]
+ (mk actual-value (u/the-id table) (:schema table)))}
+ ;; DB-level covers the batch tables — no-op.
+ :else nil)))
+
+(defn set-default-table-permissions-bulk!
+ "Bulk-set default permissions for many newly-created tables on the same
+ database. Issues the introspection SELECTs once for the whole batch and
+ a single DELETE + INSERT, deduping going-granular triggers across the
+ batch.
+
+ Caller must hold [[with-db-scoped-permissions-lock]] for `db-id`.
+ `tables+defaults` is a seq of `[table group-perm-defaults]` pairs."
+ [db-id tables+defaults]
+ (when (seq tables+defaults)
+ (let [batch-table-ids (set (map (fn [[t _]] (u/the-id t)) tables+defaults))
+ all-defaults (mapcat (fn [[t defs]] (map #(assoc % :table t) defs))
+ tables+defaults)
+ group-ids (distinct (map :group-id all-defaults))
+ perm-types (distinct (map :perm-type all-defaults))
+ ctx (assoc (load-perm-context db-id group-ids perm-types)
+ :batch-table-ids batch-table-ids)
+ annotated (mapv (fn [d] (assoc d :actual-value (compute-actual-value ctx d)))
+ all-defaults)
+ results (->> annotated
+ (group-by (juxt :group-id :perm-type))
+ (keep #(classify-key ctx %)))
+ to-delete (mapcat :deletes results)
+ to-insert (mapcat :rows results)]
+ (when (seq to-delete) (batch-delete-permissions! to-delete))
+ (when (seq to-insert) (batch-insert-permissions! to-insert)))))
- `group-perm-defaults` is a seq of {:group-id G :perm-type PT :default-value V} triples."
+(defn set-default-table-permissions!
+ "Set default permissions for a newly-created table across all relevant
+ groups. Handles three cases per `(group, perm-type)`:
+ - Group has DB-level perm matching default → no-op
+ - Group has DB-level perm but new table needs `:blocked` → going-granular
+ - Group has no DB-level perm → simple insert
+
+ `group-perm-defaults` is a seq of `{:group-id :perm-type :default-value}`
+ triples. Thin wrapper over [[set-default-table-permissions-bulk!]] for
+ the single-table case."
[table group-perm-defaults]
- (when (seq group-perm-defaults)
- (let [table (if (map? table)
- table
- (t2/select-one [:model/Table :id :db_id :schema] :id table))
- db-id (:db_id table)
- table-id (u/the-id table)
- schema (:schema table)
- group-ids (distinct (map :group-id group-perm-defaults))
- perm-types (distinct (map :perm-type group-perm-defaults))
- ;; Batch SELECT #1: all DB-level permissions for this DB across all relevant groups + perm-types
- db-level-perms (t2/select :model/DataPermissions
- {:where [:and
- [:= :db_id db-id]
- [:= :table_id nil]
- [:in :group_id group-ids]
- [:in :perm_type (mapv u/qualified-name perm-types)]]})
- ;; Index: {[group_id perm_type] → db-level-perm-row}
- db-level-idx (into {} (map (fn [p] [[(:group_id p) (:perm_type p)] p]) db-level-perms))
- ;; Batch SELECT #2: all existing table-level permissions for this DB
- ;; (needed for schema-permission-value logic and going-granular expansion)
- table-perms (t2/select :model/DataPermissions
- {:where [:and
- [:= :db_id db-id]
- [:not= :table_id nil]
- [:in :group_id group-ids]
- [:in :perm_type (mapv u/qualified-name perm-types)]]})
- ;; Index: {[group_id perm_type schema_name] → set of values}
- schema-vals-idx (reduce (fn [acc {:keys [group_id perm_type schema_name perm_value]}]
- (update-in acc [group_id perm_type schema_name] (fnil conj #{}) perm_value))
- {}
- table-perms)
- ;; Batch SELECT #3: all tables in this DB (for going-granular expansion)
- all-db-tables (t2/select [:model/Table :id :db_id :schema] :db_id db-id :active true)
- ;; Batch-fetch view-data levels for all groups at once
- view-data-levels (new-table-view-data-permission-levels db-id group-ids)
- ;; Classify each (group, perm-type) triple
- {:keys [simple-perms going-granular db-rows-to-delete]}
- (reduce
- (fn [acc {:keys [group-id perm-type default-value]}]
- (let [;; Enterprise hook override for view-data
- actual-value (or (when (= perm-type :perms/view-data)
- (get view-data-levels group-id))
- ;; Schema-level consistency: if all tables in schema have same value, use it
- (let [sv (get-in schema-vals-idx [group-id perm-type schema])]
- (when (and (seq sv) (= (count sv) 1))
- (first sv)))
- default-value)
- db-level-perm (get db-level-idx [group-id perm-type])
- new-perm {:perm_type perm-type
- :group_id group-id
- :perm_value actual-value
- :db_id db-id
- :table_id table-id
- :schema_name schema}]
- (cond
- ;; Group has DB-level perm and new table needs :blocked → going-granular
- (and db-level-perm (= actual-value :blocked))
- (-> acc
- (update :going-granular conj {:group-id group-id :perm-type perm-type
- :new-perm new-perm :db-perm db-level-perm})
- (update :db-rows-to-delete conj (:id db-level-perm)))
-
- ;; Group has no DB-level perm (already table-granular) → simple insert
- (not db-level-perm)
- (update acc :simple-perms conj new-perm)
-
- ;; Group has DB-level perm matching or compatible → no-op
- :else
- acc)))
- {:simple-perms [] :going-granular [] :db-rows-to-delete []}
- group-perm-defaults)]
- ;; Bulk DELETE: DB-level rows that need going-granular expansion
- (when (seq db-rows-to-delete)
- (batch-delete-permissions! db-rows-to-delete))
- ;; Bulk INSERT: expansion rows for going-granular groups + simple insert rows
- (let [expansion-rows (mapcat
- (fn [{:keys [group-id perm-type new-perm db-perm]}]
- (let [db-perm-value (:perm_value db-perm)
- ;; Build per-table rows for all existing tables (with the old DB-level value)
- existing-table-rows
- (keep (fn [t]
- (when (not= (:id t) table-id)
- {:perm_type perm-type
- :group_id group-id
- :perm_value (case db-perm-value
- :query-builder-and-native :query-builder
- db-perm-value)
- :db_id db-id
- :table_id (:id t)
- :schema_name (:schema t)}))
- all-db-tables)]
- (cons new-perm existing-table-rows)))
- going-granular)]
- (batch-insert-permissions! (concat expansion-rows simple-perms))))))
+ (let [table (if (map? table)
+ table
+ (t2/select-one [:model/Table :id :db_id :schema] :id table))]
+ (set-default-table-permissions-bulk! (:db_id table) [[table group-perm-defaults]])))
(defenterprise download-perms-level
"Return the download permission for the query that the given user has. OSS returns :full"
diff --git a/src/metabase/sync/analyze.clj b/src/metabase/sync/analyze.clj
index 9aa73c07eb7..b6fdb725e8c 100644
--- a/src/metabase/sync/analyze.clj
+++ b/src/metabase/sync/analyze.clj
@@ -115,15 +115,29 @@
#(sync.interestingness/score-fields-for-db! % log-fn)
interestingness-summary)])
+(mu/defn- analyze-db!*
+ "Shared core of [[analyze-db!]] and [[analyze-db-explicit!]]: the analysis work, without the
+ surrounding `*-sync-operation` wrapper (which is what applies the eligibility gating)."
+ [database :- i/DatabaseInstance]
+ (sync-util/with-emoji-progress-bar [emoji-progress-bar (inc (* 4 (sync-util/sync-tables-count database)))]
+ (u/prog1 (sync-util/run-sync-operation "analyze" database (make-analyze-steps (maybe-log-progress emoji-progress-bar)))
+ (update-fields-last-analyzed-for-db! database))))
+
(mu/defn analyze-db!
"Perform in-depth analysis on the data for all Tables in a given `database`. This is dependent on what each database
driver supports, but includes things like cardinality testing and table row counting. This also updates the
- `:last_analyzed` value for each affected Field."
+ `:last_analyzed` value for each affected Field. Subject to the `disable-auto-sync` setting; for an
+ explicit user-requested analysis use [[analyze-db-explicit!]]."
[database :- i/DatabaseInstance]
(sync-util/sync-operation :analyze database (format "Analyze data for %s" (sync-util/name-for-logging database))
- (sync-util/with-emoji-progress-bar [emoji-progress-bar (inc (* 4 (sync-util/sync-tables-count database)))]
- (u/prog1 (sync-util/run-sync-operation "analyze" database (make-analyze-steps (maybe-log-progress emoji-progress-bar)))
- (update-fields-last-analyzed-for-db! database)))))
+ (analyze-db!* database)))
+
+(mu/defn analyze-db-explicit!
+ "Like [[analyze-db!]], but for an explicit, user-requested sync (e.g. the Sync-now button): runs even
+ when the `disable-auto-sync` setting is enabled."
+ [database :- i/DatabaseInstance]
+ (sync-util/explicit-sync-operation :analyze database (format "Analyze data for %s" (sync-util/name-for-logging database))
+ (analyze-db!* database)))
(mu/defn refingerprint-db!
"Refingerprint a subset of tables in a given `database`. This will re-fingerprint tables up to a threshold amount of
diff --git a/src/metabase/sync/core.clj b/src/metabase/sync/core.clj
index 5f9acd2527b..4617e6f4833 100644
--- a/src/metabase/sync/core.clj
+++ b/src/metabase/sync/core.clj
@@ -20,7 +20,8 @@
(p/import-vars
[metabase.sync.analyze
- analyze-db!]
+ analyze-db!
+ analyze-db-explicit!]
[metabase.sync.field-values
update-field-values!
update-field-values-for-table!]
@@ -29,7 +30,8 @@
sync-database!
sync-table!]
[metabase.sync.sync-metadata
- sync-db-metadata!]
+ sync-db-metadata!
+ sync-db-metadata-explicit!]
[metabase.sync.sync-metadata.fields
sync-fields-for-table!]
[metabase.sync.sync-metadata.tables
diff --git a/src/metabase/sync/events/sync_database.clj b/src/metabase/sync/events/sync_database.clj
index ef369d2d9ae..dc1a0e984cb 100644
--- a/src/metabase/sync/events/sync_database.clj
+++ b/src/metabase/sync/events/sync_database.clj
@@ -6,6 +6,7 @@
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.quick-task :as quick-task]
+ [metabase.warehouses.core :as warehouses]
[methodical.core :as methodical]))
(derive ::event :metabase/event)
@@ -16,7 +17,7 @@
[topic {database :object :as _event}]
;; try/catch here to prevent individual topic processing exceptions from bubbling up. better to handle them here.
(try
- (when database
+ (when (and database (not (warehouses/disable-auto-sync)))
;; just kick off a sync on another thread
(quick-task/submit-task!
(fn []
diff --git a/src/metabase/sync/sync_metadata.clj b/src/metabase/sync/sync_metadata.clj
index 5dfb4c4a175..ee99278028e 100644
--- a/src/metabase/sync/sync_metadata.clj
+++ b/src/metabase/sync/sync_metadata.clj
@@ -60,16 +60,31 @@
;; Sync the metabase metadata table if it exists.
(sync-util/create-sync-step "sync-metabase-metadata" #(metabase-metadata/sync-metabase-metadata! % db-metadata))])
+(mu/defn- sync-db-metadata!*
+ "Shared core of [[sync-db-metadata!]] and [[sync-db-metadata-explicit!]]: the metadata sync work,
+ without the surrounding `*-sync-operation` wrapper (which is what applies the eligibility gating)."
+ [database :- i/DatabaseInstance]
+ (let [db-metadata (tracing/with-span :sync "sync.metadata.fetch-metadata" {:db/id (:id database)}
+ (fetch-metadata/db-metadata database))]
+ (u/prog1 (sync-util/run-sync-operation "sync" database (make-sync-steps db-metadata))
+ (if (some sync-util/abandon-sync? (map second (:steps <>)))
+ (sync-util/set-initial-database-sync-aborted! database)
+ (sync-util/set-initial-database-sync-complete! database)))))
+
(mu/defn sync-db-metadata!
- "Sync the metadata for a Metabase `database`. This makes sure child Table & Field objects are synchronized."
+ "Sync the metadata for a Metabase `database`. This makes sure child Table & Field objects are synchronized.
+ Subject to the `disable-auto-sync` setting; for an explicit user-requested sync use
+ [[sync-db-metadata-explicit!]]."
[database :- i/DatabaseInstance]
(sync-util/sync-operation :sync-metadata database (format "Sync metadata for %s" (sync-util/name-for-logging database))
- (let [db-metadata (tracing/with-span :sync "sync.metadata.fetch-metadata" {:db/id (:id database)}
- (fetch-metadata/db-metadata database))]
- (u/prog1 (sync-util/run-sync-operation "sync" database (make-sync-steps db-metadata))
- (if (some sync-util/abandon-sync? (map second (:steps <>)))
- (sync-util/set-initial-database-sync-aborted! database)
- (sync-util/set-initial-database-sync-complete! database))))))
+ (sync-db-metadata!* database)))
+
+(mu/defn sync-db-metadata-explicit!
+ "Like [[sync-db-metadata!]], but for an explicit, user-requested sync (e.g. the Sync-now button):
+ runs even when the `disable-auto-sync` setting is enabled."
+ [database :- i/DatabaseInstance]
+ (sync-util/explicit-sync-operation :sync-metadata database (format "Sync metadata for %s" (sync-util/name-for-logging database))
+ (sync-db-metadata!* database)))
(mu/defn sync-table-metadata!
"Sync the metadata for an individual `table` -- make sure Fields and FKs are up-to-date."
diff --git a/src/metabase/sync/task/sync_databases.clj b/src/metabase/sync/task/sync_databases.clj
index 2790b2fa7f9..0cf6ff5bf26 100644
--- a/src/metabase/sync/task/sync_databases.clj
+++ b/src/metabase/sync/task/sync_databases.clj
@@ -28,6 +28,7 @@
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouses.core :as warehouses]
[toucan2.core :as t2])
(:import
(org.quartz
@@ -107,7 +108,11 @@
"The sync and analyze database job, as a function that can be used in a test"
[job-context]
(when-let [database-id (job-context->database-id job-context)]
- (if (= audit/audit-db-id database-id)
+ (cond
+ (warehouses/disable-auto-sync)
+ (log/debugf "Skipping scheduled sync for Database %d: disable-auto-sync is on." database-id)
+
+ (= audit/audit-db-id database-id)
(do
(log/warn "Cannot sync Database: It is the audit db.")
(when-not config/is-prod?
@@ -115,6 +120,8 @@
{:database-id database-id
:raw-job-context job-context
:job-context (pr-str job-context)}))))
+
+ :else
(sync-and-analyze-database*! database-id))))
(task/defjob ^{org.quartz.DisallowConcurrentExecution true
@@ -126,14 +133,17 @@
"The update field values job, as a function that can be used in a test"
[job-context]
(when-let [database-id (job-context->database-id job-context)]
- (log/infof "Update Field values task triggered for Database %d." database-id)
- (when-let [database (or (t2/select-one :model/Database :id database-id)
- (do
- (unschedule-tasks-for-db! (mi/instance :model/Database {:id database-id}))
- (log/warnf "Cannot update Field values for Database %d: Database does not exist." database-id)))]
- (if (:is_full_sync database)
- (sync.field-values/update-field-values! database)
- (log/infof "Skipping update, automatic Field value updates are disabled for Database %d." database-id)))))
+ (if (warehouses/disable-auto-sync)
+ (log/debugf "Skipping scheduled field-values update for Database %d: disable-auto-sync is on." database-id)
+ (do
+ (log/infof "Update Field values task triggered for Database %d." database-id)
+ (when-let [database (or (t2/select-one :model/Database :id database-id)
+ (do
+ (unschedule-tasks-for-db! (mi/instance :model/Database {:id database-id}))
+ (log/warnf "Cannot update Field values for Database %d: Database does not exist." database-id)))]
+ (if (:is_full_sync database)
+ (sync.field-values/update-field-values! database)
+ (log/infof "Skipping update, automatic Field value updates are disabled for Database %d." database-id)))))))
(task/defjob ^{org.quartz.DisallowConcurrentExecution true
:doc "Update field values"}
diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.clj
index 4e6d0390044..1e72a6134ea 100644
--- a/src/metabase/sync/util.clj
+++ b/src/metabase/sync/util.clj
@@ -91,6 +91,19 @@
;; setups in the future
(defonce ^:private operation->db-ids (atom {}))
+;; In-JVM only, same scope as `operation->db-ids`.
+(defonce ^:private external-busy-predicates (atom []))
+
+(defn register-busy-predicate!
+ "Register a 0-arity `pred` to be checked before each `do-sync-operation`.
+ `pred` returns nil to permit the sync, or `{:reason \"...\"}` to skip it
+ (logged at WARN by the caller)."
+ [pred]
+ (swap! external-busy-predicates conj pred))
+
+(defn- external-busy []
+ (some (fn [p] (p)) @external-busy-predicates))
+
(defn with-duplicate-ops-prevented
"Run `f` in a way that will prevent it from simultaneously being ran more for a single database more than once for a
given `operation`. This prevents duplicate sync-like operations from taking place for a given DB, e.g. if a user
@@ -259,13 +272,17 @@
:analyze :fingerprint
:refingerprint :fingerprint})
-(mu/defn do-sync-operation
- "Internal implementation of [[sync-operation]]; use that instead of calling this directly."
+(mu/defn- do-sync-operation*
+ "Shared core of [[do-sync-operation]] and [[do-explicit-sync-operation]]. Runs the sync work `f`
+ wrapped in the surrounding machinery (task-history, events, logging, duplicate-op prevention).
+ Performs no eligibility gating itself — the public wrappers decide whether to call it."
[operation :- :keyword ; something like `:sync-metadata` or `:refingerprint`
database :- (ms/InstanceOf :model/Database)
message :- ms/NonBlankString
f :- fn?]
- (when (database/should-sync? database)
+ (if-let [busy (external-busy)]
+ (log/warnf "Skipping %s for database %d: %s"
+ (name operation) (u/the-id database) (:reason busy))
(let [run-type (operation->run-type operation)]
(task-history/with-task-run (when run-type
{:run_type run-type
@@ -285,14 +302,46 @@
(analytics/inc! :metabase-sync/failures {:driver (name (:engine database))}))
result)))))
+(mu/defn do-sync-operation
+ "Internal implementation of [[sync-operation]]; use that instead of calling this directly. Runs
+ only when the database is eligible for *automatic* sync (see
+ [[metabase.warehouses.models.database/should-auto-sync?]]) — i.e. it is suppressed when the
+ `disable-auto-sync` setting is on."
+ [operation :- :keyword
+ database :- (ms/InstanceOf :model/Database)
+ message :- ms/NonBlankString
+ f :- fn?]
+ (when (database/should-auto-sync? database)
+ (do-sync-operation* operation database message f)))
+
+(mu/defn do-explicit-sync-operation
+ "Internal implementation of [[explicit-sync-operation]]; use that instead of calling this directly.
+ For explicit, user-requested syncs (e.g. the Sync-now button): runs whenever the database is
+ syncable at all (see [[metabase.warehouses.models.database/should-sync?]]), ignoring the
+ `disable-auto-sync` setting, which suppresses only automatically-triggered syncs."
+ [operation :- :keyword
+ database :- (ms/InstanceOf :model/Database)
+ message :- ms/NonBlankString
+ f :- fn?]
+ (when (database/should-sync? database)
+ (do-sync-operation* operation database message f)))
+
(defmacro sync-operation
"Perform the operations in `body` as a sync operation, which wraps the code in several special macros that do things
like error handling, logging, duplicate operation prevention, and event publishing. Intended for use with the
- various top-level sync operations, such as `sync-metadata` or `analyze`."
+ various top-level sync operations, such as `sync-metadata` or `analyze`. Suppressed when
+ `disable-auto-sync` is on; for explicit user-requested syncs use [[explicit-sync-operation]]."
{:style/indent 3}
[operation database message & body]
`(do-sync-operation ~operation ~database ~message (fn [] ~@body)))
+(defmacro explicit-sync-operation
+ "Like [[sync-operation]], but for explicit, user-requested syncs: the work runs even when the
+ `disable-auto-sync` setting is enabled (that setting suppresses only automatically-triggered syncs)."
+ {:style/indent 3}
+ [operation database message & body]
+ `(do-explicit-sync-operation ~operation ~database ~message (fn [] ~@body)))
+
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | EMOJI PROGRESS METER |
;;; +----------------------------------------------------------------------------------------------------------------+
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index cde9cf2d314..e2ae2fb72e9 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -264,6 +264,24 @@
table
(group-perm-defaults table all-users-group non-magic-groups non-admin-groups)))))
+(defn set-new-tables-permissions!
+ "Batch variant of [[set-new-table-permissions!]] for many newly-inserted
+ tables on the same database. Acquires the cluster lock once and delegates
+ to [[perms/set-default-table-permissions-bulk!]]. All `tables` must share
+ `db-id`."
+ [db-id tables]
+ (when (seq tables)
+ (perms/with-db-scoped-permissions-lock db-id
+ (let [all-users-group (perms/all-users-group)
+ non-magic-groups (perms/non-magic-groups)
+ non-admin-groups (conj non-magic-groups all-users-group)
+ tables+defaults (mapv (fn [t]
+ [t (group-perm-defaults t all-users-group
+ non-magic-groups
+ non-admin-groups)])
+ tables)]
+ (perms/set-default-table-permissions-bulk! db-id tables+defaults)))))
+
(t2/define-after-insert :model/Table
[table]
(u/prog1 table
diff --git a/src/metabase/warehouses/core.clj b/src/metabase/warehouses/core.clj
index fa2767153b8..9c3c49d718a 100644
--- a/src/metabase/warehouses/core.clj
+++ b/src/metabase/warehouses/core.clj
@@ -13,7 +13,8 @@
[metabase.warehouses.provider-detection
detect-provider-from-database]
[metabase.warehouses.settings
- cloud-gateway-ips]
+ cloud-gateway-ips
+ disable-auto-sync]
[metabase.warehouses.util
get-database
test-connection-details
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 2ddb7c60c42..0f81de35354 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -28,6 +28,7 @@
[metabase.util.malli :as mu]
[metabase.util.quick-task :as quick-task]
[metabase.warehouses.provider-detection :as provider-detection]
+ [metabase.warehouses.settings :as warehouses.settings]
[methodical.core :as methodical]
[toucan2.core :as t2]
[toucan2.pipeline :as t2.pipeline]
@@ -204,16 +205,27 @@
(boolean (:router_database_id db)))
(defn should-sync?
- "Should this database be synced?"
+ "Should this database be synced at all? Destination (router-child) databases are never synced.
+ This is the gate for *explicit*, user-requested syncs (e.g. the Sync-now button), which run
+ regardless of the `disable-auto-sync` setting. For automatically-triggered syncs use
+ [[should-auto-sync?]]."
[db]
(not (is-destination? db)))
+(defn should-auto-sync?
+ "Should this database be synced *automatically* — scheduled sync/analyze, the post-add initial
+ sync, and Quartz trigger registration? False when the database isn't syncable at all, or when
+ auto-sync is globally suppressed via the `disable-auto-sync` setting."
+ [db]
+ (and (should-sync? db)
+ (not (warehouses.settings/disable-auto-sync))))
+
(defn- check-and-schedule-tasks-for-db!
"(Re)schedule sync operation tasks for `database`. (Existing scheduled tasks will be deleted first.)"
[database]
(try
;; this is done this way to avoid circular dependencies
- (when (should-sync? database)
+ (when (should-auto-sync? database)
((requiring-resolve 'metabase.sync.task.sync-databases/check-and-schedule-tasks-for-db!) database))
(catch Throwable e
(log/error e "Error scheduling tasks for DB"))))
diff --git a/src/metabase/warehouses/settings.clj b/src/metabase/warehouses/settings.clj
index adbef52d48b..dca974722c5 100644
--- a/src/metabase/warehouses/settings.clj
+++ b/src/metabase/warehouses/settings.clj
@@ -5,6 +5,20 @@
[metabase.settings.core :as setting :refer [defsetting]]
[metabase.util.i18n :refer [deferred-tru]]))
+(defsetting disable-auto-sync
+ (deferred-tru
+ (str "When true, suppresses automatically-triggered syncs: the scheduled sync-and-analyze and "
+ "update-field-values jobs do not run (and new triggers are not registered), and adding a "
+ "new database does not kick off an initial sync. "
+ "Syncs originating from an explicit request — the Sync-now REST endpoints, or a transform "
+ "finalizing its output table — are unaffected. "
+ "For deployments that load database metadata from disk at startup and should not have "
+ "Metabase re-discover it."))
+ :type :boolean
+ :default false
+ :visibility :internal
+ :export? false)
+
(defsetting cloud-gateway-ips
(deferred-tru "Metabase Cloud gateway IP addresses, to configure connections to DBs behind firewalls")
:visibility :public
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.clj
index 901bfec98cf..4344884c988 100644
--- a/src/metabase/warehouses_rest/api.clj
+++ b/src/metabase/warehouses_rest/api.clj
@@ -1152,8 +1152,9 @@
(quick-task/submit-task!
(fn []
(database-routing/with-database-routing-off
- (sync/sync-db-metadata! db)
- (sync/analyze-db! db))))
+ ;; explicit, user-requested sync — runs even when `disable-auto-sync` is enabled
+ (sync/sync-db-metadata-explicit! db)
+ (sync/analyze-db-explicit! db))))
{:status :ok}))))
;; TODO (Cam 10/28/25) -- fix this endpoint route to use kebab-case for consistency with the rest of our REST API
diff --git a/test/metabase/sync/events/sync_database_test.clj b/test/metabase/sync/events/sync_database_test.clj
new file mode 100644
index 00000000000..e4ba7c1cf41
--- /dev/null
+++ b/test/metabase/sync/events/sync_database_test.clj
@@ -0,0 +1,27 @@
+(ns metabase.sync.events.sync-database-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.events.core :as events]
+ [metabase.sync.sync :as sync]
+ [metabase.sync.sync-metadata :as sync-metadata]
+ [metabase.test :as mt]
+ [metabase.util.quick-task :as quick-task]))
+
+(set! *warn-on-reflection* true)
+
+(deftest database-create-event-respects-disable-auto-sync-test
+ (testing "The :event/database-create handler kicks off a sync on a new thread by default"
+ (mt/with-temp [:model/Database {:as db} {:is_full_sync true}]
+ (let [calls (atom 0)]
+ (with-redefs [quick-task/submit-task! (fn [task] (task))
+ sync/sync-database! (fn [_] (swap! calls inc))
+ sync-metadata/sync-db-metadata! (fn [_] (swap! calls inc))]
+ (testing "default (disable-auto-sync=false): a sync entry point is invoked"
+ (reset! calls 0)
+ (events/publish-event! :event/database-create {:object db :user-id (mt/user->id :crowberto)})
+ (is (= 1 @calls)))
+ (testing "with disable-auto-sync=true: no sync entry point is invoked"
+ (reset! calls 0)
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (events/publish-event! :event/database-create {:object db :user-id (mt/user->id :crowberto)}))
+ (is (zero? @calls))))))))
diff --git a/test/metabase/sync/task/sync_databases_test.clj b/test/metabase/sync/task/sync_databases_test.clj
index 203bd602787..06288b07f7f 100644
--- a/test/metabase/sync/task/sync_databases_test.clj
+++ b/test/metabase/sync/task/sync_databases_test.clj
@@ -7,6 +7,7 @@
[clojure.test :refer :all]
[clojurewerkz.quartzite.conversion :as qc]
[java-time.api :as t]
+ [metabase.sync.field-values :as sync.field-values]
[metabase.sync.schedules :as sync.schedules]
[metabase.sync.task.sync-databases :as task.sync-databases]
[metabase.task.core :as task]
@@ -172,6 +173,34 @@
(from-job-data [this]
(.getMergedJobDataMap this)))
+(deftest scheduled-jobs-respect-disable-auto-sync-test
+ (testing "When disable-auto-sync=true, scheduled job fns no-op even if a stale trigger fires"
+ (mt/with-temp [:model/Database {db-id :id} {:is_full_sync true}]
+ (testing "SyncAndAnalyzeDatabase: inner sync orchestrator is skipped when the flag is on"
+ (let [calls (atom 0)]
+ (with-redefs [task.sync-databases/sync-and-analyze-database*! (fn [_] (swap! calls inc))]
+ (testing "default (flag=false): job proceeds and calls the inner orchestrator"
+ (reset! calls 0)
+ (#'task.sync-databases/sync-and-analyze-database! (MockJobExecutionContext. {"db-id" db-id}))
+ (is (= 1 @calls)))
+ (testing "flag=true: job returns early; the inner orchestrator is not called"
+ (reset! calls 0)
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (#'task.sync-databases/sync-and-analyze-database! (MockJobExecutionContext. {"db-id" db-id})))
+ (is (zero? @calls))))))
+ (testing "UpdateFieldValues: field-values update is skipped when the flag is on"
+ (let [calls (atom 0)]
+ (with-redefs [sync.field-values/update-field-values! (fn [_] (swap! calls inc))]
+ (testing "default (flag=false): job proceeds and calls update-field-values!"
+ (reset! calls 0)
+ (#'task.sync-databases/update-field-values! (MockJobExecutionContext. {"db-id" db-id}))
+ (is (= 1 @calls)))
+ (testing "flag=true: job returns early; update-field-values! is not called"
+ (reset! calls 0)
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (#'task.sync-databases/update-field-values! (MockJobExecutionContext. {"db-id" db-id})))
+ (is (zero? @calls)))))))))
+
(deftest check-orphaned-jobs-removed-test
(testing "jobs for orphaned databases are removed during sync run"
(with-scheduler-setup!
diff --git a/test/metabase/transforms_base/util_test.clj b/test/metabase/transforms_base/util_test.clj
index 640a6e83dfa..86be4dcf9c3 100644
--- a/test/metabase/transforms_base/util_test.clj
+++ b/test/metabase/transforms_base/util_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer :all]
[java-time.api :as t]
[metabase.config.core :as config]
+ [metabase.sync.sync :as sync]
[metabase.test :as mt]
[metabase.transforms-base.util :as transforms-base.u]))
@@ -26,6 +27,21 @@
{:name "Routing transform"}
(mt/db)))))))))
+(deftest activate-table-syncs-despite-disable-auto-sync-test
+ (testing "disable-auto-sync gates *automatic* syncs only; a transform finalizing its"
+ (testing "output table still calls sync/sync-table! so the new table's fields are populated."
+ (let [calls (atom 0)]
+ (mt/with-temp [:model/Table _ {:db_id (mt/id)
+ :schema nil
+ :name "disable_auto_sync_target"}]
+ (with-redefs [sync/sync-table! (fn [_] (swap! calls inc))]
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (transforms-base.u/activate-table-and-mark-computed!
+ (mt/db)
+ {:type "table" :schema nil :name "disable_auto_sync_target"}))
+ (is (= 1 @calls)
+ "Expected the transform path to run sync/sync-table! exactly once with disable-auto-sync on.")))))))
+
(deftest ^:parallel full-incremental-run?-test
(testing "true for an incremental transform with no checkpoint yet"
(is (true? (transforms-base.u/full-incremental-run?
diff --git a/test/metabase/warehouses/models/database_test.clj b/test/metabase/warehouses/models/database_test.clj
index 35bb4596c27..a216717826e 100644
--- a/test/metabase/warehouses/models/database_test.clj
+++ b/test/metabase/warehouses/models/database_test.clj
@@ -34,6 +34,30 @@
(use-fixtures :once (fixtures/initialize :db :plugins :test-drivers))
+(deftest should-auto-sync?-respects-disable-auto-sync-test
+ (testing "should-auto-sync? gates automatically-triggered syncs on the disable-auto-sync setting,"
+ (testing "so that newly inserted or edited databases skip Quartz trigger registration."
+ (mt/with-temp [:model/Database db {}]
+ (testing "default (disable-auto-sync=false): a regular database is auto-sync-eligible"
+ (is (true? (database/should-auto-sync? db))))
+ (testing "with disable-auto-sync=true: a regular database is not auto-sync-eligible"
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (is (false? (database/should-auto-sync? db)))))))
+ (testing "but should-sync? (the gate for explicit, user-requested syncs) ignores the setting"
+ (mt/with-temp [:model/Database db {}]
+ (is (true? (database/should-sync? db)))
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (is (true? (database/should-sync? db))
+ "an explicit Sync-now must remain available even when auto-sync is disabled")))))
+ (testing "destination databases (router children) are excluded from both, regardless of the flag"
+ (mt/with-temp [:model/Database router {}
+ :model/Database dest {:router_database_id (:id router)}]
+ (is (false? (database/should-sync? dest)))
+ (is (false? (database/should-auto-sync? dest)))
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (is (false? (database/should-sync? dest)))
+ (is (false? (database/should-auto-sync? dest)))))))
+
(defn- trigger-for-db [db-id]
(some (fn [{trigger-key :key, :as trigger}]
(when (str/ends-with? trigger-key (str \. db-id))
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index 271389ac1e2..9662d7b4077 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -1529,10 +1529,11 @@
analyze-called? (promise)]
(mt/with-premium-features #{:audit-app}
(mt/with-temp [:model/Database {db-id :id} {:engine "h2", :details (:details (mt/db))}]
- ;; redefine quick-task/submit-task! so as not to depend on the capacity of the quick-task executor
- (with-redefs [quick-task/submit-task! future-call
- sync-metadata/sync-db-metadata! (deliver-when-db sync-called? db-id)
- analyze/analyze-db! (deliver-when-db analyze-called? db-id)]
+ ;; redefine quick-task/submit-task! so as not to depend on the capacity of the quick-task executor.
+ ;; The Sync-now endpoint dispatches to the *explicit* sync fns (which bypass disable-auto-sync).
+ (with-redefs [quick-task/submit-task! future-call
+ sync-metadata/sync-db-metadata-explicit! (deliver-when-db sync-called? db-id)
+ analyze/analyze-db-explicit! (deliver-when-db analyze-called? db-id)]
(snowplow-test/with-fake-snowplow-collector
(mt/user-http-request :crowberto :post 200 (format "database/%d/sync_schema" db-id))
;; Block waiting for the promises from sync and analyze to be delivered. Should be delivered instantly,
@@ -1551,13 +1552,39 @@
{"event" "database_manual_sync", "target_id" db-id}
(:data (last (snowplow-test/pop-event-data-and-user-id!)))))))))))))
+(deftest manual-sync-schema-runs-despite-disable-auto-sync-test
+ (testing (str "POST /api/database/:id/sync_schema is an explicit-request sync and must actually run "
+ "(not merely dispatch) when disable-auto-sync=true — the setting suppresses only "
+ "automatically-triggered syncs.")
+ ;; Resolve the test-data H2 connection details OUTSIDE the disable-auto-sync window, so loading the
+ ;; reference DB isn't itself suppressed. Create the target DB INSIDE the window so its creation event
+ ;; doesn't auto-sync it — then the only thing that can flip initial_sync_status to "complete" is the
+ ;; explicit Sync-now request under test. We deliberately do NOT stub sync-db-metadata!/analyze-db!:
+ ;; the real sync body must execute so the should-sync? gate inside do-sync-operation is genuinely
+ ;; exercised. The submit-task! redef runs the sync synchronously (so we can assert its effect) with
+ ;; H2 connections permitted on whatever thread the endpoint dispatches to.
+ (let [details (:details (mt/db))]
+ (mt/with-temporary-setting-values [disable-auto-sync true]
+ (mt/with-temp [:model/Database {db-id :id} {:engine "h2"
+ :details details
+ :initial_sync_status "incomplete"}]
+ (with-redefs [quick-task/submit-task! (fn [f]
+ (binding [driver.settings/*allow-testing-h2-connections* true]
+ (f)))]
+ (mt/user-http-request :crowberto :post 200 (format "database/%d/sync_schema" db-id)))
+ (testing "the explicit sync actually ran, not merely dispatched"
+ (is (= "complete" (t2/select-one-fn :initial_sync_status :model/Database :id db-id))
+ "Sync-now must complete the sync even when disable-auto-sync is on")
+ (is (pos? (t2/count :model/Table :db_id db-id))
+ "Sync-now must populate tables even when disable-auto-sync is on")))))))
+
(deftest sync-schema-executes-when-executor-busy-test
(testing "POST /api/database/:id/sync_schema should execute sync even when quick-task executor is busy (GHY-3254)"
(let [sync-called? (promise)
blocker-latch (CountDownLatch. 1)]
(mt/with-temp [:model/Database {db-id :id} {:engine "h2" :details (:details (mt/db))}]
- (with-redefs [sync-metadata/sync-db-metadata! (deliver-when-db sync-called? db-id)
- analyze/analyze-db! (constantly nil)]
+ (with-redefs [sync-metadata/sync-db-metadata-explicit! (deliver-when-db sync-called? db-id)
+ analyze/analyze-db-explicit! (constantly nil)]
;; Submit a blocking task with a 1-second timeout so it gets cancelled quickly.
;; This simulates a stuck sync (e.g., hanging JDBC connection) that exceeds
;; the quick-task timeout and gets evicted.