The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Metadata File Import (#74524)
Timothy S. Dean2fe12d63ee94
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex 4654efeccf1..35074be78cb 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -429,6 +429,7 @@instaparse.core/defparser clojure.core/defmetabase-enterprise.action-v2.test-util/with-test-tables! clojure.core/letmetabase-enterprise.content-translation.routes-test/with-new-secret-key! clojure.core/fn+ metabase-enterprise.serialization.metadata-file-import.wire-format-test/with-tiny-fixture! clojure.core/fnmetabase-enterprise.serialization.test-util/with-dbs clojure.core/fnmetabase-enterprise.serialization.test-util/with-random-dump-dir clojure.core/letmetabase-enterprise.workspaces.test-util/with-resources! clojure.core/letdiff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex f343247aba0..ab49279d886 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -3308,6 +3308,7 @@:api #{metabase-enterprise.serialization.apimetabase-enterprise.serialization.cmdmetabase-enterprise.serialization.core+ metabase-enterprise.serialization.initmetabase-enterprise.serialization.v2.backfill-ids}:uses #{analyticsapi@@ -3324,7 +3325,9 @@searchserversetup- util}+ sync+ util+ warehouse-schema}:model-imports :bypass}enterprise/snippet-collectionsdiff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.cljindex 31dc597a85e..8abb55639c3 100644--- a/enterprise/backend/src/metabase_enterprise/core/init.clj+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj@@ -20,6 +20,7 @@[metabase-enterprise.scim.init][metabase-enterprise.security-center.init][metabase-enterprise.semantic-search.init]+ [metabase-enterprise.serialization.init][metabase-enterprise.sso.init][metabase-enterprise.stale.init][metabase-enterprise.support-access-grants.init]diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.cljindex 440a8399c21..39a5e8d1d4c 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj@@ -4,6 +4,7 @@[clojure.string :as str][java-time.api :as t][metabase-enterprise.serialization.export :as export]+ [metabase-enterprise.serialization.metadata-file-import :as metadata-file-import][metabase-enterprise.serialization.schema :as schema][metabase-enterprise.serialization.v2.extract :as extract][metabase-enterprise.serialization.v2.ingest :as v2.ingest]@@ -26,9 +27,10 @@[metabase.util.malli :as mu][metabase.util.malli.schema :as ms][metabase.util.random :as u.random]- [ring.core.protocols :as ring.protocols])+ [ring.core.protocols :as ring.protocols]+ [ring.util.response :as response])(:import- (java.io ByteArrayOutputStream File)))+ (java.io ByteArrayOutputStream File InputStream)))(set! *warn-on-reflection* true)@@ -332,6 +334,83 @@(sr/streaming-response {:content-type "application/json; charset=utf-8"} [os _](export/export-metadata! os opts))))+;;; ----------------------------------- POST /api/ee/serialization/metadata/import -----------------------------------++(defn- spool-to-temp-file!+ "Stream `is` to a freshly-created temp file in 64 KB chunks. Returns the+ `File`. If streaming fails the temp file is removed before the throw+ propagates."+ ^File [^InputStream is]+ (let [tmp (File/createTempFile "metadata-import-" ".json")]+ (try+ (with-open [os (io/output-stream tmp)]+ (io/copy is os :buffer-size (* 64 1024)))+ tmp+ (catch Throwable t+ (.delete tmp)+ (throw t)))))++(api.macros/defendpoint :post "/metadata/import"+ :- [:map+ [:status [:= 202]]+ [:body [:map {:closed true}+ [:queued :boolean]+ [:import-id :string]]]]+ "Import warehouse metadata previously emitted by `POST /metadata/export`. The+ request body is the JSON document `{databases, tables, fields}`; sections are+ parsed incrementally so memory stays bounded regardless of payload size.++ To bypass the JSON-parsing request middleware, send with `Content-Type:+ application/octet-stream`. Restricted to superusers.++ Returns `202` immediately with an `:import-id`; the import runs+ asynchronously. Poll `GET /metadata/import/:id` for its outcome."+ [_route-params+ _query-params+ _body+ {:keys [body], :as _request}]+ (api/check-superuser)+ (cond+ (nil? body)+ (throw (ex-info "Empty request body" {:status-code 400}))++ (not (instance? InputStream body))+ (throw (ex-info (str "Expected a raw stream body. Send the request with "+ "Content-Type: application/octet-stream so the JSON "+ "middleware does not pre-parse the payload.")+ {:status-code 415}))++ :else+ (let [tmp (spool-to-temp-file! body)+ id (metadata-file-import/enqueue-import! tmp {:delete-after? true})]+ (-> (response/response {:queued true :import-id id})+ (assoc :status 202)))))++;;; ------------------------------- GET /api/ee/serialization/metadata/import/:id -------------------------------++(defn- present-import-status+ "Project an internal registry record onto the public wire shape. Deliberately+ omits the server-side `:file` path; stringifies the status keyword and the+ timestamps."+ [record]+ {:id (:id record)+ :status (name (:status record))+ :enqueued-at (str (:enqueued-at record))+ :started-at (some-> (:started-at record) str)+ :finished-at (some-> (:finished-at record) str)+ :wall-ms (:wall-ms record)+ :error (:error record)})++(api.macros/defendpoint :get "/metadata/import/:id"+ :- ::schema/import-status-response+ "Status of a metadata import previously started by `POST /metadata/import`.+ Status is retained in-memory and is not durable across server restarts.+ Restricted to superusers."+ [{:keys [id]} :- [:map [:id ms/UUIDString]]]+ (api/check-superuser)+ (or (some-> (metadata-file-import/import-status id) present-import-status)+ (throw (ex-info "Unknown or expired import id" {:status-code 404}))))+(def ^{:arglists '([request respond raise])} routes"`/api/ee/serialization` routes."(api.macros/ns-handler *ns* +auth))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/core.clj b/enterprise/backend/src/metabase_enterprise/serialization/core.cljindex a1a88a32d9e..d4269100354 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/core.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/core.clj@@ -1,6 +1,10 @@(ns metabase-enterprise.serialization.core(:require[metabase-enterprise.serialization.dump]+ [metabase-enterprise.serialization.export]+ ;; Required (without alias) so loading `serialization.core` populates the malli+ ;; registry — callers can then refer to `::serialization.schema/...` keywords.+ [metabase-enterprise.serialization.schema][metabase-enterprise.serialization.v2.extract][metabase-enterprise.serialization.v2.ingest][metabase-enterprise.serialization.v2.load]@@ -11,6 +15,7 @@(commentmetabase-enterprise.serialization.dump/keep-me+ metabase-enterprise.serialization.export/keep-memetabase-enterprise.serialization.v2.extract/keep-memetabase-enterprise.serialization.v2.ingest/keep-memetabase-enterprise.serialization.v2.load/keep-me@@ -21,6 +26,8 @@(p/import-vars[metabase-enterprise.serialization.dumpserialization-deep-sort]+ [metabase-enterprise.serialization.export+ export-metadata!][metabase-enterprise.serialization.v2.extractmake-targets-of-typeextract]diff --git a/enterprise/backend/src/metabase_enterprise/serialization/init.clj b/enterprise/backend/src/metabase_enterprise/serialization/init.cljnew file mode 100644index 00000000000..b6cd5a49416--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/init.clj@@ -0,0 +1,5 @@+(ns metabase-enterprise.serialization.init+ (:require+ [metabase-enterprise.serialization.metadata-file-import :as metadata-file-import]))++(metadata-file-import/init!)diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import.cljnew file mode 100644index 00000000000..99c549d229f--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import.clj@@ -0,0 +1,308 @@+(ns metabase-enterprise.serialization.metadata-file-import+ "Streams a metadata file into the appdb, populating+ `:model/Database` (matched, never created), `:model/Table`, and `:model/Field`.++ Composes three building blocks:++ - [[metabase-enterprise.serialization.metadata-file-import.parsers]] streams+ batches of `[line-num row]` tuples from a JSON file.+ - [[metabase-enterprise.serialization.metadata-file-import.processors]] provides+ the per-batch and per-depth SQL building blocks.+ - [[metabase-enterprise.serialization.metadata-file-import.schemas]] defines+ the per-row Malli schemas, shared with the parsers' callers.++ Strict-consistency invariants enforced by the pre-flight + depth-tagging+ steps mean the merge loop has no orphan-handling code paths. The whole+ merge is set-based SQL, one statement per phase per depth level."+ (:require+ [metabase-enterprise.serialization.metadata-file-import.parsers :as parsers]+ [metabase-enterprise.serialization.metadata-file-import.processors :as processors]+ [metabase.app-db.core :as mdb]+ [metabase.sync.util :as sync.util]+ [metabase.util.log :as log]+ [metabase.warehouse-schema.models.table :as schema.table]+ [toucan2.core :as t2])+ (:import+ (java.io File)+ (java.sql Connection PreparedStatement)+ (java.time Instant)+ (java.util UUID)+ (org.postgresql PGConnection)+ (org.postgresql.copy CopyIn CopyManager)))++(set! *warn-on-reflection* true)++;;; ============================== drain + database matching ==============================++(defn- process-databases-batch!+ "Per-batch handler for `:databases`. Updates `databases-by-source-id`+ with `source-id → name` for every row, conjs target-db-ids onto+ `matched-ids` for matches, and WARN-logs unmatched rows."+ [matched-ids databases-by-source-id batch]+ (reduce (fn [_ result]+ (vswap! databases-by-source-id assoc (:source-id result) (:name result))+ (case (:status result)+ :matched+ (vswap! matched-ids conj (:target-id result))++ :no-match+ (log/warnf "metadata-file-import: skipped source database %s (no matching target): %s"+ (pr-str (:name result)) (:detail result)))+ nil)+ nil+ (processors/process-databases! batch)))++(defn- drain-via-jdbc-batch!+ "Drive parser batches through JDBC `executeBatch` against per-staging-table+ `PreparedStatement`s. The fallback path for non-Postgres app-dbs."+ [^Connection conn ^File file matched-ids databases-by-source-id]+ (with-open [^PreparedStatement tables-ps (.prepareStatement conn (processors/tables-insert-sql))+ ^PreparedStatement fields-ps (.prepareStatement conn (processors/fields-insert-sql))]+ (parsers/stream-keyed-arrays!+ file processors/import-batch-size+ {:databases (partial process-databases-batch! matched-ids databases-by-source-id)+ :tables (fn [batch] (processors/drain-tables-batch-jdbc! tables-ps @databases-by-source-id batch))+ :fields (fn [batch] (processors/drain-fields-batch-jdbc! fields-ps batch))})))++;; Only one COPY can be active per connection at a time, so we track the+;; current handle (`:tables` or `:fields`) and close it before opening the next.+(defn- drain-via-pg-copy!+ "Drive parser batches through PG's COPY wire protocol via `CopyManager`,+ streaming TSV-formatted bytes directly to the server."+ [^Connection conn ^File file matched-ids databases-by-source-id]+ (let [^PGConnection pg-conn (.unwrap conn PGConnection)+ ^CopyManager copy-mgr (.getCopyAPI pg-conn)+ current (volatile! {:kind nil :copy-in nil})+ close-current! (fn []+ (when-let [^CopyIn ci (:copy-in @current)]+ (.endCopy ci)+ (vreset! current {:kind nil :copy-in nil})))+ ensure-copy! (fn [kind ^String sql]+ (when (not= kind (:kind @current))+ (close-current!)+ (vreset! current {:kind kind :copy-in (.copyIn copy-mgr sql)})))]+ (try+ (parsers/stream-keyed-arrays!+ file processors/import-batch-size+ {:databases (partial process-databases-batch! matched-ids databases-by-source-id)+ :tables (fn [batch]+ (ensure-copy! :tables processors/tables-copy-sql)+ (processors/drain-tables-batch-pg-copy! (:copy-in @current) @databases-by-source-id batch))+ :fields (fn [batch]+ (ensure-copy! :fields processors/fields-copy-sql)+ (processors/drain-fields-batch-pg-copy! (:copy-in @current) batch))})+ (close-current!)+ (catch Throwable t+ (try (close-current!) (catch Throwable _ nil))+ (throw t)))))++;; Opens one JDBC connection with autoCommit=false and commits at the end.+;; c3p0's autoCommitOnClose=false means we must explicitly restore+;; autoCommit=true before returning the connection to the pool — otherwise the+;; next pool user inherits an open-transaction-capable connection and can leak+;; an idle-in-transaction backend.+(defn- drain-and-match-databases!+ "Single-pass walk of `file`: match `:databases` against the live appdb,+ drain `:tables` and `:fields` into staging. Returns the set of matched+ target-db-ids."+ [^File file]+ (let [matched-ids (volatile! #{})+ databases-by-source-id (volatile! {})+ ds (mdb/data-source)]+ (with-open [^Connection conn (.getConnection ds)]+ (.setAutoCommit conn false)+ (try+ ;; PG COPY fast path; JDBC executeBatch fallback for H2/MySQL.+ (case (mdb/db-type)+ :postgres (drain-via-pg-copy! conn file matched-ids databases-by-source-id)+ (drain-via-jdbc-batch! conn file matched-ids databases-by-source-id))+ (.commit conn)+ (catch Throwable t+ (try (.rollback conn) (catch Throwable _ nil))+ (throw t))+ (finally+ (try (.setAutoCommit conn true) (catch Throwable _ nil)))))+ @matched-ids))++;;; ============================== Top-level orchestration ==============================++;; Writes via raw `:metabase_database` to bypass `:model/Database`'s+;; `:after-update` hook, which would schedule per-database Quartz sync triggers+;; — undesirable here, since the whole point of metadata-file-import is to skip+;; warehouse sync. App startup re-registers triggers.+(defn- mark-databases-sync-complete!+ "Flip every matched target Database's `initial_sync_status` to `\"complete\"`+ so the UI surfaces tables immediately."+ [matched-target-db-ids]+ (when (seq matched-target-db-ids)+ (t2/query {:update :metabase_database+ :set {:initial_sync_status "complete"}+ :where [:in :id (vec matched-target-db-ids)]})))++(defn import-metadata-file!+ "Run the full metadata import pipeline against the given file.++ `metadata-file` — `java.io.File` or path-string for the metadata file+ (databases / tables / fields).++ Returns `:ok` on success; throws on hard-fail conditions (file not+ readable, file incomplete, cycle in reference graph, mid-merge errors)."+ [metadata-file]+ (let [^File m-file (if (instance? File metadata-file)+ metadata-file+ (File. ^String metadata-file))]+ ;; with-staging-tables clears the staging tables on entry and exit+ ;; (try/finally), so a crashed prior attempt can't leak rows into this run.+ (processors/with-staging-tables+ ;; --- drain + database matching (single pass over the file) ---+ (let [matched-target-db-ids (drain-and-match-databases! m-file)]+ ;; --- analyze: feed the planner real selectivity for downstream UPDATEs ---+ (processors/analyze-staging-tables!)+ ;; --- pre-flight: every parent ref must resolve within the file ---+ (processors/assert-no-orphan-refs!)+ ;; --- pre-flight: NULL orphan fk-target refs (informational, not fatal) ---+ (let [{:keys [count sample]} (processors/null-orphan-fk-target-refs!)]+ (when (pos? count)+ (log/warnf "metadata-file-import: %d orphan fk_target_field_id ref(s) NULLed (target not present in file); sample=%s"+ count (pr-str sample))))+ ;; --- depth tagging: assign 0..max-depth, detect cycles ---+ (processors/compute-staging-depth!)+ ;; --- table resolve round 1: matched-vs-insert decision for merge-tables! ---+ (processors/resolve-target-table-ids-in-staging!)+ ;; --- merge (one txn — every live-data write all-or-nothing) ---+ (let [new-tables+ (t2/with-transaction [_]+ (let [insert-source-ids (processors/merge-tables!)]+ ;; round 2 captures INSERT-assigned ids back into table staging+ (processors/resolve-target-table-ids-in-staging!)+ ;; copy target-table-ids from table staging onto field staging+ (processors/resolve-target-table-ids-for-fields-in-staging!)+ ;; depth-walk: per-depth resolve + UPDATE matched + INSERT new+ (processors/merge-fields-by-depth!)+ (mark-databases-sync-complete! matched-target-db-ids)+ (processors/new-target-tables-from-staging insert-source-ids)))]+ ;; Permission grants run outside the merge txn — the cluster lock+ ;; shouldn't be held across the long merge work.+ (doseq [[db-id rows] (group-by :db_id new-tables)]+ (schema.table/set-new-tables-permissions! db-id rows)))+ (log/infof "metadata-file-import: complete (matched-databases=%d)"+ (count matched-target-db-ids))))+ :ok))++;;; ============================== Import registry + agent ==============================+;;; In-JVM only — matches the scope of `metabase.sync.util/operation->db-ids`.+;;; Status is not durable across server restarts.++(def ^:private terminal-record-limit+ "Max number of completed (`:ok`/`:error`) records retained in the registry.+ In-flight (`:queued`/`:running`) records are never evicted regardless of count."+ 10)++;; id -> {:id :status :file :enqueued-at :started-at :finished-at :wall-ms :error}+(defonce ^:private import-registry (atom {}))++(defonce ^:private import-agent+ (agent ::serializer :error-mode :continue))++;; The id currently being run by the agent, or nil. A single-element index over+;; `import-registry` so the sync busy-check is O(1) instead of scanning.+(defonce ^:private running-import-id (atom nil))++(defn- terminal-status? [status]+ (contains? #{:ok :error} status))++(defn- prune-terminal-records+ "Drop the oldest terminal records past `terminal-record-limit`, ordered by+ `:finished-at`. Non-terminal records are retained regardless of count."+ [registry]+ (let [terminal (filter (comp terminal-status? :status val) registry)]+ (if (<= (count terminal) terminal-record-limit)+ registry+ (->> terminal+ (sort-by (comp :finished-at val))+ (drop-last terminal-record-limit)+ (map key)+ (apply dissoc registry)))))++(defn- finalize-record+ "Merge terminal `attrs` onto record `id`, then prune."+ [registry id attrs]+ (-> registry+ (update id merge attrs)+ prune-terminal-records))++(defn import-status+ "Registry record for import `id`, or nil if the id is unknown or has been+ evicted."+ [id]+ (get @import-registry id))++(defn import-running?+ "Truthy iff a metadata-file-import is currently in flight on this JVM."+ []+ (some? @running-import-id))++(defn- import-busy-reason []+ (when-let [id @running-import-id]+ (let [{:keys [file started-at]} (get @import-registry id)]+ {:reason (format "metadata-file-import in progress (file=%s, since=%s)"+ file started-at)})))++(defonce ^:private initialized? (atom false))++(defn init!+ "Register hooks with the subsystems this module coordinates with (sync).+ Called from `metabase-enterprise.serialization.init` at boot. Idempotent —+ callers don't need to guard against multiple invocations."+ []+ (when (compare-and-set! initialized? false true)+ (sync.util/register-busy-predicate! import-busy-reason)))++(defn- run-import*+ "Agent body. Always returns `::serializer`; `:error-mode :continue` plus+ the inner try/catch means a failing import never poisons the agent."+ [_serializer id ^File file {:keys [delete-after?]}]+ (let [path (.getAbsolutePath file)+ t0 (System/nanoTime)]+ (swap! import-registry update id merge {:status :running :started-at (Instant/now)})+ (reset! running-import-id id)+ (log/infof "metadata-file-import: starting (id=%s, file=%s)" id path)+ (try+ (import-metadata-file! file)+ (let [wall-ms (/ (- (System/nanoTime) t0) 1e6)]+ (log/infof "metadata-file-import: complete (id=%s, file=%s, wall-ms=%.0f)" id path wall-ms)+ (swap! import-registry finalize-record id+ {:status :ok :finished-at (Instant/now) :wall-ms wall-ms :error nil}))+ (catch Throwable t+ (let [wall-ms (/ (- (System/nanoTime) t0) 1e6)]+ (log/errorf t "metadata-file-import: failed (id=%s, file=%s, wall-ms=%.0f)" id path wall-ms)+ (swap! import-registry finalize-record id+ {:status :error :finished-at (Instant/now) :wall-ms wall-ms :error (str t)})))+ (finally+ (reset! running-import-id nil)+ (when delete-after?+ (try (.delete file) (catch Throwable _ nil)))))+ ::serializer))++(defn enqueue-import!+ "Submit `file` to the import agent and return its import id (a string UUID).+ Imports execute in arrival order; a `:queued` registry record is created+ synchronously before this returns, so the caller can immediately look the id+ up via `import-status`. With `{:delete-after? true}`, the agent deletes+ `file` once it finishes (success or failure)."+ ([^File file]+ (enqueue-import! file {}))+ ([^File file opts]+ (let [id (str (UUID/randomUUID))+ path (.getAbsolutePath file)]+ (swap! import-registry assoc id {:id id+ :status :queued+ :file path+ :enqueued-at (Instant/now)+ :started-at nil+ :finished-at nil+ :wall-ms nil+ :error nil})+ (log/infof "metadata-file-import: queued (id=%s, file=%s)" id path)+ (send-off import-agent run-import* id file opts)+ id)))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/parsers.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/parsers.cljnew file mode 100644index 00000000000..dcf85bbce5b--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/parsers.clj@@ -0,0 +1,132 @@+(ns metabase-enterprise.serialization.metadata-file-import.parsers+ "Streaming JSON parser for the metadata file importer. Walks a top-level JSON+ object, advances to a named array, and emits its items in batches via a+ callback. Memory bounded by the in-flight item plus the current batch buffer."+ (:require+ [metabase.util.performance :as perf])+ (:import+ (com.fasterxml.jackson.core JsonParser JsonToken)+ (com.fasterxml.jackson.databind ObjectMapper)+ (java.io File FileInputStream InputStreamReader Reader)+ (java.nio.charset StandardCharsets)+ (java.util LinkedHashMap)))++(set! *warn-on-reflection* true)++(def ^:private ^ObjectMapper object-mapper (ObjectMapper.))++(defn- advance-to-array!+ "Advance `parser` from start-of-input through a top-level JSON object to the+ array valued at `target-key` (parser has just consumed the START_ARRAY+ token). Throws `:bad-shape` if the document doesn't begin with an object or+ the value at `target-key` isn't an array; throws `:missing-key` if the key+ is absent."+ [^JsonParser parser ^String target-key]+ (let [t (.nextToken parser)]+ (when-not (= t JsonToken/START_OBJECT)+ (throw (ex-info "Expected JSON document to begin with an object"+ {:kind :bad-shape, :target-key target-key}))))+ (loop []+ (let [t (.nextToken parser)]+ (cond+ (or (nil? t) (= t JsonToken/END_OBJECT))+ (throw (ex-info (format "Key %s not found in top-level object" (pr-str target-key))+ {:kind :missing-key, :key target-key}))++ (= t JsonToken/FIELD_NAME)+ (if (= target-key (.getCurrentName parser))+ (let [vt (.nextToken parser)]+ (when-not (= vt JsonToken/START_ARRAY)+ (throw (ex-info (format "Value of %s must be an array" (pr-str target-key))+ {:kind :bad-shape, :key target-key}))))+ (do (.nextToken parser) (.skipChildren parser) (recur)))++ :else (recur)))))++(defn- consume-array-as-batches!+ "After START_ARRAY has been consumed, walk array items into batches of+ `[line-num row]` tuples (up to `batch-size`), calling `process-batch!` per+ batch. Returns when END_ARRAY is consumed."+ [^JsonParser parser array-key batch-size process-batch!]+ (loop [batch (transient [])+ line-num 0]+ (let [t (.nextToken parser)]+ (cond+ (= t JsonToken/END_ARRAY)+ (when (pos? (count batch))+ (process-batch! (persistent! batch)))++ (= t JsonToken/START_OBJECT)+ ;; LinkedHashMap fails (map? x), so coerce before keywordize-keys.+ (let [raw (.readValueAs parser LinkedHashMap)+ item (perf/keywordize-keys (into {} raw))+ ln (inc line-num)+ next-batch (conj! batch [ln item])]+ (if (>= (count next-batch) batch-size)+ (do (process-batch! (persistent! next-batch))+ (recur (transient []) ln))+ (recur next-batch ln)))++ (nil? t)+ (throw (ex-info (format "Unexpected end-of-input in array %s" (pr-str array-key))+ {:kind :bad-shape, :key array-key}))++ :else+ (throw (ex-info (format "Unexpected token %s in array %s"+ (some-> t .asString) (pr-str array-key))+ {:kind :bad-shape, :key array-key}))))))++(defn- open-parser+ ^JsonParser [^Reader reader]+ (.createParser (.getFactory object-mapper) reader))++(defn stream-array-batches!+ "Stream `file`'s top-level named array (string or keyword `array-key`) through+ `process-batch!`. Each batch is a vector of `[line-num row]` tuples (up to+ `batch-size`); `line-num` is 1-indexed and continues across batch boundaries.+ Items are parsed into Clojure maps with keyword keys. Caller's responsibility+ to ensure the file exists and is readable; opening errors propagate."+ [^File file array-key batch-size process-batch!]+ (with-open [is (FileInputStream. file)+ reader (InputStreamReader. is StandardCharsets/UTF_8)+ parser (open-parser reader)]+ (advance-to-array! parser (name array-key))+ (consume-array-as-batches! parser array-key batch-size process-batch!)))++(defn stream-keyed-arrays!+ "Single-pass walk of `file`'s top-level JSON object, dispatching arrays to+ per-key handlers. `handlers` is a map of keyword → fn-of-batch (each batch is+ a vector of `[line-num row]` tuples, up to `batch-size`). Arrays for keys not+ in `handlers` are skipped without materialization.++ Throws `:bad-shape` if the document doesn't begin with an object or if a+ known key's value isn't an array. Missing keys are silently OK."+ [^File file batch-size handlers]+ (with-open [is (FileInputStream. file)+ reader (InputStreamReader. is StandardCharsets/UTF_8)+ parser (open-parser reader)]+ (let [t (.nextToken parser)]+ (when-not (= t JsonToken/START_OBJECT)+ (throw (ex-info "Expected JSON document to begin with an object"+ {:kind :bad-shape}))))+ (loop []+ (let [t (.nextToken parser)]+ (cond+ (or (nil? t) (= t JsonToken/END_OBJECT))+ nil++ (= t JsonToken/FIELD_NAME)+ (let [field-name (.getCurrentName parser)+ handler-fn (get handlers (keyword field-name))]+ (if handler-fn+ (let [vt (.nextToken parser)]+ (when-not (= vt JsonToken/START_ARRAY)+ (throw (ex-info (format "Value of %s must be an array" (pr-str field-name))+ {:kind :bad-shape, :key field-name})))+ (consume-array-as-batches! parser field-name batch-size handler-fn)+ (recur))+ (do (.nextToken parser)+ (.skipChildren parser)+ (recur))))++ :else (recur))))))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/processors.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/processors.cljnew file mode 100644index 00000000000..898a1adb765--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/processors.clj@@ -0,0 +1,834 @@+(ns metabase-enterprise.serialization.metadata-file-import.processors+ "SQL building blocks for the metadata file importer's drain-then-merge flow.++ Errors propagate as `ex-info` with `:kind` for typed failure handling+ (`:invalid-input`, `:file-incomplete`, `:cycle-in-field-graph`, `:depth-tagging-cap-exceeded`)."+ (:require+ [malli.error :as me]+ [metabase-enterprise.serialization.metadata-file-import.schemas :as schemas]+ [metabase.app-db.core :as mdb]+ [metabase.models.humanization :as humanization]+ [metabase.util.json :as json]+ [metabase.util.malli.registry :as mr]+ [toucan2.core :as t2])+ (:import+ (java.nio.charset StandardCharsets)+ (java.sql PreparedStatement Types)+ (org.postgresql.copy CopyIn)))++(set! *warn-on-reflection* true)++(def import-batch-size+ "Row batch size shared by the drain and merge processors."+ ;; 250 was the sweet spot in local benchmarks+ 250)++;;; ============================== Validation ==============================++(defn- validate-line!+ "Validate `line` against the registered Malli `schema-ref`.+ On failure throws `ex-info` with `:kind :invalid-input`, `:line`, a humanized `:detail`,+ and `extras` merged into the ex-data."+ [schema-ref line-num line extras]+ (when-let [humanized (me/humanize (mr/explain schema-ref line))]+ (throw (ex-info (format "Invalid %s row on line %d" (name schema-ref) line-num)+ (merge extras+ {:kind :invalid-input+ :line line-num+ :detail (pr-str humanized)})))))++;;; ============================== Staging tables ==============================++(defn clear-staging-tables!+ "Empty `metabase_table_import` and `metabase_field_import`."+ []+ ;; TRUNCATE (not DELETE) so multiple imports in one process lifetime don't+ ;; accumulate dead tuples in staging.+ ;; Called outside any outer transaction, so MySQL's implicit-commit-on-+ ;; TRUNCATE doesn't break composition.+ (t2/query {:truncate :metabase_table_import})+ (t2/query {:truncate :metabase_field_import}))++(defn analyze-staging-tables!+ "Refresh planner statistics on `metabase_table_import` and `metabase_field_import`."+ []+ (case (mdb/db-type)+ :postgres+ (do (t2/query "ANALYZE metabase_table_import")+ (t2/query "ANALYZE metabase_field_import"))+ :mysql+ (do (t2/query "ANALYZE TABLE metabase_table_import")+ (t2/query "ANALYZE TABLE metabase_field_import"))+ :h2+ (t2/query "ANALYZE")))++(defmacro with-staging-tables+ "Run `body` with staging tables cleared on entry and on exit."+ [& body]+ `(do (clear-staging-tables!)+ (try ~@body+ (finally (clear-staging-tables!)))))++;;; ============================== databases (batch) ==============================++(defn- engine-name+ "Normalize `engine` (string or keyword) to a string for natural-key comparison.+ Toucan2's `:model/Database` reads `:engine` as a keyword via `define-after-select`,+ but file-imported rows always carry strings. Normalize both sides to string."+ [engine]+ (when engine (name engine)))++(defn- match-databases-batch+ "Look up every existing target Database matching any source row in the+ batch. Returns `{[name engine-string] → existing-id}` for matching rows."+ [lines]+ (let [pairs (into #{}+ (map (fn [{:keys [name engine]}] [name (engine-name engine)]))+ lines)+ names (into #{} (map first) pairs)+ engines (into #{} (map second) pairs)]+ (if (or (empty? names) (empty? engines))+ {}+ ;; Over-include via `(name IN ..., engine IN ...)` then intersect in+ ;; Clojure — keeps the SQL portable (no `(col, col) IN ((?, ?), ...)`+ ;; tuple form across Postgres / H2 / MySQL).+ (let [rows (t2/select [:model/Database :id :name :engine]+ {:where [:and+ [:in :name names]+ [:in :engine engines]]})]+ (into {}+ (comp (map (fn [{:keys [id name engine]}]+ [[name (engine-name engine)] id]))+ (filter (fn [[pair _]] (contains? pairs pair))))+ rows)))))++(defn process-databases!+ "Process a batch of database rows. Returns an eduction of result maps.++ Result shapes:+ `{:source-id <int> :name <string> :target-id <int> :status :matched}`+ `{:source-id <int> :name <string> :status :no-match :line L :detail S}`++ Validation failures throw; lookup misses produce `:no-match` results+ (non-fatal)."+ [batch]+ (doseq [[ln line] batch]+ (validate-line! ::schemas/database-info ln line {:source-id (:id line)}))+ (let [match-idx (match-databases-batch (mapv second batch))]+ (eduction+ (map (fn [[ln {:keys [id name engine]}]]+ (if-let [target (match-idx [name (engine-name engine)])]+ {:source-id id :name name :target-id target :status :matched}+ {:source-id id :name name+ :status :no-match+ :line ln+ :detail (format "No database with name=%s engine=%s"+ (pr-str name) (pr-str (engine-name engine)))})))+ batch)))++;;; ============================== tables — drain + merge ==============================++(defn- encode-path-or-nil+ "Encode an `:nfc_path` coll as a JSON string. NULL for empty/nil."+ [coll]+ (when (seq coll) (json/encode (vec coll))))++;;; ============================== JDBC drain (perf path) ==============================+;;;+;;; Hand-rolled JDBC `executeBatch` against single per-staging-table prepared+;;; statements. Used by the loader's `drain-and-match-databases!` to bypass+;;; the t2/insert! + HoneySQL VALUES formatting overhead — measured ~15×+;;; slower per row than PG's COPY floor on multi-million-row drains.++(defn tables-insert-sql+ "Parameterized INSERT for `metabase_table_import`. Column order matches the+ `set*-batch!` parameter-binding order in [[drain-tables-batch-jdbc!]]."+ []+ (str "INSERT INTO metabase_table_import"+ ;; `schema` is a reserved word in MySQL/MariaDB+ " (source_id, source_db_id, db_name, " (mdb/quote-for-application-db "schema")+ ", name, description, display_name)"+ " VALUES (?, ?, ?, ?, ?, ?, ?)"))++(defn fields-insert-sql+ "Parameterized INSERT for `metabase_field_import`. Column order matches the+ `set*-batch!` parameter-binding order in [[drain-fields-batch-jdbc!]]."+ []+ (str "INSERT INTO metabase_field_import"+ " (source_id, source_table_id, source_parent_id, source_fk_target_id,"+ " name, base_type, database_type, effective_type, semantic_type,"+ " coercion_strategy, description, nfc_path)"+ " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"))++(defn- set-int-or-null! [^PreparedStatement ps ^long idx v]+ (if v+ (.setInt ps idx (int v))+ (.setNull ps idx Types/INTEGER)))++(defn- set-string-or-null! [^PreparedStatement ps ^long idx v]+ (if v+ (.setString ps idx (str v))+ (.setNull ps idx Types/VARCHAR)))++(defn drain-tables-batch-jdbc!+ "Per-batch handler for `:tables` that binds rows onto `ps` and flushes via+ `executeBatch`. Validates each row first; throws on schema failure.++ `ps` must be a `PreparedStatement` for [[tables-insert-sql]]; the caller+ owns the connection + statement lifecycle."+ [^PreparedStatement ps databases-by-source-id batch]+ (doseq [[ln line] batch]+ (validate-line! ::schemas/table-info ln line {:source-id (:id line)}))+ (when (seq batch)+ (doseq [[_ {:keys [id db_id schema name description]}] batch]+ (.setInt ps 1 (int id))+ (.setInt ps 2 (int db_id))+ (.setString ps 3 (str (get databases-by-source-id db_id)))+ (set-string-or-null! ps 4 schema)+ (.setString ps 5 (str name))+ (set-string-or-null! ps 6 description)+ (.setString ps 7 (humanization/name->human-readable-name name))+ (.addBatch ps))+ (.executeBatch ps)))++(defn drain-fields-batch-jdbc!+ "Per-batch handler for `:fields` that binds rows onto `ps` and flushes via+ `executeBatch`. Validates each row first; throws on schema failure.++ `ps` must be a `PreparedStatement` for [[fields-insert-sql]]; the caller+ owns the connection + statement lifecycle."+ [^PreparedStatement ps batch]+ (doseq [[ln line] batch]+ (validate-line! ::schemas/field-info ln line {:source-id (:id line)}))+ (when (seq batch)+ (doseq [[_ {:keys [id table_id parent_id fk_target_field_id+ name base_type database_type+ effective_type semantic_type+ coercion_strategy description nfc_path]}] batch]+ (.setInt ps 1 (int id))+ (.setInt ps 2 (int table_id))+ (set-int-or-null! ps 3 parent_id)+ (set-int-or-null! ps 4 fk_target_field_id)+ (.setString ps 5 (str name))+ (.setString ps 6 (str base_type))+ (.setString ps 7 (str database_type))+ (set-string-or-null! ps 8 effective_type)+ (set-string-or-null! ps 9 semantic_type)+ (set-string-or-null! ps 10 coercion_strategy)+ (set-string-or-null! ps 11 description)+ (set-string-or-null! ps 12 (encode-path-or-nil nfc_path))+ (.addBatch ps))+ (.executeBatch ps)))++;;; ============================== PG-specific COPY drain ==============================+;;;+;;; Drives staging-table inserts through PostgreSQL's COPY wire protocol via+;;; `CopyManager` / `CopyIn`. Streams TSV-formatted row bytes directly to the+;;; server — no per-statement parsing, no JDBC parameter binding overhead.+;;; Approaches PG's raw bulk-load throughput; in practice ~5× faster than the+;;; JDBC `executeBatch` path at multi-million-row scale.++(def tables-copy-sql+ "PostgreSQL COPY statement for `metabase_table_import`. Column order matches+ the TSV emitted by [[table-tsv-line]]."+ (str "COPY metabase_table_import"+ " (source_id, source_db_id, db_name, schema, name, description, display_name)"+ " FROM STDIN"))++(def fields-copy-sql+ "PostgreSQL COPY statement for `metabase_field_import`. Column order matches+ the TSV emitted by [[field-tsv-line]]."+ (str "COPY metabase_field_import"+ " (source_id, source_table_id, source_parent_id, source_fk_target_id,"+ " name, base_type, database_type, effective_type, semantic_type,"+ " coercion_strategy, description, nfc_path)"+ " FROM STDIN"))++(defn- tsv-escape ^String [^String s]+ (-> s+ (.replace "\\" "\\\\")+ (.replace "\t" "\\t")+ (.replace "\n" "\\n")+ (.replace "\r" "\\r")))++(defn- tsv-cell ^String [v]+ (cond+ (nil? v) "\\N"+ (number? v) (str v)+ (boolean? v) (str v)+ :else (tsv-escape (str v))))++(defn- table-tsv-line ^String [row databases-by-source-id]+ (let [{:keys [id db_id schema name description]} row]+ (str (tsv-cell id) "\t"+ (tsv-cell db_id) "\t"+ (tsv-cell (get databases-by-source-id db_id)) "\t"+ (tsv-cell schema) "\t"+ (tsv-cell name) "\t"+ (tsv-cell description) "\t"+ (tsv-cell (humanization/name->human-readable-name name)) "\n")))++(defn- field-tsv-line ^String [row]+ (let [{:keys [id table_id parent_id fk_target_field_id+ name base_type database_type+ effective_type semantic_type+ coercion_strategy description nfc_path]} row]+ (str (tsv-cell id) "\t"+ (tsv-cell table_id) "\t"+ (tsv-cell parent_id) "\t"+ (tsv-cell fk_target_field_id) "\t"+ (tsv-cell name) "\t"+ (tsv-cell base_type) "\t"+ (tsv-cell database_type) "\t"+ (tsv-cell effective_type) "\t"+ (tsv-cell semantic_type) "\t"+ (tsv-cell coercion_strategy) "\t"+ (tsv-cell description) "\t"+ (tsv-cell (encode-path-or-nil nfc_path)) "\n")))++(defn drain-tables-batch-pg-copy!+ "Per-batch handler for `:tables` that streams TSV rows over PG's COPY wire+ protocol via the supplied `CopyIn` handle. Validates each row first."+ [^CopyIn copy-in databases-by-source-id batch]+ (doseq [[ln line] batch]+ (validate-line! ::schemas/table-info ln line {:source-id (:id line)}))+ (doseq [[_ row] batch]+ (let [^bytes bs (.getBytes (table-tsv-line row databases-by-source-id) StandardCharsets/UTF_8)]+ (.writeToCopy copy-in bs 0 (alength bs)))))++(defn drain-fields-batch-pg-copy!+ "Per-batch handler for `:fields` that streams TSV rows over PG's COPY wire+ protocol via the supplied `CopyIn` handle. Validates each row first."+ [^CopyIn copy-in batch]+ (doseq [[ln line] batch]+ (validate-line! ::schemas/field-info ln line {:source-id (:id line)}))+ (doseq [[_ row] batch]+ (let [^bytes bs (.getBytes (field-tsv-line row) StandardCharsets/UTF_8)]+ (.writeToCopy copy-in bs 0 (alength bs)))))++(defn resolve-target-table-ids-in-staging!+ "Set `metabase_table_import.target_id` to the int id of the matching+ `metabase_table` row for every staging row whose match key resolves. The+ match key is `(db_name, schema, name)` against `(d.name, t.schema, t.name)`,+ restricted to active, non-defective live rows.++ Rows that do not match keep `target_id` NULL. Inactive matches are+ deliberately not resolved so a re-import after a deactivation creates a+ fresh active row.++ Idempotent — running again with the same staging contents produces the+ same assignments."+ []+ (t2/query+ {:update :metabase_table_import+ :set {:target_id+ ;; MIN(t.id): `metabase_database`'s unique constraint is+ ;; `(router_database_id, name)`, so two NULL-router databases+ ;; can share a name → two `(db_name, schema, name)` matches+ ;; (GHY-3549).+ {:select [[[:min :t.id]]]+ :from [[:metabase_table :t]]+ :join [[:metabase_database :d] [:= :d.id :t.db_id]]+ :where [:and+ [:= :metabase_table_import.db_name :d.name]+ [:= [:coalesce :t.schema [:inline ""]]+ [:coalesce :metabase_table_import.schema [:inline ""]]]+ [:= :t.name :metabase_table_import.name]+ [:= :t.is_defective_duplicate [:inline false]]+ [:= :t.active [:inline true]]]}}}))++;; UPDATE before INSERT: on a clean-schema import the UPDATE matches nothing+;; (fast no-op) and the INSERT then writes each row exactly once.+;;+;; Callers re-run [[resolve-target-table-ids-in-staging!]] and pass the set to+;; [[new-target-tables-from-staging]] to read back the freshly-created rows.+(defn merge-tables!+ "Merge `metabase_table_import` into `metabase_table` atomically: UPDATE the+ staging rows that matched a live table, INSERT the rest. Returns the set of+ staging `source_id`s that were INSERTed (snapshot taken before the INSERT).++ Pre-condition: [[resolve-target-table-ids-in-staging!]] must have run, so+ matched staging rows carry a `target_id`."+ []+ (t2/with-transaction [_]+ ;; t2 joins an outer txn rather than creating a savepoint, so wrapping+ ;; this call in a larger transaction is safe.++ ;; Capture which staging rows are about to be INSERTed (target_id NULL).+ ;; After the INSERT + a subsequent resolve round, these source_ids will+ ;; have their `target_id` populated with the freshly-inserted live id.+ (let [insert-source-ids+ (into #{} (map :source_id)+ (t2/query {:select [:source_id]+ :from [:metabase_table_import]+ :where [:= :target_id nil]}))]+ ;; UPDATE matched rows first (clobber description, bump updated_at).+ ;; Skip-if-unchanged: the EXISTS subquery additionally requires that+ ;; description actually differs (NULL-safe via COALESCE-with-empty-+ ;; string), so a re-import with identical description is a true no-op+ ;; — no UPDATE fires, no dead tuple. updated_at is deliberately NOT+ ;; in the predicate (otherwise the UPDATE would always fire,+ ;; defeating the optimization).+ (t2/query+ {:update :metabase_table+ :set {:description {;; ORDER BY + LIMIT 1: two staging rows can+ ;; resolve to the same `target_id` (GHY-3549).+ :select [:it.description]+ :from [[:metabase_table_import :it]]+ :where [:= :it.target_id :metabase_table.id]+ :order-by [[:it.source_id :asc]]+ :limit 1}+ :updated_at :%now}+ :where [:and+ [:= :metabase_table.is_defective_duplicate [:inline false]]+ [:= :metabase_table.active [:inline true]]+ [:exists {:select [[[:inline 1]]]+ :from [[:metabase_table_import :it]]+ :where [:and+ [:= :it.target_id :metabase_table.id]+ [:!= [:coalesce :metabase_table.description [:inline ""]]+ [:coalesce :it.description [:inline ""]]]]}]]})+ ;; INSERT bypasses :model/Table's :after-insert hook — that hook+ ;; schedules per-DB Quartz triggers we don't want and fires+ ;; `set-new-table-permissions!` once per row. The JOIN on db_name+ ;; silently drops staging rows whose source DB has no target appdb.+ (t2/query+ {:insert-into+ [[:metabase_table [:db_id :schema :name :description :display_name :data_layer+ :active :show_in_getting_started :is_defective_duplicate+ :created_at :updated_at]]+ {:select [:d.id :it.schema :it.name :it.description :it.display_name+ [[:inline "internal"]]+ [[:inline true]] [[:inline false]] [[:inline false]]+ :%now :%now]+ :from [[:metabase_table_import :it]]+ :join [[:metabase_database :d] [:= :d.name :it.db_name]]+ :where [:= :it.target_id nil]}]})+ insert-source-ids)))++(defn new-target-tables-from-staging+ "Look up the `metabase_table` rows that were just INSERTed by+ [[merge-tables!]]. `insert-source-ids` is the set returned by+ [[merge-tables!]]. Pre-condition:+ [[resolve-target-table-ids-in-staging!]] has run since the INSERT, so+ staging's `target_id` is populated for these rows."+ [insert-source-ids]+ (when (seq insert-source-ids)+ (t2/query {:select [:t.id :t.db_id :t.schema]+ :from [[:metabase_table :t]]+ :join [[:metabase_table_import :it] [:= :it.target_id :t.id]]+ :where [:in :it.source_id (vec insert-source-ids)]})))++;;; ============================== Pre-flight orphan check ==============================++(def ^:private orphan-sample-cap+ "How many orphan rows to surface in the error data when bailing out. Visibility+ without exploding the exception payload."+ 10)++(defn- orphan-not-exists-predicate+ "WHERE-clause fragment matching `metabase_field_import` rows whose `column-key` (a `:source_*_id` column)+ is non-null but references no other staging row's `source_id` — i.e. an orphan reference."+ [column-key]+ ;; Correlated NOT EXISTS, not NOT IN: at 9M+ rows PG can't plan+ ;; `NOT IN (subquery)` (NULL-semantics force full-materialization of the+ ;; inner set) and it effectively never terminates. NOT EXISTS resolves to a+ ;; proper anti-join via the PK index on `source_id` — O(N log N), not O(N²).+ ;;+ ;; The inner FROM is a `(SELECT source_id FROM ...)` derived table so the+ ;; predicate stays valid inside an UPDATE on `metabase_field_import`: MySQL+ ;; forbids referencing the UPDATE's target table directly in a subquery; the+ ;; wrap defeats that, and PG/H2 inline it away. Aliased `:s` so the inner+ ;; WHERE's unqualified column-key resolves to the outer table.+ (let [outer-col (keyword (str "metabase_field_import." (name column-key)))]+ [:and+ [:not= column-key nil]+ [:not [:exists+ {:select [[[:inline 1]]]+ :from [[{:select [:source_id]+ :from [:metabase_field_import]} :s]]+ :where [:= :s.source_id outer-col]}]]]))++(defn- orphan-count+ "Number of `metabase_field_import` rows whose `column-key` (a `:source_*_id`+ column) is non-null but doesn't reference any other row's `source_id`."+ [column-key]+ (t2/count :metabase_field_import {:where (orphan-not-exists-predicate column-key)}))++(defn- orphan-sample+ "Up to `orphan-sample-cap` `[source_id, <column-key>]` pairs for orphan rows."+ [column-key]+ (t2/query+ {:select [:source_id column-key]+ :from [:metabase_field_import]+ :where (orphan-not-exists-predicate column-key)+ :limit orphan-sample-cap}))++(defn assert-no-orphan-refs!+ "Pre-flight check after drain: every staging field row's `source_parent_id`+ must reference another staging row's `source_id`. Throws `ex-info` with+ `:kind :file-incomplete` and a sample of orphan rows in the error data+ when any orphan parent ref exists.++ Parent refs are structural — a field claiming to be a child of a missing+ field can't be positioned, so the file is genuinely incomplete. Orphan+ `source_fk_target_id` refs are not fatal here; see+ [[null-orphan-fk-target-refs!]]."+ []+ (let [parent-count (orphan-count :source_parent_id)]+ (when (pos? parent-count)+ (throw (ex-info (format "metadata-file-import: file is incomplete — %d orphan parent ref(s)"+ parent-count)+ {:kind :file-incomplete+ :orphan-parent-count parent-count+ :orphan-parent-sample (orphan-sample :source_parent_id)})))))++(defn null-orphan-fk-target-refs!+ "Find staging rows whose `source_fk_target_id` points at a `source_id`+ not present in staging, and NULL the `source_fk_target_id` column on+ those rows. Returns `{:count N :sample [...]}` when at least one row was+ scrubbed, `{:count 0}` otherwise."+ []+ ;; Why NULL instead of abort: fk_target refs that cross a hidden/archived+ ;; table boundary on the source side (the table is `active=false` or+ ;; `visibility_type='hidden'`) survive in `metabase_field` but get filtered+ ;; out by the export's table-visibility join. The exported file then contains+ ;; fk_target refs whose targets aren't emitted — a real-data shape we observe+ ;; in production appdbs. Treating these as fatal would block legitimate+ ;; imports; treating them as no-fk is the lossless choice for the importer.+ (let [n (orphan-count :source_fk_target_id)]+ (if (zero? n)+ {:count 0}+ (let [sample (orphan-sample :source_fk_target_id)]+ (t2/query+ {:update :metabase_field_import+ :set {:source_fk_target_id nil}+ :where (orphan-not-exists-predicate :source_fk_target_id)})+ {:count n :sample sample}))))++;;; ============================== Depth tagging ==============================++(def depth-iteration-cap+ "Iteration cap on the depth-tagging fixpoint loop; exceeding it throws+ `:cycle-in-field-graph` with the un-tagged sample."+ ;; Field parent chains are shallow in practice (a few levels at most), so 50+ ;; is a comfortable safety belt — exceeding it means the algorithm is the+ ;; bug, not the data.+ 50)++(defn- mark-roots-at-depth-zero!+ "Tag every staging row with no parent and no fk_target ref as `depth = 0`.+ Bootstraps the iteration."+ []+ (t2/query+ {:update :metabase_field_import+ :set {:depth 0}+ :where [:and+ [:= :source_parent_id nil]+ [:= :source_fk_target_id nil]]}))++(defn- mark-rows-at-depth!+ "Tag every still-untagged staging row whose `source_parent_id` and+ `source_fk_target_id` (when non-NULL) reference rows that already have+ `depth < d`."+ [d]+ ;; MySQL forbids referencing the UPDATE's target table directly inside a+ ;; subquery; wrap in a derived table there. PG/H2 accept the direct form.+ (let [staging-source (case (mdb/db-type)+ :mysql {:select [:source_id :depth]+ :from [:metabase_field_import]}+ :metabase_field_import)]+ (t2/query+ {:update :metabase_field_import+ :set {:depth d}+ :where [:and+ [:= :metabase_field_import.depth nil]+ [:or+ [:= :metabase_field_import.source_parent_id nil]+ [:exists {:select [[[:inline 1]]]+ :from [[staging-source :p]]+ :where [:and+ [:= :p.source_id :metabase_field_import.source_parent_id]+ [:not= :p.depth nil]+ [:< :p.depth d]]}]]+ [:or+ [:= :metabase_field_import.source_fk_target_id nil]+ [:exists {:select [[[:inline 1]]]+ :from [[staging-source :f]]+ :where [:and+ [:= :f.source_id :metabase_field_import.source_fk_target_id]+ [:not= :f.depth nil]+ [:< :f.depth d]]}]]]})))++(defn- untagged-staging-row-count+ "Number of `metabase_field_import` rows still at `depth IS NULL`. Used as+ the convergence signal in `compute-staging-depth!`."+ []+ (t2/count :metabase_field_import :depth nil))++(defn- untagged-staging-row-sample+ "Up to `orphan-sample-cap` un-tagged rows for cycle-error diagnostics."+ []+ (t2/query+ {:select [:source_id :source_parent_id :source_fk_target_id]+ :from [:metabase_field_import]+ :where [:= :depth nil]+ :limit orphan-sample-cap}))++;; The fixpoint loop terminates on convergence (no untagged rows) or a+;; no-progress round. The `assert-no-orphan-refs!` pre-condition is what lets+;; us read no-progress as a cycle: with orphans already ruled out, a row that+;; can never be tagged must be part of one.+(defn compute-staging-depth!+ "Tag every `metabase_field_import` row with a non-NULL `depth`: 0 for roots+ (no parent, no fk_target ref), `d` for rows whose deps are all at depth+ < `d`. Returns the maximum depth in staging.++ Pre-condition: [[assert-no-orphan-refs!]] must have run successfully.++ Throws `:cycle-in-field-graph` if rows can't be tagged (a reference cycle)+ or `:depth-tagging-cap-exceeded` if the loop runs past [[depth-iteration-cap]]."+ []+ (mark-roots-at-depth-zero!)+ (loop [d 1]+ (let [before (untagged-staging-row-count)]+ (cond+ (zero? before)+ nil++ (>= d depth-iteration-cap)+ (throw (ex-info (format "metadata-file-import: depth-tagging exceeded cap of %d iterations" depth-iteration-cap)+ {:kind :depth-tagging-cap-exceeded+ :iterations d+ :remaining-rows-count before+ :remaining-rows-sample (untagged-staging-row-sample)}))++ :else+ (do+ (mark-rows-at-depth! d)+ (let [after (untagged-staging-row-count)]+ (if (= before after)+ (throw (ex-info (format "metadata-file-import: %d staging row(s) could not be tagged with depth — cycle in file's parent/fk_target reference graph"+ after)+ {:kind :cycle-in-field-graph+ :remaining-rows-count after+ :iterations d+ :remaining-rows-sample (untagged-staging-row-sample)}))+ (recur (inc d))))))))+ (or (:max (first (t2/query {:select [[[:max :depth] :max]]+ :from [:metabase_field_import]})))+ 0))++;;; ============================== Per-depth field merge ==============================++(def ^:private field-clobber-cols+ "The field payload columns the importer owns: on a matched row they are+ overwritten from the file's values (the import is an alternate sync)."+ ;; Each is also its staging column name, so [[update-matched-fields-at-depth!]]+ ;; can build the SET from identically-named correlated subqueries.+ [:base_type :database_type :description+ :effective_type :semantic_type :coercion_strategy :nfc_path])++(defn- target-id-clobber-subquery+ "Correlated subquery yielding `staging-col`'s value from the staging row+ that resolved to this `metabase_field`."+ [staging-col]+ {:select [(keyword (str "fi." (name staging-col)))]+ :from [[:metabase_field_import :fi]]+ :where [:= :fi.target_id :metabase_field.id]+ ;; ORDER BY + LIMIT 1: two staging rows can resolve to the same target_id+ ;; (GHY-3549); pick one deterministically rather than erroring.+ :order-by [[:fi.source_id :asc]]+ :limit 1})++(def ^:private field-payload-changed-predicate+ "True when a matched `metabase_field` row's observable payload differs from+ its staging row — the predicate behind the merge UPDATE's skip-if-unchanged+ behavior."+ ;; Coalesce-with-sentinel is the portable NULL-safe `IS DISTINCT FROM`:+ ;; empty string for text columns, -1 (never a real id) for FK ids.+ [[:!= [:coalesce :metabase_field.base_type [:inline ""]] [:coalesce :fi.base_type [:inline ""]]]+ [:!= [:coalesce :metabase_field.database_type [:inline ""]] [:coalesce :fi.database_type [:inline ""]]]+ [:!= [:coalesce :metabase_field.description [:inline ""]] [:coalesce :fi.description [:inline ""]]]+ ;; `effective_type` coalesces against `base_type`: the export omits it when+ ;; the two are equal, and the app reads NULL effective_type as base_type.+ [:!= [:coalesce :metabase_field.effective_type :metabase_field.base_type]+ [:coalesce :fi.effective_type :fi.base_type]]+ [:!= [:coalesce :metabase_field.semantic_type [:inline ""]] [:coalesce :fi.semantic_type [:inline ""]]]+ [:!= [:coalesce :metabase_field.coercion_strategy [:inline ""]] [:coalesce :fi.coercion_strategy [:inline ""]]]+ [:!= [:coalesce :metabase_field.nfc_path [:inline ""]] [:coalesce :fi.nfc_path [:inline ""]]]+ ;; staging's FK column is `target_fk_target_id`, not `fk_target_field_id`.+ [:!= [:coalesce :metabase_field.fk_target_field_id [:inline -1]] [:coalesce :fi.target_fk_target_id [:inline -1]]]+ ;; `active` compares against TRUE — the merge SET always sets it TRUE.+ [:!= :metabase_field.active [:inline true]]])++(defn resolve-target-table-ids-for-fields-in-staging!+ "Populate `metabase_field_import.target_table_id` by joining through+ `metabase_table_import.source_id → metabase_table_import.target_id`.++ Pre-condition: `metabase_table_import.target_id` must be populated+ (via [[resolve-target-table-ids-in-staging!]]) for both matched and+ newly-inserted target rows.++ Field rows whose source table has no target (orphan-source-database+ case) keep `target_table_id NULL`."+ []+ (t2/query+ {:update :metabase_field_import+ :set {:target_table_id+ {:select [:ti.target_id]+ :from [[:metabase_table_import :ti]]+ :where [:= :ti.source_id :metabase_field_import.source_table_id]}}}))++(defn fill-target-parent-ids-at-depth!+ "Populate `target_parent_id` for depth-`d` staging rows by self-joining+ staging on `source_parent_id → source_id` and reading the parent's+ `target_id`. Pre-condition: all prior-depth staging rows have `target_id`+ resolved (the depth walk processes lower depths first, so this is+ guaranteed when called in order)."+ [d]+ ;; Same MySQL same-table-in-UPDATE workaround as [[mark-rows-at-depth!]].+ (let [parent-source (case (mdb/db-type)+ :mysql {:select [:source_id :target_id]+ :from [:metabase_field_import]}+ :metabase_field_import)]+ (t2/query+ {:update :metabase_field_import+ :set {:target_parent_id+ {:select [:p.target_id]+ :from [[parent-source :p]]+ :where [:= :p.source_id :metabase_field_import.source_parent_id]}}+ :where [:and+ [:= :metabase_field_import.depth d]+ [:not= :metabase_field_import.source_parent_id nil]]})))++(defn fill-target-fk-target-ids-at-depth!+ "Populate `target_fk_target_id` for depth-`d` staging rows by self-joining+ staging on `source_fk_target_id → source_id`."+ [d]+ ;; Same MySQL same-table-in-UPDATE workaround as [[mark-rows-at-depth!]].+ (let [fk-source (case (mdb/db-type)+ :mysql {:select [:source_id :target_id]+ :from [:metabase_field_import]}+ :metabase_field_import)]+ (t2/query+ {:update :metabase_field_import+ :set {:target_fk_target_id+ {:select [:f.target_id]+ :from [[fk-source :f]]+ :where [:= :f.source_id :metabase_field_import.source_fk_target_id]}}+ :where [:and+ [:= :metabase_field_import.depth d]+ [:not= :metabase_field_import.source_fk_target_id nil]]})))++(defn resolve-target-field-ids-at-depth!+ "Set `target_id` for depth-`d` staging rows by natural-key match against `metabase_field`. Matches+ by `(target_table_id, name, target_parent_id)` with NULL-safe parent comparison (a staging row with+ no parent matches only against `metabase_field.parent_id IS NULL`). Skips defective duplicates on+ the target side.++ Run twice per depth — once before the UPDATE/INSERT step (to identify matches for the clobber path)+ and once after (to capture INSERT ids so higher-depth rows can find them as parents/FK targets)."+ [d]+ (t2/query+ {:update :metabase_field_import+ :set {:target_id+ ;; MIN(f.id) tiebreaker — `metabase_field` has no uniqueness+ ;; constraint on `(table_id, name, parent_id)` (GHY-3549).+ {:select [[[:min :f.id]]]+ :from [[:metabase_field :f]]+ :where [:and+ [:= :f.table_id :metabase_field_import.target_table_id]+ [:= :f.name :metabase_field_import.name]+ [:or+ [:and [:= :metabase_field_import.target_parent_id nil]+ [:= :f.parent_id nil]]+ [:= :f.parent_id :metabase_field_import.target_parent_id]]+ [:= :f.is_defective_duplicate [:inline false]]]}}+ :where [:and+ [:= :metabase_field_import.depth d]+ [:= :metabase_field_import.target_id nil]]}))++(defn update-matched-fields-at-depth!+ "Bring the live `metabase_field` rows that depth-`d` staging matched into line with the file. Rows+ whose payload already matches staging are left untouched."+ [d]+ (t2/query+ {:update :metabase_field+ ;; `fk_target_field_id` reads staging's `target_fk_target_id` — it isn't+ ;; part of the natural-key match, so a matched field's FK can drift.+ :set (-> (into {} (map (fn [c] [c (target-id-clobber-subquery c)])) field-clobber-cols)+ (assoc :fk_target_field_id (target-id-clobber-subquery :target_fk_target_id)+ :active [:inline true]+ :updated_at :%now))+ :where [:and+ [:= :metabase_field.is_defective_duplicate [:inline false]]+ ;; Scope the outer walk to the ~staging-row-count IDs at depth `d`.+ ;; Without this, PG starts from `metabase_field` and probes staging+ ;; per row — a full scan on an appdb with millions of field rows.+ [:in :metabase_field.id {:select [:target_id]+ :from [:metabase_field_import]+ :where [:and+ [:= :depth d]+ [:not= :target_id nil]]}]+ ;; Skip-if-unchanged: only touch rows whose payload actually differs.+ [:exists {:select [[[:inline 1]]]+ :from [[:metabase_field_import :fi]]+ :where [:and+ [:= :fi.target_id :metabase_field.id]+ [:= :fi.depth d]+ (into [:or] field-payload-changed-predicate)]}]]}))++(defn insert-new-fields-at-depth!+ "INSERT `metabase_field` for depth-`d` staging rows that didn't match a+ live row (`target_id IS NULL`). The new row's `parent_id` and+ `fk_target_field_id` come from staging's `target_parent_id` /+ `target_fk_target_id` — populated earlier in this depth's iteration.++ Rows whose `target_table_id IS NULL` (source table didn't match target)+ are silently dropped. Rows whose `source_parent_id IS NOT NULL` but+ `target_parent_id IS NULL` are also dropped — that's the case where the+ parent field's table didn't match target, cascading the orphan downward."+ [d]+ (t2/query+ {:insert-into+ [[:metabase_field [:table_id :name :base_type :database_type :description+ :effective_type :semantic_type :coercion_strategy+ :nfc_path :parent_id :fk_target_field_id+ :is_defective_duplicate :active :created_at :updated_at]]+ {:select [:fi.target_table_id :fi.name :fi.base_type :fi.database_type :fi.description+ :fi.effective_type :fi.semantic_type :fi.coercion_strategy+ :fi.nfc_path :fi.target_parent_id :fi.target_fk_target_id+ [[:inline false]] [[:inline true]] :%now :%now]+ :from [[:metabase_field_import :fi]]+ :where [:and+ [:= :fi.depth d]+ [:= :fi.target_id nil]+ [:not= :fi.target_table_id nil]+ [:or+ [:= :fi.source_parent_id nil]+ [:not= :fi.target_parent_id nil]]]}]}))++(defn merge-fields-by-depth!+ "Merge `metabase_field_import` into `metabase_field`, walking depth 0 to+ max-depth so a row's parents and FK targets are resolved before the row+ itself. All depths run inside one transaction.++ Pre-conditions: [[compute-staging-depth!]] has populated `depth` and+ [[resolve-target-table-ids-for-fields-in-staging!]] has populated+ `target_table_id`."+ []+ (let [max-d (or (:max (first (t2/query {:select [[[:max :depth] :max]]+ :from [:metabase_field_import]})))+ 0)]+ ;; One transaction across all depths — t2 joins the import's outer txn.+ (t2/with-transaction [_]+ (doseq [d (range (inc max-d))]+ (fill-target-parent-ids-at-depth! d)+ (fill-target-fk-target-ids-at-depth! d)+ (resolve-target-field-ids-at-depth! d)+ (update-matched-fields-at-depth! d)+ (insert-new-fields-at-depth! d)+ ;; Re-resolve to capture INSERT ids, so higher depths can find these+ ;; rows as parents / FK targets.+ (resolve-target-field-ids-at-depth! d)))))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/schemas.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/schemas.cljnew file mode 100644index 00000000000..6b5ab8968ae--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/schemas.clj@@ -0,0 +1,64 @@+(ns metabase-enterprise.serialization.metadata-file-import.schemas+ "Malli schemas describing the wire-format rows streamed by+ `GET /api/ee/serialization/metadata/export` and consumed by+ `POST /api/ee/serialization/metadata/import`.++ IDs are **integers from the source appdb**. They are internally consistent+ within one file (so `:fields[].parent_id` references a `:fields[].id` in the+ same file) but mean nothing across files.++ Cross-row references:+ - `:tables[].db_id` → `:databases[].id`+ - `:fields[].table_id` → `:tables[].id`+ - `:fields[].parent_id` → another `:fields[].id` in the same file+ - `:fields[].fk_target_field_id` → another `:fields[].id` in the same file++ Optional columns (everything outside the always-present set per row type) are+ *omitted from the row* by the export's `remove-nils` step rather than emitted+ as `null`. Schemas mark these as `{:optional true} [:maybe T]` — absent and+ `null` are accepted equivalently.++ `:effective_type` is special-cased: emitted only when `≠ :base_type`. Importer+ treats absent as `≡ :base_type`."+ (:require+ [metabase.util.malli.registry :as mr]))++(set! *warn-on-reflection* true)++(mr/def ::database-info+ "Database row. All three keys are required."+ [:map+ [:id :int]+ [:name :string]+ [:engine :string]])++(mr/def ::table-info+ "Table row. `:id`, `:db_id`, and `:name` always present. `:schema` and+ `:description` are optional and may be nil."+ [:map+ [:id :int]+ [:db_id :int]+ [:name :string]+ [:schema {:optional true} [:maybe :string]]+ [:description {:optional true} [:maybe :string]]])++(mr/def ::field-info+ "Field row. Five required keys + several optional, per the export's+ `metadata-query-format :model/Field`."+ [:map+ [:id :int]+ [:table_id :int]+ [:name :string]+ [:base_type :string]+ [:database_type :string]+ [:parent_id {:optional true} [:maybe :int]]+ [:fk_target_field_id {:optional true} [:maybe :int]]+ [:description {:optional true} [:maybe :string]]+ [:effective_type {:optional true} [:maybe :string]]+ [:semantic_type {:optional true} [:maybe :string]]+ [:coercion_strategy {:optional true} [:maybe :string]]+ [:nfc_path {:optional true}+ [:maybe+ [:fn {:error/message "must be a sequence of strings"}+ (fn [x] (and (instance? java.util.List x)+ (every? string? x)))]]]])diff --git a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj b/enterprise/backend/src/metabase_enterprise/serialization/schema.cljindex d05dfa1c6ec..815adb38d2a 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/schema.clj@@ -67,3 +67,16 @@[:databases {:optional true} [:sequential ::exported-database]][:tables {:optional true} [:sequential ::exported-table]][:fields {:optional true} [:sequential ::exported-field]]])++(mr/def ::import-status-response+ "Wire shape of `GET /metadata/import/:id`. Timestamps are ISO-8601 strings.+ `:started-at`/`:finished-at`/`:wall-ms` are nil until the relevant lifecycle+ stage is reached; `:error` is nil unless the import failed."+ [:map {:closed true}+ [:id :string]+ [:status [:enum "queued" "running" "ok" "error"]]+ [:enqueued-at :string]+ [:started-at [:maybe :string]]+ [:finished-at [:maybe :string]]+ [:wall-ms [:maybe number?]]+ [:error [:maybe :string]]])diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex ee08740fee9..c28b539add8 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -6,6 +6,7 @@[clojure.walk :as walk][medley.core :as m][metabase-enterprise.serialization.api :as api.serialization]+ [metabase-enterprise.serialization.metadata-file-import :as metadata-file-import][metabase-enterprise.serialization.v2.ingest :as v2.ingest][metabase.analytics.snowplow-test :as snowplow-test][metabase.models.serialization :as serdes]@@ -14,10 +15,11 @@[metabase.search.test-util :as search.tu][metabase.test :as mt][metabase.util.compress :as u.compress]+ [metabase.util.json :as json][metabase.util.random :as u.random][toucan2.core :as t2])(:import- (java.io File)+ (java.io ByteArrayInputStream File)(org.apache.commons.compress.archivers.tar TarArchiveEntry TarArchiveInputStream)(org.apache.commons.compress.compressors.gzip GzipCompressorInputStream)))@@ -668,6 +670,151 @@"Serialization"(mt/user-http-request :crowberto :post 402 "ee/serialization/metadata/export")))))+;;; --------------------------------------- /metadata/import ---------------------------------------++(defn- import-metadata!+ "Helper: JSON-encode `payload` and POST it as an octet-stream body to+ `/api/ee/serialization/metadata/import`."+ ([payload] (import-metadata! :crowberto 202 payload))+ ([user expected-status payload]+ (mt/user-http-request+ user :post expected-status "ee/serialization/metadata/import"+ {:request-options+ {:headers {"content-type" "application/octet-stream"}+ :body (ByteArrayInputStream.+ (.getBytes ^String (json/encode payload) "UTF-8"))}})))++(defn- import-status!+ "Helper: GET `/api/ee/serialization/metadata/import/:id`."+ ([id] (import-status! :crowberto 200 id))+ ([user expected-status id]+ (mt/user-http-request+ user :get expected-status (str "ee/serialization/metadata/import/" id))))++(deftest metadata-import-roundtrip-test+ (testing "POST /api/ee/serialization/metadata/import is the inverse of POST /metadata/export"+ ;; The agent runs the import on a separate connection from the test thread;+ ;; mt/with-temp's default rollback-only wrapper would hide the temp rows+ ;; from that connection. Real commits + auto-sync disabled lets the agent+ ;; see what the test set up.+ ;; Warm test-data before disabling sync: the :model/Field with-temp default resolves+ ;; `(data/id :checkins)`, which needs a synced test-data DB.+ (mt/db)+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"+ :description "round trip"}+ :model/Field _ {:table_id t-id+ :base_type :type/Integer+ :database_type "BIGINT"+ :semantic_type :type/PK}]+ (let [exported (mt/user-http-request :crowberto :post 202+ "ee/serialization/metadata/export"+ :with-databases true+ :with-tables true+ :with-fields true)+ before-tables (t2/count :model/Table :db_id db-id)+ before-fields (t2/count :model/Field :table_id t-id)+ resp (import-metadata! exported)]+ (is (true? (:queued resp)) "POST acknowledges the import was queued")+ (is (string? (:import-id resp)) "POST returns a string import-id")+ ;; The import endpoint hands the file off to an agent and returns+ ;; immediately. Block until the agent has drained it, then assert+ ;; the agent didn't silently swallow an exception (without this+ ;; check, the row-count assertions below would falsely pass on a+ ;; failed re-import since nothing would have changed).+ (await @#'metadata-file-import/import-agent)+ (is (= :ok (:status (metadata-file-import/import-status (:import-id resp))))+ "agent finished the import successfully")+ (testing "no rows are added by re-importing the same payload"+ (is (= before-tables (t2/count :model/Table :db_id db-id)))+ (is (= before-fields (t2/count :model/Field :table_id t-id))))+ (testing "the table description is preserved"+ (is (= "round trip"+ (t2/select-one-fn :description :model/Table t-id)))))))))))++(deftest metadata-import-post-returns-202-and-import-id-test+ (testing "POST /api/ee/serialization/metadata/import returns 202 with :queued true and a string :import-id"+ (mt/with-premium-features #{:serialization}+ (let [resp (import-metadata! :crowberto 202 {:databases [] :tables [] :fields []})]+ (is (true? (:queued resp)))+ (is (string? (:import-id resp)) ":import-id is a string")+ (is (uuid? (java.util.UUID/fromString (:import-id resp)))+ ":import-id parses as a UUID")+ (await @#'metadata-file-import/import-agent)))))++(deftest metadata-import-status-endpoint-lifecycle-test+ (testing "GET /metadata/import/:id reports the import status across its lifecycle"+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{:serialization}+ (let [resp (import-metadata! :crowberto 202 {:databases [] :tables [] :fields []})+ id (:import-id resp)]+ ;; settle the agent, then the GET endpoint should report a terminal status+ (await @#'metadata-file-import/import-agent)+ (let [status (import-status! :crowberto 200 id)]+ (is (= id (:id status)) "the status response echoes the id")+ (is (string? (:status status)) ":status is serialized as a string")+ (is (contains? #{"ok" "error"} (:status status))+ "after the agent settles the status is terminal")+ (is (contains? status :enqueued-at))+ (is (contains? status :started-at))+ (is (contains? status :finished-at))+ (is (contains? status :wall-ms)))))))))++(deftest metadata-import-status-omits-file-path-test+ (testing "GET /metadata/import/:id never leaks the server-side temp file path"+ (mt/with-premium-features #{:serialization}+ (let [resp (import-metadata! :crowberto 202 {:databases [] :tables [] :fields []})+ id (:import-id resp)]+ (await @#'metadata-file-import/import-agent)+ (let [status (import-status! :crowberto 200 id)]+ (is (not (contains? status :file))+ "the wire shape must not include :file")+ (is (= #{:id :status :enqueued-at :started-at :finished-at :wall-ms :error}+ (set (keys status)))+ "the wire shape is exactly the documented key set"))))))++(deftest metadata-import-status-unknown-id-404-test+ (testing "GET /metadata/import/:id returns 404 for an unknown or evicted id"+ (mt/with-premium-features #{:serialization}+ (import-status! :crowberto 404 (str (java.util.UUID/randomUUID))))))++(deftest metadata-import-status-superuser-test+ (testing "GET /metadata/import/:id — non-admins get a 403"+ (mt/with-premium-features #{:serialization}+ (is (= "You don't have permissions to do that."+ (import-status! :rasta 403 (str (java.util.UUID/randomUUID))))))))++(deftest metadata-import-status-token-feature-test+ (testing "GET /metadata/import/:id requires the :serialization premium feature"+ (mt/with-premium-features #{}+ (mt/assert-has-premium-feature-error+ "Serialization"+ (import-status! :crowberto 402 (str (java.util.UUID/randomUUID)))))))++(deftest metadata-import-wrong-content-type-test+ (testing "POST /api/ee/serialization/metadata/import — JSON content-type is rejected with 415"+ (mt/with-premium-features #{:serialization}+ (mt/user-http-request :crowberto :post 415+ "ee/serialization/metadata/import"+ {:tables [] :fields []}))))++(deftest metadata-import-superuser-test+ (testing "POST /api/ee/serialization/metadata/import — non-admins get a 403"+ (mt/with-premium-features #{:serialization}+ (is (= "You don't have permissions to do that."+ (import-metadata! :rasta 403 {:tables [] :fields []}))))))++(deftest metadata-import-token-feature-test+ (testing "POST /api/ee/serialization/metadata/import requires the :serialization premium feature"+ (mt/with-premium-features #{}+ (mt/assert-has-premium-feature-error+ "Serialization"+ (import-metadata! :crowberto 402 {:tables [] :fields []})))))+(deftest serialization-cleanup-test(testing "No temp files are left behind after export/import operations"(let [known-files (set (.list (io/file api.serialization/parent-dir)))]diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/concurrency_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/concurrency_test.cljnew file mode 100644index 00000000000..9a2e81e3f9f--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/concurrency_test.clj@@ -0,0 +1,259 @@+(ns metabase-enterprise.serialization.metadata-file-import.concurrency-test+ "Tests for the in-JVM concurrency guard on metadata-file-import. The guard+ is a single Clojure agent serializing all imports plus a busy-predicate+ registered with `metabase.sync.util` so that warehouse-sync skips with a+ WARN log line whenever an import is in flight."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.metadata-file-import :as mfi]+ [metabase.sync.util :as sync.util]+ [metabase.test :as mt]+ [metabase.util.json :as json])+ (:import+ (java.io File)))++(set! *warn-on-reflection* true)++;; Kondo flags `test-helpers-set-global-values!` inside any fixture as+;; "destructive in parallel context". Here it is *intentionally* the opt-out+;; switch — without it the agent runs on a separate connection that can't see+;; the test's mt/with-temp rollback-only state.+#_{:clj-kondo/ignore [:metabase/validate-deftest]}+(use-fixtures :once+ (fn [thunk]+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.+ (mt/db)+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (mt/test-helpers-set-global-values!+ (thunk)))))++(defn- tiny-meta-file ^File [data]+ (let [f (File/createTempFile "mfi-concurrency-" ".json")]+ (.deleteOnExit f)+ (spit f (json/encode data))+ f))++(defn- tiny-payload [db-name]+ {:databases [{:id 7 :name db-name :engine "postgres"}]+ :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]+ :fields [{:id 1000 :table_id 100 :name "x"+ :base_type "type/Integer" :database_type "integer"}]})++;;; ============================== import-running? ==============================++(deftest import-running-false-when-idle-test+ (testing "import-running? returns false when no import is in flight"+ (await @#'mfi/import-agent)+ (is (false? (mfi/import-running?)))))++(deftest import-running-toggles-around-enqueue-test+ (testing "import-running? is true mid-flight, false after the agent settles"+ (mt/with-temp [:model/Database {} {:name "mfi-running-toggles" :engine :postgres}]+ (let [file (tiny-meta-file (tiny-payload "mfi-running-toggles"))+ observed (atom nil)+ real-import mfi/import-metadata-file!]+ (with-redefs [mfi/import-metadata-file!+ (fn [f] (reset! observed (mfi/import-running?))+ (real-import f))]+ (mfi/enqueue-import! file)+ (await @#'mfi/import-agent))+ (is (true? @observed)+ "import-running? is true while the agent is inside the import fn")+ (is (false? (mfi/import-running?))+ "import-running? is false again once the agent finishes")))))++;;; ============================== Serialization ==============================++(deftest two-enqueues-run-sequentially-test+ (testing "two enqueued imports execute one-at-a-time, in arrival order"+ (mt/with-temp [:model/Database {} {:name "mfi-seq-1" :engine :postgres}+ :model/Database {} {:name "mfi-seq-2" :engine :postgres}]+ (let [f1 (tiny-meta-file (tiny-payload "mfi-seq-1"))+ f2 (tiny-meta-file (tiny-payload "mfi-seq-2"))+ events (atom [])+ real-import mfi/import-metadata-file!]+ (with-redefs [mfi/import-metadata-file!+ (fn [^java.io.File f]+ (swap! events conj [:start (.getName f) (System/nanoTime)])+ (real-import f)+ (swap! events conj [:end (.getName f) (System/nanoTime)]))]+ (mfi/enqueue-import! f1)+ (mfi/enqueue-import! f2)+ (await @#'mfi/import-agent))+ (let [evs @events]+ (is (= 4 (count evs)) "two starts + two ends")+ (is (= :start (first (nth evs 0))))+ (is (= :end (first (nth evs 1))))+ (is (= :start (first (nth evs 2))))+ (is (= :end (first (nth evs 3))))+ (is (= (second (nth evs 0)) (second (nth evs 1)))+ "first import's start and end are for the same file")+ (is (= (second (nth evs 2)) (second (nth evs 3)))+ "second import's start and end are for the same file")+ (is (<= (nth (nth evs 1) 2) (nth (nth evs 2) 2))+ "second import does not start until first has finished"))))))++;;; ============================== Agent does not poison ==============================++(deftest agent-survives-failing-import-test+ (testing "an exception inside the import body does not leave the agent in a failed state"+ (await @#'mfi/import-agent)+ (with-redefs [mfi/import-metadata-file!+ (fn [_f] (throw (ex-info "synthetic" {:kind :test-failure})))]+ (mfi/enqueue-import! (tiny-meta-file (tiny-payload "mfi-fail")))+ (await @#'mfi/import-agent))+ (is (nil? (agent-error @#'mfi/import-agent))+ "agent did not enter failed state")+ (is (false? (mfi/import-running?))+ "state reset to :idle after the failure")))++;;; ============================== Registry ==============================++(deftest enqueue-import-returns-string-id-and-creates-queued-record-test+ (testing "enqueue-import! returns a string UUID id and synchronously inserts a :queued record"+ (await @#'mfi/import-agent)+ ;; Block the agent so we can observe the :queued record before it transitions.+ (let [release (promise)]+ (with-redefs [mfi/import-metadata-file! (fn [_f] @release :ok)]+ (let [id (mfi/enqueue-import! (tiny-meta-file (tiny-payload "mfi-queued")))]+ (is (string? id) "enqueue-import! returns a string id, not :queued")+ (is (uuid? (java.util.UUID/fromString id))+ "the returned id parses as a UUID")+ (let [record (mfi/import-status id)]+ (is (some? record) "a registry record exists for the returned id")+ (is (= id (:id record)))+ (is (contains? #{:queued :running} (:status record))+ "record is :queued (or already :running if the agent picked it up)")+ (is (some? (:enqueued-at record)) ":enqueued-at is set on insert"))+ (deliver release :go)+ (await @#'mfi/import-agent))))))++(deftest record-transitions-queued-running-ok-test+ (testing "the registry record moves :queued -> :running -> :ok through the agent"+ (mt/with-temp [:model/Database {} {:name "mfi-lifecycle-ok" :engine :postgres}]+ (await @#'mfi/import-agent)+ (let [file (tiny-meta-file (tiny-payload "mfi-lifecycle-ok"))+ id (mfi/enqueue-import! file)]+ ;; Capturing the record mid-flight is racy; settle the agent and assert+ ;; the terminal record — the recorded timestamps prove it passed+ ;; through :running.+ (await @#'mfi/import-agent)+ (let [record (mfi/import-status id)]+ (is (some? record))+ (is (= :ok (:status record)) "terminal status is :ok on success")+ (is (some? (:started-at record)) ":started-at set => passed through :running")+ (is (some? (:finished-at record)) ":finished-at set on completion")+ (is (number? (:wall-ms record)) ":wall-ms recorded on completion")+ (is (nil? (:error record)) "no :error on a successful import"))))))++(deftest record-transitions-to-error-on-failing-import-test+ (testing "a failing import yields an :error record and does not poison the agent"+ (await @#'mfi/import-agent)+ (let [id (with-redefs [mfi/import-metadata-file!+ (fn [_f] (throw (ex-info "synthetic failure" {:kind :test-failure})))]+ (let [id (mfi/enqueue-import! (tiny-meta-file (tiny-payload "mfi-lifecycle-err")))]+ (await @#'mfi/import-agent)+ id))]+ (is (nil? (agent-error @#'mfi/import-agent))+ "agent did not enter a failed state")+ (let [record (mfi/import-status id)]+ (is (some? record))+ (is (= :error (:status record)) "terminal status is :error on failure")+ (is (some? (:started-at record)))+ (is (some? (:finished-at record)))+ (is (number? (:wall-ms record)))+ (is (string? (:error record)) ":error holds a stringified exception")))))++(deftest import-status-nil-for-unknown-id-test+ (testing "import-status returns nil for an id that was never enqueued"+ (is (nil? (mfi/import-status (str (java.util.UUID/randomUUID))))+ "unknown id => nil")+ (is (nil? (mfi/import-status "not-even-a-uuid"))+ "garbage id => nil")))++;;; ============================== Eviction ==============================++(defn- terminal-record [i]+ [(str "id-" i) {:id (str "id-" i) :status :ok+ :finished-at (java.time.Instant/ofEpochSecond i)}])++(deftest prune-terminal-records-noop-under-bound-test+ (testing "prune-terminal-records leaves the registry untouched when terminal count is within the bound"+ (let [limit @#'mfi/terminal-record-limit+ reg (into {} (map terminal-record) (range (dec limit)))]+ (is (= reg (#'mfi/prune-terminal-records reg))))))++(deftest prune-terminal-records-drops-oldest-terminal-test+ (testing "prune-terminal-records caps terminal records at the bound, dropping the oldest by :finished-at"+ (let [limit @#'mfi/terminal-record-limit+ extra 5+ reg (into {} (map terminal-record) (range (+ limit extra)))+ pruned (#'mfi/prune-terminal-records reg)]+ (is (= limit (count pruned))+ "terminal records are capped at the bound")+ (is (every? #(contains? pruned (str "id-" %)) (range extra (+ limit extra)))+ "the most-recent `limit` records are retained")+ (is (not-any? #(contains? pruned (str "id-" %)) (range extra))+ "the oldest records are evicted"))))++(deftest prune-terminal-records-never-drops-in-flight-test+ (testing ":queued and :running records are never pruned, regardless of count"+ (let [limit @#'mfi/terminal-record-limit+ in-flight (into {} (for [i (range (* 3 limit))]+ [(str "q-" i) {:id (str "q-" i)+ :status (if (even? i) :queued :running)}]))+ terminal (into {} (map terminal-record) (range (* 2 limit)))+ pruned (#'mfi/prune-terminal-records (merge in-flight terminal))]+ (is (every? #(contains? pruned (str "q-" %)) (range (* 3 limit)))+ "every in-flight record survives")+ (is (= limit (count (filter (comp #{:ok} :status val) pruned)))+ "terminal records are still capped at the bound"))))++;;; ============================== Sync-side skip ==============================++(deftest sync-skips-when-import-is-running-test+ (testing "sync.util/do-sync-operation skips its body when an import is in flight"+ (mt/with-temp [:model/Database db {:name "mfi-sync-skip" :engine :postgres}]+ (await @#'mfi/import-agent)+ ;; Drive a real :running record into the registry by blocking the agent+ ;; inside a redef'd slow import, then exercise the busy-check while it sits.+ (let [release (promise)]+ (with-redefs [mfi/import-metadata-file! (fn [_f] @release :ok)]+ (mfi/enqueue-import! (tiny-meta-file (tiny-payload "mfi-sync-skip")))+ ;; Wait until the agent has actually flipped a record to :running.+ (let [deadline (+ (System/currentTimeMillis) 5000)]+ (while (and (not (mfi/import-running?))+ (< (System/currentTimeMillis) deadline))+ (Thread/sleep 10)))+ (try+ (is (true? (mfi/import-running?))+ "an import is in flight before we exercise the busy-check")+ (let [sync-ran? (atom false)]+ (sync.util/do-sync-operation+ :sync-metadata+ db+ "test-sync"+ (fn [] (reset! sync-ran? true) :result-not-checked))+ (is (false? @sync-ran?)+ "sync body did NOT run because import-busy short-circuited it"))+ (finally+ (deliver release :go)+ (await @#'mfi/import-agent))))))))++(deftest sync-runs-normally-when-no-import-in-flight-test+ (testing "control: sync body executes normally when no import is in flight"+ ;; The namespace fixture sets `disable-auto-sync true`, which itself makes+ ;; `should-sync?` short-circuit before the busy-check runs. Re-enable it+ ;; locally so we're actually exercising the busy-check path.+ (mt/with-temporary-setting-values [disable-auto-sync false]+ (mt/with-temp [:model/Database db {:name "mfi-sync-control" :engine :postgres}]+ (await @#'mfi/import-agent)+ (let [sync-ran? (atom false)]+ (sync.util/do-sync-operation+ :sync-metadata+ db+ "test-sync-control"+ (fn [] (reset! sync-ran? true) :result-not-checked))+ (is (true? @sync-ran?)+ "sync body DID run because no import is busy"))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/depth_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/depth_test.cljnew file mode 100644index 00000000000..805ad21befe--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/depth_test.clj@@ -0,0 +1,205 @@+(ns metabase-enterprise.serialization.metadata-file-import.depth-test+ "Tests for `compute-staging-depth!` — the post-drain step that tags every+ `metabase_field_import` row with a `depth` value, where 0 = root and depth+ d = max(parent depth, fk_target depth) + 1.++ Pre-condition for the tagging step is that orphan refs have already been+ rejected by `assert-no-orphan-refs!`. These tests assume that condition+ holds (every fixture below is consistent)."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.metadata-file-import.processors :as p]+ [toucan2.core :as t2]))++(defn- insert-staging-fields!+ "Insert a sequence of partial field rows into `metabase_field_import`,+ filling in required NOT NULL columns with sensible defaults."+ [rows]+ (let [defaults {:name "x" :base_type "type/Integer" :database_type "int"}]+ (t2/insert! :metabase_field_import+ (mapv (fn [row] (merge defaults {:source_table_id 1} row)) rows))))++(defn- depth-by-source-id+ "Return `{source_id → depth}` after tagging."+ []+ (into {}+ (map (juxt :source_id :depth))+ (t2/query "SELECT source_id, depth FROM metabase_field_import ORDER BY source_id")))++;;; ============================== empty + roots ==============================++(deftest empty-staging-is-noop-test+ (try+ (p/clear-staging-tables!)+ (is (= 0 (p/compute-staging-depth!)))+ (finally (p/clear-staging-tables!))))++(deftest all-roots-get-depth-zero-test+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 1}+ {:source_id 2}+ {:source_id 3}])+ (is (= 0 (p/compute-staging-depth!))+ "max depth assigned is 0 when all rows are roots")+ (is (= {1 0, 2 0, 3 0} (depth-by-source-id)))+ (finally (p/clear-staging-tables!))))++;;; ============================== parent chains ==============================++(deftest one-level-parent-chain-test+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 10}+ {:source_id 11 :source_parent_id 10}])+ (is (= 1 (p/compute-staging-depth!)))+ (is (= {10 0, 11 1} (depth-by-source-id)))+ (finally (p/clear-staging-tables!))))++(deftest three-level-parent-chain-test+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 100}+ {:source_id 101 :source_parent_id 100}+ {:source_id 102 :source_parent_id 101}+ {:source_id 103 :source_parent_id 102}])+ (is (= 3 (p/compute-staging-depth!)))+ (is (= {100 0, 101 1, 102 2, 103 3} (depth-by-source-id)))+ (finally (p/clear-staging-tables!))))++;;; ============================== fk_target chains ==============================++(deftest fk-target-chain-test+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 200}+ {:source_id 201 :source_fk_target_id 200}+ {:source_id 202 :source_fk_target_id 201}])+ (is (= 2 (p/compute-staging-depth!)))+ (is (= {200 0, 201 1, 202 2} (depth-by-source-id)))+ (finally (p/clear-staging-tables!))))++;;; ============================== mixed (max-depth) semantics ==============================++(deftest depth-is-max-of-parent-and-fk-target-depths-test+ (try+ (p/clear-staging-tables!)+ ;; Build:+ ;; A (root) — depth 0+ ;; B → A (parent) — depth 1+ ;; C → B (parent) — depth 2+ ;; D → A (parent, depth 1) AND fk_target → C (depth 2)+ ;; → D should be depth 3, not 2 (max of 1, 2 + 1)+ (insert-staging-fields! [{:source_id 10}+ {:source_id 20 :source_parent_id 10}+ {:source_id 30 :source_parent_id 20}+ {:source_id 40 :source_parent_id 10 :source_fk_target_id 30}])+ (is (= 3 (p/compute-staging-depth!)))+ (is (= {10 0, 20 1, 30 2, 40 3} (depth-by-source-id))+ "row with parent at depth 1 and fk_target at depth 2 lands at depth 3")+ (finally (p/clear-staging-tables!))))++;;; ============================== unfolded leaves ==============================++(deftest unfolded-leaves-are-roots-test+ (try+ (p/clear-staging-tables!)+ ;; Leaves with no source_parent_id (the leaf isn't materialized as a+ ;; separate parent storage row) are roots.+ (insert-staging-fields! [{:source_id 50}+ {:source_id 51}])+ (is (= 0 (p/compute-staging-depth!)))+ (is (= {50 0, 51 0} (depth-by-source-id)))+ (finally (p/clear-staging-tables!))))++;;; ============================== disconnected components ==============================++(deftest multiple-disconnected-chains-tag-in-parallel-test+ (try+ (p/clear-staging-tables!)+ ;; Two unrelated chains.+ (insert-staging-fields! [{:source_id 100} ; root in chain A+ {:source_id 101 :source_parent_id 100} ; depth 1 chain A+ {:source_id 200} ; root in chain B+ {:source_id 201 :source_parent_id 200} ; depth 1 chain B+ {:source_id 202 :source_parent_id 201}]) ; depth 2 chain B+ (is (= 2 (p/compute-staging-depth!)))+ (is (= {100 0, 101 1, 200 0, 201 1, 202 2} (depth-by-source-id)))+ (finally (p/clear-staging-tables!))))++;;; ============================== cycle detection ==============================++(deftest two-row-parent-cycle-throws-test+ (try+ (p/clear-staging-tables!)+ ;; X.parent = Y, Y.parent = X — neither is a root, neither can ever+ ;; be tagged.+ (insert-staging-fields! [{:source_id 1 :source_parent_id 2}+ {:source_id 2 :source_parent_id 1}])+ (let [thrown (try (p/compute-staging-depth!) nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (some? thrown) "should have thrown")+ (is (= :cycle-in-field-graph (:kind (ex-data thrown))))+ (is (= 2 (:remaining-rows-count (ex-data thrown))))+ (is (= 2 (count (:remaining-rows-sample (ex-data thrown))))))+ (finally (p/clear-staging-tables!))))++(deftest self-referential-parent-id-throws-test+ ;; A row pointing at itself can never be tagged: it would need depth d+ ;; where its own depth is < d.+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 5 :source_parent_id 5}])+ (let [thrown (try (p/compute-staging-depth!) nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (some? thrown))+ (is (= :cycle-in-field-graph (:kind (ex-data thrown))))+ (is (= 1 (:remaining-rows-count (ex-data thrown)))))+ (finally (p/clear-staging-tables!))))++(deftest cycle-via-fk-target-throws-test+ ;; X.fk_target = Y, Y.fk_target = X — same problem via the FK relationship+ ;; rather than parent.+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 1 :source_fk_target_id 2}+ {:source_id 2 :source_fk_target_id 1}])+ (let [thrown (try (p/compute-staging-depth!) nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (some? thrown))+ (is (= :cycle-in-field-graph (:kind (ex-data thrown)))))+ (finally (p/clear-staging-tables!))))++(deftest cycle-with-some-tagged-rows-test+ ;; Two roots and a 2-cycle. The roots get depth 0; the cycle never+ ;; resolves. The error data should reflect just the cycle rows.+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 1} ; root+ {:source_id 2} ; root+ {:source_id 100 :source_parent_id 101} ; cycle+ {:source_id 101 :source_parent_id 100}]) ; cycle+ (let [thrown (try (p/compute-staging-depth!) nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (= :cycle-in-field-graph (:kind (ex-data thrown))))+ (is (= 2 (:remaining-rows-count (ex-data thrown)))))+ (finally (p/clear-staging-tables!))))++;;; ============================== idempotence ==============================++(deftest depth-tagging-is-idempotent-test+ ;; Running twice on the same data should produce the same result.+ ;; Useful contract: the merge phase shouldn't have to worry about whether+ ;; depths are stale.+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 10}+ {:source_id 11 :source_parent_id 10}+ {:source_id 12 :source_parent_id 11}])+ (let [first-result (p/compute-staging-depth!)+ first-depths (depth-by-source-id)+ second-result (p/compute-staging-depth!)+ second-depths (depth-by-source-id)]+ (is (= first-result second-result))+ (is (= first-depths second-depths)))+ (finally (p/clear-staging-tables!))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/drain_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/drain_test.cljnew file mode 100644index 00000000000..e3632eb74e4--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/drain_test.clj@@ -0,0 +1,308 @@+(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.drain-test+ "Tests for the drain phase of the metadata file importer. Exercises+ `process-databases!` and the per-batch drain handlers directly against+ hand-built batches — bypassing the parser and orchestrator — and asserts+ the resulting staging-table contents.++ The production drain has two implementations: JDBC `executeBatch` for any+ appdb (the portable path) and PG `CopyManager` for PostgreSQL (the perf+ path). Per-batch tests below use the JDBC variant via the helpers+ [[drain-tables!]] / [[drain-fields!]] (each opens its own connection ++ PreparedStatement). The drain-paths-equivalence test below additionally+ asserts the PG-COPY variant produces the same staging-table state."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.metadata-file-import.processors :as p]+ [metabase.app-db.core :as mdb]+ [metabase.test :as mt]+ [metabase.util.json :as json]+ [toucan2.core :as t2])+ (:import+ (java.sql Connection PreparedStatement)+ (org.postgresql PGConnection)+ (org.postgresql.copy CopyIn)))++(set! *warn-on-reflection* true)++(use-fixtures :once+ (fn [thunk]+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.+ (mt/db)+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (thunk))))++(defn- staging-tables+ "Return the contents of `metabase_table_import` ordered by `source_id`.+ HoneySQL (not raw SQL) so the `schema` column is dialect-quoted correctly+ on MySQL/MariaDB, where `SCHEMA` is a reserved word."+ []+ (t2/query {:select [:source_id :source_db_id :db_name :schema :name+ :description :display_name :target_id]+ :from [:metabase_table_import]+ :order-by [[:source_id :asc]]}))++(defn- staging-fields+ "Return the contents of `metabase_field_import` ordered by `source_id`."+ []+ (t2/query {:select [:source_id :source_table_id :source_parent_id :source_fk_target_id+ :name :base_type :database_type :effective_type :semantic_type+ :coercion_strategy :description :nfc_path+ :depth :target_id :target_table_id :target_parent_id :target_fk_target_id]+ :from [:metabase_field_import]+ :order-by [[:source_id :asc]]}))++(defn- batch+ "Wrap rows in `[line-num row]` tuples as the parser would emit."+ [rows]+ (mapv (fn [i row] [(inc i) row]) (range) rows))++(defn- drain-tables!+ "Open a connection + PreparedStatement and drain `tbl-batch` via the JDBC+ variant. For tests that don't care which production path runs."+ [databases-by-source-id tbl-batch]+ (with-open [^Connection conn (.getConnection (mdb/data-source))+ ^PreparedStatement ps (.prepareStatement conn (p/tables-insert-sql))]+ (p/drain-tables-batch-jdbc! ps databases-by-source-id tbl-batch)))++(defn- drain-fields!+ "Open a connection + PreparedStatement and drain `fld-batch` via the JDBC+ variant. For tests that don't care which production path runs."+ [fld-batch]+ (with-open [^Connection conn (.getConnection (mdb/data-source))+ ^PreparedStatement ps (.prepareStatement conn (p/fields-insert-sql))]+ (p/drain-fields-batch-jdbc! ps fld-batch)))++;;; ============================== process-databases! ==============================++(deftest process-databases!-matches-by-name-and-engine-test+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}]+ (let [results (vec (p/process-databases!+ (batch [{:id 100 :name db-name :engine "h2"}])))]+ (testing "matched result carries source-id (int), name, target-id"+ (is (= [{:source-id 100 :name db-name :target-id db-id :status :matched}]+ results))))))++(deftest process-databases!-no-match-test+ (let [results (vec (p/process-databases!+ (batch [{:id 200 :name "nonexistent-db" :engine "h2"}])))]+ (testing "missing target produces :no-match (non-fatal)"+ (is (= 1 (count results)))+ (is (= :no-match (:status (first results))))+ (is (= 200 (:source-id (first results))))+ (is (= "nonexistent-db" (:name (first results)))))))++;;; ============================== tables drain ==============================++(deftest drain-tables-writes-source-and-target-columns-test+ (try+ (p/clear-staging-tables!)+ (let [databases-by-source-id {7 "warehouse-a" 9 "warehouse-b"}]+ (drain-tables!+ databases-by-source-id+ (batch [{:id 100 :db_id 7 :schema "public" :name "users"+ :description "user table"}+ {:id 101 :db_id 7 :schema nil :name "events"}+ {:id 200 :db_id 9 :schema "analytics" :name "purchases"}]))+ (let [rows (staging-tables)]+ (testing "three rows landed"+ (is (= 3 (count rows))))+ (testing "source ids and natural-key columns populated"+ (is (= [100 7 "warehouse-a" "public" "users" "user table"]+ ((juxt :source_id :source_db_id :db_name :schema :name :description) (nth rows 0))))+ (is (= [101 7 "warehouse-a" nil "events" nil]+ ((juxt :source_id :source_db_id :db_name :schema :name :description) (nth rows 1))))+ (is (= [200 9 "warehouse-b" "analytics" "purchases" nil]+ ((juxt :source_id :source_db_id :db_name :schema :name :description) (nth rows 2)))))+ (testing "display_name pre-humanized at drain time"+ (is (= "Users" (:display_name (nth rows 0))))+ (is (= "Events" (:display_name (nth rows 1))))+ (is (= "Purchases" (:display_name (nth rows 2)))))+ (testing "target_id starts NULL — populated by the resolve step"+ (is (every? nil? (map :target_id rows))))))+ (finally (p/clear-staging-tables!))))++;;; ============================== fields drain ==============================++(deftest drain-fields-writes-all-shapes-test+ (try+ (p/clear-staging-tables!)+ (drain-fields!+ (batch+ ;; Flat field — no parent, no FK, no nfc_path+ [{:id 1000 :table_id 100 :name "id"+ :base_type "type/Integer" :database_type "int"}+ ;; Dictionary parent+ {:id 1001 :table_id 100 :name "data"+ :base_type "type/Dictionary" :database_type "json"}+ ;; nested leaf — parent_id + nfc_path+ {:id 1002 :table_id 100 :name "value"+ :base_type "type/Text" :database_type "text"+ :parent_id 1001+ :nfc_path ["data" "value"]}+ ;; unfolded leaf — nfc_path only, no parent_id+ {:id 1003 :table_id 100 :name "embedded → leaf"+ :base_type "type/Text" :database_type "text"+ :nfc_path ["embedded" "leaf"]}+ ;; FK source+ {:id 1004 :table_id 100 :name "account_fk"+ :base_type "type/Integer" :database_type "int"+ :fk_target_field_id 2000}+ ;; Field with effective_type set explicitly (≠ base_type case)+ {:id 1005 :table_id 100 :name "casted_value"+ :base_type "type/Text" :database_type "varchar"+ :effective_type "type/Text"+ :semantic_type "type/Email"+ :coercion_strategy "Coercion/UNIXSeconds->DateTime"+ :description "an email field"}]))+ (let [rows (staging-fields)+ by-id (into {} (map (juxt :source_id identity) rows))]+ (testing "six rows landed"+ (is (= 6 (count rows))))+ (testing "flat field has no parent or FK source ids"+ (let [r (by-id 1000)]+ (is (nil? (:source_parent_id r)))+ (is (nil? (:source_fk_target_id r)))+ (is (nil? (:nfc_path r)))))+ (testing "Dictionary parent row has no parent_id and no nfc_path"+ (let [r (by-id 1001)]+ (is (nil? (:source_parent_id r)))+ (is (nil? (:nfc_path r)))))+ (testing "nested leaf carries source_parent_id pointing at the parent's source_id"+ (let [r (by-id 1002)]+ (is (= 1001 (:source_parent_id r)))+ (is (= ["data" "value"] (json/decode (:nfc_path r))))))+ (testing "unfolded leaf has nfc_path but no source_parent_id"+ (let [r (by-id 1003)]+ (is (nil? (:source_parent_id r)))+ (is (= ["embedded" "leaf"] (json/decode (:nfc_path r))))))+ (testing "FK source carries source_fk_target_id"+ (let [r (by-id 1004)]+ (is (= 2000 (:source_fk_target_id r)))))+ (testing "optional payload columns roundtrip through staging"+ (let [r (by-id 1005)]+ (is (= "type/Text" (:effective_type r)))+ (is (= "type/Email" (:semantic_type r)))+ (is (= "Coercion/UNIXSeconds->DateTime" (:coercion_strategy r)))+ (is (= "an email field" (:description r)))))+ (testing "resolve outputs all start NULL"+ (doseq [{:keys [depth target_id target_table_id target_parent_id target_fk_target_id]} rows]+ (is (nil? depth))+ (is (nil? target_id))+ (is (nil? target_table_id))+ (is (nil? target_parent_id))+ (is (nil? target_fk_target_id)))))+ (finally (p/clear-staging-tables!))))++;;; ============================== empty-batch behavior ==============================++(deftest drain-tables-empty-batch-is-noop-test+ (try+ (p/clear-staging-tables!)+ (drain-tables! {} (batch []))+ (is (zero? (count (staging-tables))))+ (finally (p/clear-staging-tables!))))++(deftest drain-fields-empty-batch-is-noop-test+ (try+ (p/clear-staging-tables!)+ (drain-fields! (batch []))+ (is (zero? (count (staging-fields))))+ (finally (p/clear-staging-tables!))))++;;; ============================== validation failures ==============================++(deftest drain-tables-rejects-rows-missing-required-keys-test+ (try+ (p/clear-staging-tables!)+ (testing "missing :id throws with :kind :invalid-input"+ (is (= :invalid-input+ (try+ (drain-tables! {} (batch [{:db_id 7 :name "no-id" :schema "public"}]))+ nil+ (catch clojure.lang.ExceptionInfo e (:kind (ex-data e)))))))+ (testing "no rows landed in staging on validation failure"+ (is (zero? (count (staging-tables)))))+ (finally (p/clear-staging-tables!))))++(deftest drain-fields-rejects-rows-missing-required-keys-test+ (try+ (p/clear-staging-tables!)+ (testing "missing :base_type throws with :kind :invalid-input"+ (is (= :invalid-input+ (try+ (drain-fields! (batch [{:id 1 :table_id 100 :name "x" :database_type "text"}]))+ nil+ (catch clojure.lang.ExceptionInfo e (:kind (ex-data e)))))))+ (finally (p/clear-staging-tables!))))++;;; ============================== Drain-path equivalence ==============================++(def ^:private equivalence-databases-by-source-id+ {7 "warehouse-a" 9 "warehouse-b"})++(def ^:private equivalence-table-batch-rows+ [{:id 100 :db_id 7 :schema "public" :name "users" :description "user table"}+ {:id 101 :db_id 7 :schema nil :name "events"}+ {:id 200 :db_id 9 :schema "analytics" :name "purchases"}])++(def ^:private equivalence-field-batch-rows+ [{:id 1000 :table_id 100 :name "id"+ :base_type "type/Integer" :database_type "int"}+ {:id 1001 :table_id 100 :name "data"+ :base_type "type/Dictionary" :database_type "json"}+ {:id 1002 :table_id 100 :name "value"+ :base_type "type/Text" :database_type "text"+ :parent_id 1001 :nfc_path ["data" "value"]}+ {:id 1003 :table_id 100 :name "embedded → leaf"+ :base_type "type/Text" :database_type "text"+ :nfc_path ["embedded" "leaf"]}+ {:id 1004 :table_id 100 :name "account_fk"+ :base_type "type/Integer" :database_type "int"+ :fk_target_field_id 2000}+ {:id 1005 :table_id 100 :name "casted_value"+ :base_type "type/Text" :database_type "varchar"+ :effective_type "type/Text" :semantic_type "type/Email"+ :coercion_strategy "Coercion/UNIXSeconds->DateTime"+ :description "an email field"}])++(defn- snapshot-after! [drain-fn]+ (try+ (p/clear-staging-tables!)+ (drain-fn)+ {:tables (staging-tables) :fields (staging-fields)}+ (finally (p/clear-staging-tables!))))++(deftest drain-paths-produce-identical-staging-test+ (testing "the JDBC and PG-COPY drain paths produce the same staging-table+ contents for the same batch — guards against silent drift if+ one path's row-shaping diverges from the other. Only meaningful+ on a PG appdb (the JDBC variant runs on every appdb; the+ PG-COPY variant runs only on PG)."+ (when (= :postgres (mdb/db-type))+ (let [tbl-batch (batch equivalence-table-batch-rows)+ fld-batch (batch equivalence-field-batch-rows)+ jdbc (snapshot-after!+ (fn []+ (with-open [^Connection conn (.getConnection (mdb/data-source))+ ^PreparedStatement tps (.prepareStatement conn (p/tables-insert-sql))+ ^PreparedStatement fps (.prepareStatement conn (p/fields-insert-sql))]+ (p/drain-tables-batch-jdbc! tps equivalence-databases-by-source-id tbl-batch)+ (p/drain-fields-batch-jdbc! fps fld-batch))))+ pg-copy (snapshot-after!+ (fn []+ (with-open [^Connection conn (.getConnection (mdb/data-source))]+ (let [^PGConnection pg-conn (.unwrap conn PGConnection)+ copy-mgr (.getCopyAPI pg-conn)+ t-ci (.copyIn copy-mgr ^String p/tables-copy-sql)]+ (try+ (p/drain-tables-batch-pg-copy! t-ci equivalence-databases-by-source-id tbl-batch)+ (finally (.endCopy ^CopyIn t-ci))))+ (let [^PGConnection pg-conn (.unwrap conn PGConnection)+ copy-mgr (.getCopyAPI pg-conn)+ f-ci (.copyIn copy-mgr ^String p/fields-copy-sql)]+ (try+ (p/drain-fields-batch-pg-copy! f-ci fld-batch)+ (finally (.endCopy ^CopyIn f-ci)))))))]+ (is (= jdbc pg-copy)+ "JDBC drain produces the same staging rows as the PG-COPY drain")))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/fields_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/fields_test.cljnew file mode 100644index 00000000000..cde6b147089--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/fields_test.clj@@ -0,0 +1,497 @@+(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.fields-test+ "Tests for the per-depth field-merge functions: the three resolves+ (`resolve-target-table-ids-for-fields-in-staging!`,+ `fill-target-parent-ids-at-depth!`, `fill-target-fk-target-ids-at-depth!`,+ `resolve-target-field-ids-at-depth!`), the UPDATE/INSERT pair+ (`update-matched-fields-at-depth!`, `insert-new-fields-at-depth!`), and+ the orchestration loop (`merge-fields-by-depth!`)."+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.metadata-file-import.processors :as p]+ [metabase.test :as mt]+ [metabase.util.json :as json]+ [toucan2.core :as t2]))++(use-fixtures :once+ (fn [thunk]+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.+ (mt/db)+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (thunk))))++(defn- insert-staging-table-row!+ "Helper: drop one row into metabase_table_import. `overrides` can include+ `:target_id` to pre-resolve the row."+ [source-id db-name name & {:as overrides}]+ (t2/insert! :metabase_table_import+ (merge {:source_id source-id+ :source_db_id 1+ :db_name db-name+ :schema "PUBLIC"+ :name name+ :display_name (str/capitalize name)}+ overrides)))++(defn- insert-staging-field-row!+ "Helper: drop one row into metabase_field_import. Fills in NOT NULL+ defaults; `overrides` is everything else."+ [source-id name & {:as overrides}]+ (t2/insert! :metabase_field_import+ (merge {:source_id source-id+ :source_table_id 1+ :name name+ :base_type "type/Integer"+ :database_type "int"}+ overrides)))++(defn- field-staging-row+ "Read a staging row by source_id for assertion."+ [source-id]+ (t2/select-one+ [:metabase_field_import+ :source_id :source_table_id :source_parent_id :source_fk_target_id+ :name :base_type :database_type :nfc_path :depth+ :target_id :target_table_id :target_parent_id :target_fk_target_id]+ :source_id source-id))++;;; ============================== resolve-target-table-ids-for-fields ==============================++(deftest resolve-target-table-ids-for-fields-copies-from-table-staging-test+ (try+ (p/clear-staging-tables!)+ ;; Two staging tables, one pre-resolved (target_id = 42), one not.+ (insert-staging-table-row! 1 "warehouse" "users" :target_id 42)+ (insert-staging-table-row! 2 "warehouse" "ghost")+ (insert-staging-field-row! 100 "id" :source_table_id 1)+ (insert-staging-field-row! 200 "x" :source_table_id 2)+ (p/resolve-target-table-ids-for-fields-in-staging!)+ (is (= 42 (:target_table_id (field-staging-row 100))))+ (is (nil? (:target_table_id (field-staging-row 200)))+ "field whose table has no target_id stays NULL")+ (finally (p/clear-staging-tables!))))++;;; ============================== fill-target-parent-ids-at-depth ==============================++(deftest fill-target-parent-ids-only-touches-the-given-depth-test+ (try+ (p/clear-staging-tables!)+ ;; root at depth 0 (target_id = 1000 — pretend it's already been merged)+ (insert-staging-field-row! 10 "root" :depth 0 :target_id 1000)+ ;; child at depth 1 with parent pointing at the root+ (insert-staging-field-row! 11 "child" :depth 1 :source_parent_id 10)+ ;; another row at depth 2 — should NOT be touched by depth=1 call+ (insert-staging-field-row! 12 "grandchild" :depth 2 :source_parent_id 11)+ (p/fill-target-parent-ids-at-depth! 1)+ (is (= 1000 (:target_parent_id (field-staging-row 11)))+ "depth-1 row's target_parent_id picked up from parent at depth 0")+ (is (nil? (:target_parent_id (field-staging-row 12)))+ "depth-2 row not touched by depth=1 call")+ (finally (p/clear-staging-tables!))))++(deftest fill-target-parent-ids-skips-rows-without-source-parent-test+ (try+ (p/clear-staging-tables!)+ (insert-staging-field-row! 10 "root" :depth 0 :target_id 1000)+ (insert-staging-field-row! 11 "no-parent" :depth 1)+ (p/fill-target-parent-ids-at-depth! 1)+ (is (nil? (:target_parent_id (field-staging-row 11))))+ (finally (p/clear-staging-tables!))))++;;; ============================== fill-target-fk-target-ids-at-depth ==============================++(deftest fill-target-fk-target-ids-at-depth-test+ (try+ (p/clear-staging-tables!)+ (insert-staging-field-row! 50 "id" :depth 0 :target_id 5000)+ (insert-staging-field-row! 51 "account_fk" :depth 1 :source_fk_target_id 50)+ (p/fill-target-fk-target-ids-at-depth! 1)+ (is (= 5000 (:target_fk_target_id (field-staging-row 51))))+ (finally (p/clear-staging-tables!))))++;;; ============================== resolve-target-field-ids-at-depth ==============================++(deftest resolve-target-field-ids-matches-by-natural-key-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}+ :model/Field {target-id :id} {:table_id t-id :name "user_id"+ :base_type :type/Integer}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-field-row! 100 "user_id" :depth 0 :target_table_id t-id)+ (p/resolve-target-field-ids-at-depth! 0)+ (is (= target-id (:target_id (field-staging-row 100))))+ (finally (p/clear-staging-tables!)))))++(deftest resolve-target-field-ids-honors-parent-id-in-match-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}+ :model/Field {parent-id :id} {:table_id t-id :name "data" :base_type :type/Dictionary}+ :model/Field {child-id :id} {:table_id t-id :name "value"+ :base_type :type/Text :parent_id parent-id}+ ;; Sibling at the root level with the same name as the nested child+ :model/Field {root-id :id} {:table_id t-id :name "value" :base_type :type/Text}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-field-row! 100 "value" :depth 1 :target_table_id t-id+ :target_parent_id parent-id)+ (insert-staging-field-row! 101 "value" :depth 0 :target_table_id t-id)+ (p/resolve-target-field-ids-at-depth! 1)+ (p/resolve-target-field-ids-at-depth! 0)+ (is (= child-id (:target_id (field-staging-row 100)))+ "nested 'value' (parent_id set) matches the nested target")+ (is (= root-id (:target_id (field-staging-row 101)))+ "flat 'value' (parent_id NULL) matches the root target")+ (finally (p/clear-staging-tables!)))))++(deftest resolve-target-field-ids-leaves-null-when-no-match-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-field-row! 100 "no_such_field" :depth 0 :target_table_id t-id)+ (p/resolve-target-field-ids-at-depth! 0)+ (is (nil? (:target_id (field-staging-row 100))))+ (finally (p/clear-staging-tables!)))))++(deftest resolve-target-field-ids-skips-defective-duplicate-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}+ :model/Field {} {:table_id t-id :name "user_id"+ :base_type :type/Integer+ :is_defective_duplicate true}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-field-row! 100 "user_id" :depth 0 :target_table_id t-id)+ (p/resolve-target-field-ids-at-depth! 0)+ (is (nil? (:target_id (field-staging-row 100))))+ (finally (p/clear-staging-tables!)))))++(deftest resolve-target-field-ids-is-idempotent-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}+ :model/Field {target-id :id} {:table_id t-id :name "x" :base_type :type/Text}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id)+ (p/resolve-target-field-ids-at-depth! 0)+ (p/resolve-target-field-ids-at-depth! 0)+ (is (= target-id (:target_id (field-staging-row 100))))+ (finally (p/clear-staging-tables!)))))++;;; ============================== update-matched-fields-at-depth ==============================++(deftest update-matched-clobbers-payload-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}+ :model/Field {target-id :id} {:table_id t-id :name "x"+ :base_type :type/Integer+ :description "old"+ :database_type "int"+ :semantic_type nil}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id :target_id target-id+ :base_type "type/Integer" :database_type "bigint"+ :description "new" :semantic_type "type/PK")+ (p/update-matched-fields-at-depth! 0)+ (let [after (t2/select-one :model/Field :id target-id)]+ (is (= "new" (:description after)))+ (is (= "bigint" (:database_type after)))+ (is (= :type/PK (:semantic_type after))))+ (finally (p/clear-staging-tables!)))))++(deftest update-matched-skip-if-unchanged-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}+ :model/Field {target-id :id} {:table_id t-id :name "x"+ :base_type :type/Integer+ :description "same"+ :database_type "int"+ :active true}]+ (try+ (t2/query {:update :metabase_field+ :set {:updated_at #t "2020-01-01T00:00:00Z"}+ :where [:= :id target-id]})+ (let [orig (:updated_at (t2/select-one :model/Field :id target-id))]+ (p/clear-staging-tables!)+ ;; Staging carries identical payload — UPDATE must NOT fire.+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id :target_id target-id+ :base_type "type/Integer" :database_type "int"+ :description "same")+ (p/update-matched-fields-at-depth! 0)+ (is (= orig (:updated_at (t2/select-one :model/Field :id target-id)))+ "identical payload → no UPDATE → updated_at unchanged"))+ (finally (p/clear-staging-tables!)))))++(deftest update-matched-fires-when-payload-differs-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}+ :model/Field {target-id :id} {:table_id t-id :name "x"+ :base_type :type/Integer+ :description "old"+ :database_type "int"}]+ (try+ (t2/query {:update :metabase_field+ :set {:updated_at #t "2020-01-01T00:00:00Z"}+ :where [:= :id target-id]})+ (let [orig (:updated_at (t2/select-one :model/Field :id target-id))]+ (p/clear-staging-tables!)+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id :target_id target-id+ :base_type "type/Integer" :database_type "int"+ :description "new")+ (p/update-matched-fields-at-depth! 0)+ (is (not= orig (:updated_at (t2/select-one :model/Field :id target-id)))+ "different payload → UPDATE fires → updated_at bumped"))+ (finally (p/clear-staging-tables!)))))++;;; ============================== insert-new-fields-at-depth ==============================++(deftest insert-new-inserts-root-field-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-field-row! 100 "user_id" :depth 0 :target_table_id t-id+ :base_type "type/Integer" :database_type "int")+ (p/insert-new-fields-at-depth! 0)+ (let [inserted (t2/select-one :model/Field :table_id t-id :name "user_id")]+ (is (some? inserted))+ (is (= :type/Integer (:base_type inserted)))+ (is (= "int" (:database_type inserted)))+ (is (true? (:active inserted)))+ (is (nil? (:parent_id inserted)))+ (is (nil? (:fk_target_field_id inserted))))+ (finally (p/clear-staging-tables!)))))++(deftest insert-new-uses-target-parent-id-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}+ :model/Field {parent-id :id} {:table_id t-id :name "data" :base_type :type/Dictionary}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-field-row! 100 "value" :depth 1 :target_table_id t-id+ :base_type "type/Text" :database_type "text"+ :target_parent_id parent-id+ :nfc_path (json/encode ["data" "value"]))+ (p/insert-new-fields-at-depth! 1)+ (let [inserted (t2/select-one :model/Field :table_id t-id :name "value" :parent_id parent-id)]+ (is (some? inserted))+ (is (= parent-id (:parent_id inserted)))+ (is (= ["data" "value"] (:nfc_path inserted))))+ (finally (p/clear-staging-tables!)))))++(deftest insert-new-uses-target-fk-target-id-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}+ :model/Field {target-id :id} {:table_id t-id :name "id" :base_type :type/Integer}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-field-row! 100 "account_fk" :depth 1 :target_table_id t-id+ :base_type "type/Integer" :database_type "int"+ :target_fk_target_id target-id)+ (p/insert-new-fields-at-depth! 1)+ (let [inserted (t2/select-one :model/Field :table_id t-id :name "account_fk")]+ (is (= target-id (:fk_target_field_id inserted))))+ (finally (p/clear-staging-tables!)))))++(deftest insert-new-preserves-null-effective-type-test+ ;; The application reads NULL effective_type as "equal to base_type" at+ ;; use sites, so the stored NULL is correct.+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id+ :base_type "type/Integer" :database_type "int"+ :effective_type nil)+ (p/insert-new-fields-at-depth! 0)+ (let [inserted (t2/select-one :model/Field :table_id t-id :name "x")]+ (is (some? inserted))+ (is (nil? (:effective_type inserted))+ "NULL staging effective_type → NULL target effective_type"))+ (finally (p/clear-staging-tables!)))))++(deftest insert-new-preserves-explicit-effective-type-test+ ;; When source had effective_type ≠ base_type, the wire row carries the+ ;; explicit value. Staging carries it. INSERT writes it.+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id+ :base_type "type/Text" :database_type "varchar"+ :effective_type "type/PostgresEnum")+ (p/insert-new-fields-at-depth! 0)+ (let [inserted (t2/select-one :model/Field :table_id t-id :name "x")]+ (is (= :type/PostgresEnum (:effective_type inserted))))+ (finally (p/clear-staging-tables!)))))++(deftest update-matched-skip-if-unchanged-handles-effective-type-coalesce-test+ ;; Cross-checks the COALESCE branch in field-payload-changed-predicate:+ ;; target.effective_type explicitly == base_type, staging.effective_type+ ;; NULL → COALESCED values match → skip-if-unchanged fires.+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}+ :model/Field {target-id :id} {:table_id t-id :name "x"+ :base_type :type/Integer+ :effective_type :type/Integer+ :database_type "int"}]+ (try+ (t2/query {:update :metabase_field+ :set {:updated_at #t "2020-01-01T00:00:00Z"}+ :where [:= :id target-id]})+ (let [orig (:updated_at (t2/select-one :model/Field :id target-id))]+ (p/clear-staging-tables!)+ ;; Staging matches except effective_type is NULL (the export-omitted case).+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id :target_id target-id+ :base_type "type/Integer" :database_type "int"+ :effective_type nil)+ (p/update-matched-fields-at-depth! 0)+ (is (= orig (:updated_at (t2/select-one :model/Field :id target-id)))+ "COALESCE(target.effective_type, target.base_type) == COALESCE(NULL, staging.base_type) — no UPDATE"))+ (finally (p/clear-staging-tables!)))))++(deftest insert-new-skips-row-with-no-target-table-test+ (try+ (p/clear-staging-tables!)+ (let [count-before (t2/count :metabase_field)]+ (insert-staging-field-row! 100 "orphan" :depth 0 :target_table_id nil)+ (p/insert-new-fields-at-depth! 0)+ (is (= count-before (t2/count :metabase_field))))+ (finally (p/clear-staging-tables!))))++(deftest insert-new-skips-row-whose-parent-was-orphaned-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]+ (try+ (p/clear-staging-tables!)+ ;; Row has a source_parent_id but target_parent_id wasn't resolved+ ;; (would happen if the parent's table didn't match target).+ (insert-staging-field-row! 100 "orphaned_child" :depth 1+ :target_table_id t-id+ :source_parent_id 999)+ (p/insert-new-fields-at-depth! 1)+ (is (zero? (t2/count :model/Field :table_id t-id :name "orphaned_child")))+ (finally (p/clear-staging-tables!)))))++(deftest insert-new-keeps-row-whose-fk-target-was-dropped-test+ (testing "a field whose FK target was dropped during merge (source_fk_target_id+ set, target_fk_target_id unresolved) is still inserted with+ fk_target_field_id NULL — the FK is a semantic annotation, not structural."+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]+ (try+ (p/clear-staging-tables!)+ ;; Field sits in a matched table (target_table_id set) and has an FK whose+ ;; target was dropped: source_fk_target_id is set, target_fk_target_id NULL.+ (insert-staging-field-row! 100 "user_id" :depth 1+ :target_table_id t-id+ :source_fk_target_id 999)+ (p/insert-new-fields-at-depth! 1)+ (let [inserted (t2/select-one :model/Field :table_id t-id :name "user_id")]+ (is (some? inserted)+ "field in a matched table is inserted even though its FK target didn't resolve")+ (is (nil? (:fk_target_field_id inserted))+ "the unresolved FK is stored as NULL, not the field dropped"))+ (finally (p/clear-staging-tables!))))))++;;; ============================== merge-fields-by-depth (integration) ==============================++(deftest merge-fields-by-depth-handles-flat-fields-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)+ (insert-staging-field-row! 100 "a" :depth 0 :target_table_id t-id)+ (insert-staging-field-row! 101 "b" :depth 0 :target_table_id t-id)+ (p/merge-fields-by-depth!)+ (is (= #{"a" "b"} (set (map :name (t2/select :model/Field :table_id t-id)))))+ (finally (p/clear-staging-tables!)))))++(deftest merge-fields-by-depth-handles-parent-chain-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)+ ;; root → parent → grandchild+ (insert-staging-field-row! 10 "data" :depth 0 :target_table_id t-id+ :base_type "type/Dictionary" :database_type "json")+ (insert-staging-field-row! 11 "inner" :depth 1 :target_table_id t-id+ :base_type "type/Dictionary" :database_type "json"+ :source_parent_id 10+ :nfc_path (json/encode ["data"]))+ (insert-staging-field-row! 12 "leaf" :depth 2 :target_table_id t-id+ :base_type "type/Text" :database_type "text"+ :source_parent_id 11+ :nfc_path (json/encode ["data" "inner"]))+ (p/merge-fields-by-depth!)+ (let [root (t2/select-one :model/Field :table_id t-id :name "data")+ inner (t2/select-one :model/Field :table_id t-id :name "inner")+ leaf (t2/select-one :model/Field :table_id t-id :name "leaf")]+ (is (some? root))+ (is (= (:id root) (:parent_id inner)))+ (is (= (:id inner) (:parent_id leaf)))+ (is (= ["data" "inner"] (:nfc_path leaf))))+ (finally (p/clear-staging-tables!)))))++(deftest merge-fields-by-depth-handles-fk-chain-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)+ ;; A is non-FK, B points at A via fk_target.+ (insert-staging-field-row! 10 "a" :depth 0 :target_table_id t-id)+ (insert-staging-field-row! 11 "b" :depth 1 :target_table_id t-id+ :source_fk_target_id 10)+ (p/merge-fields-by-depth!)+ (let [a (t2/select-one :model/Field :table_id t-id :name "a")+ b (t2/select-one :model/Field :table_id t-id :name "b")]+ (is (some? a))+ (is (some? b))+ (is (= (:id a) (:fk_target_field_id b))))+ (finally (p/clear-staging-tables!)))))++(deftest merge-fields-by-depth-reconciles-fk-on-matched-field-test+ (testing "a matched field's fk_target_field_id is reconciled from the import+ file, even when no other payload column differs — the FK isn't part+ of the natural-key match, so it can drift independently of row identity."+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}+ :model/Field {id-field :id} {:table_id t-id :name "id"+ :base_type :type/Integer :database_type "int"}+ :model/Field {acct-field :id} {:table_id t-id :name "account_id"+ :base_type :type/Integer :database_type "int"}]+ (try+ (is (nil? (:fk_target_field_id (t2/select-one :model/Field :id acct-field)))+ "fixture starts with account_id un-FK'd")+ (p/clear-staging-tables!)+ (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)+ ;; Staging payloads are identical to the live rows except account_id now+ ;; carries an FK pointing at id — so only the FK has drifted.+ (insert-staging-field-row! 10 "id" :depth 0 :target_table_id t-id)+ (insert-staging-field-row! 11 "account_id" :depth 1 :target_table_id t-id+ :source_fk_target_id 10)+ (p/merge-fields-by-depth!)+ (is (= id-field (:fk_target_field_id (t2/select-one :model/Field :id acct-field)))+ "matched field's fk_target_field_id is reconciled from the file")+ (finally (p/clear-staging-tables!))))))++(deftest merge-fields-by-depth-is-idempotent-test+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]+ (try+ (p/clear-staging-tables!)+ (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)+ (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id)+ (p/merge-fields-by-depth!)+ (let [first-id (:id (t2/select-one :model/Field :table_id t-id :name "x"))+ _ (p/merge-fields-by-depth!)+ second-id (:id (t2/select-one :model/Field :table_id t-id :name "x"))]+ (is (= first-id second-id))+ (is (= 1 (t2/count :model/Field :table_id t-id :name "x"))))+ (finally (p/clear-staging-tables!)))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/orphan_check_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/orphan_check_test.cljnew file mode 100644index 00000000000..45bf495c0ed--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/orphan_check_test.clj@@ -0,0 +1,167 @@+(ns metabase-enterprise.serialization.metadata-file-import.orphan-check-test+ "Tests for the pre-flight orphan handling that runs after drain and before+ the depth-walk merge. Asymmetric by ref kind:++ - `source_parent_id` orphans → fatal (`assert-no-orphan-refs!` throws+ `:file-incomplete`). A field claiming a non-present parent has no+ structurally valid position.+ - `source_fk_target_id` orphans → lossy (`null-orphan-fk-target-refs!`+ NULLs the ref and returns the count for the caller to WARN-log).+ A missing fk target degrades cleanly to 'no fk relationship known'."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.metadata-file-import.processors :as p]+ [toucan2.core :as t2]))++(defn- insert-staging-fields!+ "Insert a sequence of partial field rows directly into `metabase_field_import`,+ filling in required columns with sensible defaults so we can focus the test+ on the source_parent_id / source_fk_target_id graph."+ [rows]+ (let [defaults {:name "x" :base_type "type/Integer" :database_type "int"}]+ (t2/insert! :metabase_field_import+ (mapv (fn [row] (merge defaults {:source_table_id 1} row)) rows))))++;;; ============================== happy path ==============================++(deftest empty-staging-passes-test+ (try+ (p/clear-staging-tables!)+ (is (nil? (p/assert-no-orphan-refs!)))+ (finally (p/clear-staging-tables!))))++(deftest staging-with-only-roots-passes-test+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 1}+ {:source_id 2}+ {:source_id 3}])+ (is (nil? (p/assert-no-orphan-refs!)))+ (finally (p/clear-staging-tables!))))++(deftest staging-with-resolvable-parent-and-fk-passes-test+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 10}+ {:source_id 11 :source_parent_id 10}+ {:source_id 12 :source_fk_target_id 11}+ {:source_id 13 :source_parent_id 11 :source_fk_target_id 10}])+ (is (nil? (p/assert-no-orphan-refs!)))+ (finally (p/clear-staging-tables!))))++;;; ============================== orphan parent ref ==============================++(deftest single-orphan-parent-ref-throws-test+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 100}+ {:source_id 101 :source_parent_id 999}])+ (let [thrown (try (p/assert-no-orphan-refs!) nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (some? thrown) "should have thrown")+ (is (= :file-incomplete (:kind (ex-data thrown))))+ (is (= 1 (:orphan-parent-count (ex-data thrown))))+ (is (= [{:source_id 101 :source_parent_id 999}]+ (:orphan-parent-sample (ex-data thrown))))+ (testing "error message includes the count"+ (is (re-find #"1 orphan parent" (ex-message thrown)))))+ (finally (p/clear-staging-tables!))))++;;; ============================== orphan fk-target ref ==============================++(deftest orphan-fk-target-ref-is-not-fatal-for-assert-test+ ;; Handled separately by null-orphan-fk-target-refs!.+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 200}+ {:source_id 201 :source_fk_target_id 9999}])+ (is (nil? (p/assert-no-orphan-refs!))+ "orphan fk-target alone should not throw")+ (finally (p/clear-staging-tables!))))++(deftest orphan-fk-target-ref-gets-nulled-test+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 200}+ {:source_id 201 :source_fk_target_id 9999}])+ (let [result (p/null-orphan-fk-target-refs!)]+ (is (= 1 (:count result)))+ (is (= [{:source_id 201 :source_fk_target_id 9999}]+ (:sample result))))+ (testing "the orphan ref is NULL'd in staging"+ (is (= [{:source_id 201 :source_fk_target_id nil}]+ (t2/query+ {:select [:source_id :source_fk_target_id]+ :from [:metabase_field_import]+ :where [:= :source_id 201]})))+ (testing "the non-orphan row is untouched"+ (is (= [{:source_id 200 :source_fk_target_id nil}]+ (t2/query+ {:select [:source_id :source_fk_target_id]+ :from [:metabase_field_import]+ :where [:= :source_id 200]})))))+ (finally (p/clear-staging-tables!))))++(deftest valid-fk-target-ref-survives-null-pass-test+ ;; null-orphan-fk-target-refs! must not touch rows whose fk target IS+ ;; present in staging.+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 210}+ {:source_id 211 :source_fk_target_id 210}])+ (let [result (p/null-orphan-fk-target-refs!)]+ (is (= 0 (:count result)))+ (is (nil? (:sample result))))+ (is (= [{:source_id 211 :source_fk_target_id 210}]+ (t2/query+ {:select [:source_id :source_fk_target_id]+ :from [:metabase_field_import]+ :where [:= :source_id 211]})))+ (finally (p/clear-staging-tables!))))++;;; ============================== both kinds of orphan ==============================++(deftest both-orphan-types-handled-asymmetrically-test+ ;; Orphan parent still throws; orphan fk-target is handled by the separate+ ;; null-pass. The throw happens before the null-pass would run.+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 300}+ {:source_id 301 :source_parent_id 9001}+ {:source_id 302 :source_fk_target_id 9002}])+ (let [thrown (try (p/assert-no-orphan-refs!) nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (some? thrown) "assert-no-orphan-refs! should still throw on parent orphan")+ (is (= :file-incomplete (:kind (ex-data thrown))))+ (is (= 1 (:orphan-parent-count (ex-data thrown))))+ (is (seq (:orphan-parent-sample (ex-data thrown)))))+ (finally (p/clear-staging-tables!))))++;;; ============================== sample truncation ==============================++(deftest orphan-sample-caps-at-ten-rows-test+ (try+ (p/clear-staging-tables!)+ ;; 25 rows all pointing at a non-existent parent+ (insert-staging-fields!+ (cons {:source_id 1}+ (for [i (range 1000 1025)] {:source_id i :source_parent_id 999})))+ (let [thrown (try (p/assert-no-orphan-refs!) nil+ (catch clojure.lang.ExceptionInfo e e))]+ (testing "count reflects all 25 orphans"+ (is (= 25 (:orphan-parent-count (ex-data thrown)))))+ (testing "sample is capped at 10"+ (is (= 10 (count (:orphan-parent-sample (ex-data thrown)))))))+ (finally (p/clear-staging-tables!))))++;;; ============================== self-reference is not an orphan ==============================++(deftest self-referential-parent-id-is-not-an-orphan-test+ ;; A row whose source_parent_id equals its own source_id is a cycle —+ ;; that's a different problem (caught by depth tagging) but it's NOT an+ ;; orphan since the referenced source_id IS in the file.+ (try+ (p/clear-staging-tables!)+ (insert-staging-fields! [{:source_id 500 :source_parent_id 500}])+ (is (nil? (p/assert-no-orphan-refs!)))+ (finally (p/clear-staging-tables!))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/parsers_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/parsers_test.cljnew file mode 100644index 00000000000..4fc46cb6149--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/parsers_test.clj@@ -0,0 +1,125 @@+(ns ^:parallel metabase-enterprise.serialization.metadata-file-import.parsers-test+ "Tests for the streaming JSON parser. Each test writes a temp file, then+ asserts on the batches passed to `process-batch!`."+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.metadata-file-import.parsers :as parsers])+ (:import+ (java.io File)))++(set! *warn-on-reflection* true)++(defn- temp-json-file ^File [^String content]+ (let [f (File/createTempFile "parsers-test-" ".json")]+ (.deleteOnExit f)+ (spit f content)+ f))++(defn- collect-batches!+ "Stream `json-string` through the parser and return a vector of all batches that+ the callback received, in order. Each batch is a vector of `[line-num row]` tuples."+ [json-string array-key batch-size]+ (let [file (temp-json-file json-string)+ batches (atom [])]+ (try+ (parsers/stream-array-batches! file array-key batch-size+ (fn [batch] (swap! batches conj (vec batch))))+ @batches+ (finally (.delete file)))))++(deftest streams-rows-as-keyword-keyed-clojure-maps-test+ (testing "each item in the named array is parsed into a Clojure map with keyword keys+ (not LinkedHashMap with string keys), wrapped in a [line-num row] tuple"+ (let [batches (collect-batches!+ "{\"databases\":[{\"id\":17,\"name\":\"pg\",\"engine\":\"postgres\"},+ {\"id\":18,\"name\":\"h2\",\"engine\":\"h2\"}]}"+ :databases 100)]+ (is (= 1 (count batches)))+ (is (= [[1 {:id 17 :name "pg" :engine "postgres"}]+ [2 {:id 18 :name "h2" :engine "h2"}]]+ (first batches))))))++(deftest line-numbers-are-1-indexed-across-batches-test+ (testing "line-num is 1-indexed and continues across batch boundaries — so the third+ item in the source array is `line-num 3`, regardless of which batch it lands in"+ (let [json (str "{\"xs\":["+ (str/join "," (for [i (range 1 6)] (format "{\"v\":%d}" i)))+ "]}")+ batches (collect-batches! json :xs 2)]+ (is (= 3 (count batches))) ;; 2 + 2 + 1+ (is (= [[1 {:v 1}] [2 {:v 2}]] (nth batches 0)))+ (is (= [[3 {:v 3}] [4 {:v 4}]] (nth batches 1)))+ (is (= [[5 {:v 5}]] (nth batches 2))))))++(deftest empty-array-invokes-callback-zero-times-test+ (testing "if the named array is empty, the callback is not invoked at all — caller+ sees no batches, but the parser does not throw"+ (is (= [] (collect-batches! "{\"xs\":[]}" :xs 100)))))++(deftest accepts-keyword-or-string-array-key-test+ (testing "array-key may be passed as a Clojure keyword or as a string; both forms+ normalize to the same JSON key lookup"+ (let [json "{\"databases\":[{\"id\":1}]}"]+ (is (= [[[1 {:id 1}]]] (collect-batches! json :databases 10)))+ (is (= [[[1 {:id 1}]]] (collect-batches! json "databases" 10))))))++(deftest missing-array-key-throws-shape-error-test+ (testing "a top-level object that doesn't contain the requested key throws ex-info with+ :kind :missing-key"+ (try+ (collect-batches! "{\"other\":[]}" :databases 10)+ (is false "should have thrown")+ (catch clojure.lang.ExceptionInfo e+ (is (= :missing-key (:kind (ex-data e))))+ (is (= "databases" (:key (ex-data e))))))))++(deftest non-array-value-at-key-throws-shape-error-test+ (testing "a top-level key whose value is not an array throws ex-info with :kind :bad-shape"+ (try+ (collect-batches! "{\"databases\":{\"not\":\"an array\"}}" :databases 10)+ (is false "should have thrown")+ (catch clojure.lang.ExceptionInfo e+ (is (= :bad-shape (:kind (ex-data e))))))))++(deftest non-object-top-level-throws-shape-error-test+ (testing "a document that doesn't begin with a top-level object throws"+ (try+ (collect-batches! "[1, 2, 3]" :anything 10)+ (is false "should have thrown")+ (catch clojure.lang.ExceptionInfo e+ (is (= :bad-shape (:kind (ex-data e))))))))++(deftest skips-other-top-level-keys-before-target-test+ (testing "the parser advances past unrelated top-level keys (and their values, including+ nested structures) before reaching the target array"+ (let [json (str "{\"version\":1,"+ "\"unrelated_object\":{\"a\":1,\"b\":[2,3]},"+ "\"unrelated_array\":[4,5,6],"+ "\"databases\":[{\"id\":99}]}")+ batches (collect-batches! json :databases 10)]+ (is (= [[[1 {:id 99}]]] batches)))))++(deftest exact-batch-size-emits-single-batch-test+ (testing "boundary: when item count exactly equals batch size, callback fires once"+ (let [json "{\"xs\":[{\"v\":1},{\"v\":2}]}"+ batches (collect-batches! json :xs 2)]+ (is (= 1 (count batches)))+ (is (= 2 (count (first batches)))))))++(deftest single-item-with-batch-size-1-test+ (testing "boundary: batch size 1 fires the callback once per item"+ (let [json "{\"xs\":[{\"v\":1},{\"v\":2},{\"v\":3}]}"+ batches (collect-batches! json :xs 1)]+ (is (= 3 (count batches)))+ (is (= [[1 {:v 1}]] (nth batches 0)))+ (is (= [[2 {:v 2}]] (nth batches 1)))+ (is (= [[3 {:v 3}]] (nth batches 2))))))++(deftest file-handle-is-closed-after-streaming-test+ (testing "the parser manages the file lifecycle: the InputStream and Reader are+ closed after streaming completes (or throws). On Windows a sharing+ violation would surface if the FD were leaked."+ (let [file (temp-json-file "{\"xs\":[]}")]+ (parsers/stream-array-batches! file :xs 10 (fn [_]))+ (is (.delete file)))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/processors_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/processors_test.cljnew file mode 100644index 00000000000..2e315c6c229--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/processors_test.clj@@ -0,0 +1,151 @@+(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.processors-test+ "Tests for the pure batch processors. Each processor is exercised against an+ appdb populated via `mt/with-temp` — no HTTP, no streaming parser. Tests+ verify the processor's behavior contract: input shape, return shape, batch+ ordering, error attribution, and observable side-effects on the appdb."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.metadata-file-import.processors :as processors]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(use-fixtures :once+ (fn [thunk]+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.+ (mt/db)+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (thunk))))++;;; ============================== with-staging-tables ==============================++(defn- count-staging+ "Total row count across both staging tables."+ []+ (+ (t2/count :metabase_table_import)+ (t2/count :metabase_field_import)))++(deftest clear-staging-tables-empties-both-tables-test+ (testing "clear-staging-tables! deletes every row from both staging tables"+ (t2/insert! :metabase_table_import {:source_id 1 :source_db_id 1 :db_name "x" :name "t"})+ (t2/insert! :metabase_field_import {:source_id 1 :source_table_id 1+ :name "f" :base_type "type/Integer" :database_type "int"})+ (processors/clear-staging-tables!)+ (is (zero? (count-staging))+ "both staging tables empty after clear-staging-tables!")))++(deftest with-staging-tables-returns-body-value-test+ (testing "with-staging-tables yields the value of the body's last form"+ (is (= ::sentinel+ (processors/with-staging-tables ::sentinel)))))++(deftest with-staging-tables-clears-on-entry-test+ (testing "rows present before the macro are gone by the time the body runs"+ (t2/insert! :metabase_table_import {:source_id 1 :source_db_id 1 :db_name "leftover" :name "t"})+ (t2/insert! :metabase_field_import {:source_id 1 :source_table_id 1+ :name "f" :base_type "type/Integer" :database_type "int"})+ (let [observed (processors/with-staging-tables (count-staging))]+ (is (zero? observed)+ "macro entry wipes pre-existing rows so the body sees an empty staging area"))))++(deftest with-staging-tables-clears-on-successful-exit-test+ (testing "rows inserted inside the body are gone after the macro returns normally"+ (processors/with-staging-tables+ (t2/insert! :metabase_table_import {:source_id 1 :source_db_id 1 :db_name "x" :name "t"})+ (t2/insert! :metabase_field_import {:source_id 1 :source_table_id 1+ :name "f" :base_type "type/Integer" :database_type "int"}))+ (is (zero? (count-staging))+ "macro exit wipes rows the body added")))++(deftest with-staging-tables-clears-on-thrown-exit-and-propagates-test+ (testing "if the body throws, the exception bubbles AND staging tables are still wiped (try/finally)"+ (let [thrown (atom nil)]+ (try+ (processors/with-staging-tables+ (t2/insert! :metabase_table_import {:source_id 1 :source_db_id 1 :db_name "x" :name "t"})+ (t2/insert! :metabase_field_import {:source_id 1 :source_table_id 1+ :name "f" :base_type "type/Integer" :database_type "int"})+ (throw (ex-info "boom" {:kind ::test-error})))+ (catch clojure.lang.ExceptionInfo e+ (reset! thrown e)))+ (is (= ::test-error (:kind (ex-data @thrown)))+ "exception propagates with its ex-data intact")+ (is (zero? (count-staging))+ "staging tables wiped even though body threw"))))++;;; ============================== process-databases ==============================++(deftest process-databases-matches-by-name-and-engine-test+ (testing "a source row whose (name, engine) pair matches an existing target Database+ produces a :matched result whose :target-id is the existing row's id.+ :source-id carries the wire :id (source appdb's integer database id);+ :name is also surfaced for the orchestrator's db_name denormalization map."+ (mt/with-temp [:model/Database {target-id :id} {:name "imported-db" :engine :postgres}]+ (is (= [{:source-id 100 :name "imported-db" :target-id target-id :status :matched}]+ (into [] (processors/process-databases!+ [[1 {:id 100 :name "imported-db" :engine "postgres"}]])))))))++(deftest process-databases-emits-no-match-when-name-or-engine-differs-test+ (testing "an unmatched row emits a :no-match result with line attribution and a+ human-readable detail string. Mismatches are non-fatal — the loader logs and+ skips dependents rather than aborting boot."+ (let [[result] (into [] (processors/process-databases!+ [[7 {:id 200 :name "no-such-db-zzz" :engine "h2"}]]))]+ (is (= 200 (:source-id result)))+ (is (= "no-such-db-zzz" (:name result)))+ (is (= :no-match (:status result)))+ (is (= 7 (:line result)))+ (is (string? (:detail result)))+ (is (not (contains? result :target-id))+ "no :target-id key on no-match (so callers can use (:target-id r) as a presence check)"))))++(deftest process-databases-disambiguates-by-engine-pair-test+ (testing "two databases share a name but differ by engine; matching uses the+ (name, engine) pair, not just name"+ (mt/with-temp [:model/Database {pg-id :id} {:name "shared-name-test" :engine :postgres}+ :model/Database {h2-id :id} {:name "shared-name-test" :engine :h2}]+ (let [[r] (into [] (processors/process-databases!+ [[1 {:id 300 :name "shared-name-test" :engine "h2"}]]))]+ (is (= h2-id (:target-id r)))+ (is (not= pg-id (:target-id r)))))))++(deftest process-databases-validation-failure-throws-with-attribution-test+ (testing "a malformed row (missing required key) throws ex-info carrying+ :kind :invalid-input, the file line number, and the row's wire :id+ as the :source-id (for attribution in the boot-time error message)."+ (let [e (is (thrown? clojure.lang.ExceptionInfo+ (into [] (processors/process-databases!+ [[42 {:id 400 :name "missing-engine"}]])))) ;; no :engine+ data (ex-data e)]+ (is (= :invalid-input (:kind data)))+ (is (= 42 (:line data)))+ (is (= 400 (:source-id data))))))++(deftest process-databases-preserves-input-order-test+ (testing "results are in input order regardless of internal SELECT ordering or+ which entries match"+ (mt/with-temp [:model/Database {a-id :id} {:name "order-a" :engine :postgres}+ :model/Database {b-id :id} {:name "order-b" :engine :postgres}]+ (let [results (into [] (processors/process-databases!+ [[1 {:id 501 :name "order-b" :engine "postgres"}]+ [2 {:id 502 :name "no-such-name-q" :engine "h2"}]+ [3 {:id 503 :name "order-a" :engine "postgres"}]]))]+ (is (= [501 502 503] (mapv :source-id results)))+ (is (= ["order-b" "no-such-name-q" "order-a"] (mapv :name results)))+ (is (= [b-id nil a-id] (mapv :target-id results)))+ (is (= [:matched :no-match :matched] (mapv :status results)))))))++(deftest process-databases-empty-batch-test+ (testing "empty input → empty output, no SQL, no exception"+ (is (= [] (into [] (processors/process-databases! []))))))++(deftest process-databases-result-is-streamable-test+ (testing "the return value supports both `reduce` (the loader's fold path) and+ seq/iteration (the test path) without re-running the eager batch work"+ (mt/with-temp [:model/Database {target-id :id} {:name "stream-probe" :engine :postgres}]+ (let [result (processors/process-databases!+ [[1 {:id 700 :name "stream-probe" :engine "postgres"}]])+ via-reduce (reduce (fn [acc r] (assoc acc (:source-id r) (:target-id r))) {} result)+ via-seq (vec (seq result))]+ (is (= {700 target-id} via-reduce))+ (is (= 1 (count via-seq)))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/schemas_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/schemas_test.cljnew file mode 100644index 00000000000..bbd98ab9bfd--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/schemas_test.clj@@ -0,0 +1,145 @@+(ns ^:parallel metabase-enterprise.serialization.metadata-file-import.schemas-test+ "Tests for the Malli schemas validating per-line shapes streamed by the+ metadata file importer. The schemas describe the wire format+ (source-side integer IDs); each row's `:id` is an integer, and cross-row+ references (`:db_id`, `:table_id`, `:parent_id`, `:fk_target_field_id`)+ are integers pointing at another row's `:id` in the same file.++ The wire-format contract against actual export output is anchored by+ `wire_format_test.clj`; these tests focus on schema-level+ acceptance/rejection semantics in isolation."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.metadata-file-import.schemas :as schemas]+ [metabase.util.malli.registry :as mr]))++(defn- al+ "Build a `java.util.ArrayList` from `xs` — mimics what Jackson hands the+ importer for JSON arrays."+ [xs]+ (java.util.ArrayList. ^java.util.List xs))++;;; ============================== ::database-info =====================================++(deftest database-info-accepts-valid-shape-test+ (is (mr/validate ::schemas/database-info {:id 7 :name "warehouse" :engine "postgres"})))++(deftest database-info-rejects-missing-required-keys-test+ (is (not (mr/validate ::schemas/database-info {:name "warehouse" :engine "postgres"}))+ "missing :id")+ (is (not (mr/validate ::schemas/database-info {:id 7 :name "warehouse"}))+ "missing :engine")+ (is (not (mr/validate ::schemas/database-info {:id 7 :engine "postgres"}))+ "missing :name")+ (is (not (mr/validate ::schemas/database-info {}))))++(deftest database-info-rejects-wrong-types-test+ (is (not (mr/validate ::schemas/database-info {:id "not-an-int" :name "warehouse" :engine "postgres"}))+ ":id must be an integer")+ (is (not (mr/validate ::schemas/database-info {:id 7 :name 5 :engine "postgres"}))+ ":name must be a string")+ (is (not (mr/validate ::schemas/database-info {:id 7 :name "warehouse" :engine 42}))+ ":engine must be a string"))++;;; ============================== ::table-info ========================================++(deftest table-info-accepts-valid-shape-test+ (testing "required keys only"+ (is (mr/validate ::schemas/table-info+ {:id 100 :db_id 7 :name "orders"})))+ (testing "with optional :schema and :description"+ (is (mr/validate ::schemas/table-info+ {:id 100 :db_id 7 :name "orders"+ :schema "public" :description "customer orders"})))+ (testing ":schema present-and-null is accepted"+ (is (mr/validate ::schemas/table-info+ {:id 100 :db_id 7 :name "orders" :schema nil}))))++(deftest table-info-rejects-missing-required-keys-test+ (is (not (mr/validate ::schemas/table-info {:db_id 7 :name "orders"}))+ "missing :id")+ (is (not (mr/validate ::schemas/table-info {:id 100 :name "orders"}))+ "missing :db_id")+ (is (not (mr/validate ::schemas/table-info {:id 100 :db_id 7}))+ "missing :name"))++(deftest table-info-rejects-wrong-types-test+ (is (not (mr/validate ::schemas/table-info {:id "wh" :db_id 7 :name "orders"}))+ ":id must be an integer")+ (is (not (mr/validate ::schemas/table-info {:id 100 :db_id "wh" :name "orders"}))+ ":db_id must be an integer"))++;;; ============================== ::field-info ========================================++(deftest field-info-accepts-flat-root-field-test+ (testing "flat root field — required keys only, no parent or nfc_path"+ (is (mr/validate ::schemas/field-info+ {:id 1000 :table_id 100 :name "id"+ :base_type "type/Integer" :database_type "integer"}))))++(deftest field-info-accepts-row-with-parent-id-and-nfc-path-test+ (testing "nested leaf: both :parent_id and :nfc_path set"+ (is (mr/validate ::schemas/field-info+ {:id 1002 :table_id 100 :name "zip"+ :base_type "type/Text" :database_type "text"+ :parent_id 1001+ :nfc_path ["address"]}))))++(deftest field-info-accepts-fk-target-test+ (is (mr/validate ::schemas/field-info+ {:id 1003 :table_id 100 :name "user_id"+ :base_type "type/Integer" :database_type "integer"+ :fk_target_field_id 2000})))++(deftest field-info-accepts-row-with-nfc-path-and-no-parent-id-test+ (testing "unfolded leaf: :nfc_path set, no :parent_id (no parent storage row exists)"+ (is (mr/validate ::schemas/field-info+ {:id 1004 :table_id 100 :name "payload → address → zip"+ :base_type "type/Text" :database_type "text"+ :nfc_path ["payload" "address" "zip"]})))+ (testing "Jackson ArrayList for :nfc_path also validates"+ (is (mr/validate ::schemas/field-info+ {:id 1004 :table_id 100 :name "payload → address → zip"+ :base_type "type/Text" :database_type "text"+ :nfc_path (al ["payload" "address" "zip"])}))))++(deftest field-info-accepts-all-optional-keys-test+ (is (mr/validate ::schemas/field-info+ {:id 1005 :table_id 100 :name "amount"+ :base_type "type/Float" :database_type "DECIMAL"+ :description "order total"+ :effective_type "type/Float"+ :semantic_type "type/Currency"+ :coercion_strategy "Coercion/UNIXSeconds->DateTime"+ :parent_id 1001+ :fk_target_field_id 2000+ :nfc_path ["amount"]})))++(deftest field-info-rejects-missing-required-keys-test+ (let [base {:id 1000 :table_id 100 :name "x"+ :base_type "type/Integer" :database_type "integer"}]+ (is (not (mr/validate ::schemas/field-info (dissoc base :id))) "missing :id")+ (is (not (mr/validate ::schemas/field-info (dissoc base :table_id))) "missing :table_id")+ (is (not (mr/validate ::schemas/field-info (dissoc base :name))) "missing :name")+ (is (not (mr/validate ::schemas/field-info (dissoc base :base_type))) "missing :base_type")+ (is (not (mr/validate ::schemas/field-info (dissoc base :database_type))) "missing :database_type")))++(deftest field-info-rejects-wrong-id-types-test+ (let [base {:id 1000 :table_id 100 :name "x"+ :base_type "type/Integer" :database_type "integer"}]+ (is (not (mr/validate ::schemas/field-info (assoc base :id "not-int")))+ ":id must be an integer")+ (is (not (mr/validate ::schemas/field-info (assoc base :table_id "not-int")))+ ":table_id must be an integer")+ (is (not (mr/validate ::schemas/field-info (assoc base :parent_id "not-int")))+ ":parent_id (when present) must be an integer")+ (is (not (mr/validate ::schemas/field-info (assoc base :fk_target_field_id "not-int")))+ ":fk_target_field_id (when present) must be an integer")))++(deftest field-info-rejects-malformed-nfc-path-test+ (let [base {:id 1000 :table_id 100 :name "x"+ :base_type "type/Integer" :database_type "integer"}]+ (is (not (mr/validate ::schemas/field-info (assoc base :nfc_path "not-a-list")))+ ":nfc_path must be a list/vector")+ (is (not (mr/validate ::schemas/field-info (assoc base :nfc_path [1 2 3])))+ ":nfc_path elements must be strings")))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/tables_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/tables_test.cljnew file mode 100644index 00000000000..633bac0e132--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/tables_test.clj@@ -0,0 +1,258 @@+(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.tables-test+ "Tests for `resolve-target-table-ids-in-staging!` and `merge-tables!` —+ the table-merge half of the pipeline. These functions key on the+ natural-key tuple (db_name, schema, name) against `metabase_table`,+ filtered to active non-defective rows."+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.metadata-file-import.processors :as p]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(use-fixtures :once+ (fn [thunk]+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.+ (mt/db)+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (thunk))))++(defn- staging-row [source-id db-name name & {:as overrides}]+ (merge {:source_id source-id+ :source_db_id 1+ :db_name db-name+ :schema "PUBLIC"+ :name name+ :display_name (str/capitalize name)}+ overrides))++(defn- staging-rows-by-source-id []+ ;; HoneySQL (not raw SQL) so `schema` is dialect-quoted correctly on MySQL.+ (into {} (map (juxt :source_id identity))+ (t2/query {:select [:source_id :db_name :schema :name :description :target_id]+ :from [:metabase_table_import]})))++;;; ============================== resolve-target-table-ids ==============================++(deftest resolve-populates-target-id-when-match-exists-test+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"}]+ (try+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])+ (p/resolve-target-table-ids-in-staging!)+ (let [row (get (staging-rows-by-source-id) 100)]+ (is (= target-id (:target_id row))))+ (finally (p/clear-staging-tables!)))))++(deftest resolve-leaves-target-id-null-when-no-match-test+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {} {:db_id db-id :schema "PUBLIC" :name "users"}]+ (try+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "no_such_table")])+ (p/resolve-target-table-ids-in-staging!)+ (let [row (get (staging-rows-by-source-id) 100)]+ (is (nil? (:target_id row))))+ (finally (p/clear-staging-tables!)))))++(deftest resolve-handles-nil-schema-test+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {target-id :id} {:db_id db-id :schema nil :name "raw_table"}]+ (try+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "raw_table" :schema nil)])+ (p/resolve-target-table-ids-in-staging!)+ (let [row (get (staging-rows-by-source-id) 100)]+ (is (= target-id (:target_id row))+ "NULL-safe match: staging schema=NULL ⇔ target schema=NULL"))+ (finally (p/clear-staging-tables!)))))++(deftest resolve-skips-inactive-target-test+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {} {:db_id db-id :schema "PUBLIC" :name "users" :active false}]+ (try+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])+ (p/resolve-target-table-ids-in-staging!)+ (let [row (get (staging-rows-by-source-id) 100)]+ (is (nil? (:target_id row))+ "inactive target rows are intentionally not resolved — re-import creates a fresh active row"))+ (finally (p/clear-staging-tables!)))))++(deftest resolve-skips-defective-duplicate-target-test+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {} {:db_id db-id :schema "PUBLIC" :name "users" :is_defective_duplicate true}]+ (try+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])+ (p/resolve-target-table-ids-in-staging!)+ (let [row (get (staging-rows-by-source-id) 100)]+ (is (nil? (:target_id row))))+ (finally (p/clear-staging-tables!)))))++(deftest resolve-is-idempotent-test+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"}]+ (try+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])+ (p/resolve-target-table-ids-in-staging!)+ (p/resolve-target-table-ids-in-staging!)+ (let [row (get (staging-rows-by-source-id) 100)]+ (is (= target-id (:target_id row))))+ (finally (p/clear-staging-tables!)))))++(deftest resolve-tolerates-duplicate-natural-key-target-test+ (testing "If two live (db_name, schema, name) candidates exist — possible+ because `metabase_database`'s unique constraint is+ `(router_database_id, name)` and NULL routers don't collide —+ the resolve subquery picks a stable tiebreaker (MIN(id)) rather+ than throwing `subquery returned more than one row`. Regression+ for GHY-3549."+ (mt/with-temp [:model/Database {db-id-1 :id db-name :name} {:engine :h2}+ :model/Database {db-id-2 :id} {:engine :h2}]+ ;; Make the second Database share the first's name; mt/with-temp gives+ ;; each a unique name but PG's unique constraint is `(router_database_id,+ ;; name)`, so an UPDATE that sets a duplicate (NULL, name) is permitted.+ (t2/query {:update :metabase_database :set {:name db-name} :where [:= :id db-id-2]})+ (let [t1-id (:id (t2/insert-returning-instance!+ :model/Table {:db_id db-id-1 :schema "PUBLIC" :name "users"+ :display_name "Users" :active true}))+ t2-id (:id (t2/insert-returning-instance!+ :model/Table {:db_id db-id-2 :schema "PUBLIC" :name "users"+ :display_name "Users" :active true}))]+ (try+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])+ ;; The bug pre-fix: resolve throws "subquery returned more than one row".+ ;; With the fix: resolve completes and target_id is set to MIN(id).+ (p/resolve-target-table-ids-in-staging!)+ (let [row (get (staging-rows-by-source-id) 100)]+ (is (= (min t1-id t2-id) (:target_id row))+ "MIN(id) tiebreaker: target_id is the smaller of the two candidates"))+ (finally+ (p/clear-staging-tables!)))))))++;;; ============================== merge-tables ==============================++(deftest merge-empty-staging-is-noop-test+ (try+ (p/clear-staging-tables!)+ (let [before (t2/count :metabase_table)]+ (p/merge-tables!)+ (is (= before (t2/count :metabase_table))))+ (finally (p/clear-staging-tables!))))++(deftest merge-inserts-unmatched-table-test+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}]+ (try+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "new_table"+ :description "first import")])+ (p/resolve-target-table-ids-in-staging!)+ (p/merge-tables!)+ (let [inserted (t2/select-one :model/Table :db_id db-id :name "new_table")]+ (is (some? inserted))+ (is (= "PUBLIC" (:schema inserted)))+ (is (= "first import" (:description inserted)))+ (is (true? (:active inserted)))+ (is (= "internal" (name (:data_layer inserted)))))+ (finally+ (t2/delete! :model/Table :db_id db-id)+ (p/clear-staging-tables!)))))++(deftest merge-updates-description-on-match-test+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"+ :description "old description"}]+ (try+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users"+ :description "new description")])+ (p/resolve-target-table-ids-in-staging!)+ (p/merge-tables!)+ (is (= "new description" (:description (t2/select-one :model/Table :id target-id))))+ (finally (p/clear-staging-tables!)))))++(deftest merge-skips-row-whose-db-has-no-target-test+ (try+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 "nonexistent_db" "ghost_table")])+ (p/resolve-target-table-ids-in-staging!)+ (let [before (t2/count :model/Table :name "ghost_table")]+ (p/merge-tables!)+ (is (= before (t2/count :model/Table :name "ghost_table"))+ "INSERT JOINs metabase_database on db_name; no match → row silently skipped"))+ (finally (p/clear-staging-tables!))))++(deftest merge-handles-nil-schema-test+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}]+ (try+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "raw_table" :schema nil)])+ (p/resolve-target-table-ids-in-staging!)+ (p/merge-tables!)+ (let [inserted (t2/select-one :model/Table :db_id db-id :name "raw_table")]+ (is (some? inserted))+ (is (nil? (:schema inserted))))+ (finally+ (t2/delete! :model/Table :db_id db-id)+ (p/clear-staging-tables!)))))++(deftest merge-is-idempotent-test+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}]+ (try+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users" :description "v1")])+ (p/resolve-target-table-ids-in-staging!)+ (p/merge-tables!)+ (let [first-id (:id (t2/select-one :model/Table :db_id db-id :name "users"))]+ ;; second pass — re-resolve and re-merge against the same staging contents+ (p/resolve-target-table-ids-in-staging!)+ (p/merge-tables!)+ (let [second-id (:id (t2/select-one :model/Table :db_id db-id :name "users"))]+ (is (= first-id second-id) "re-import doesn't insert a duplicate")+ (is (= 1 (t2/count :model/Table :db_id db-id :name "users")))))+ (finally+ (t2/delete! :model/Table :db_id db-id)+ (p/clear-staging-tables!)))))++(deftest merge-skip-if-unchanged-test+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"+ :description "same"}]+ (try+ ;; Stamp updated_at to a known-old value so we can detect if the UPDATE fires.+ (t2/query {:update :metabase_table+ :set {:updated_at #t "2020-01-01T00:00:00Z"}+ :where [:= :id target-id]})+ (let [original-updated-at (:updated_at (t2/select-one :model/Table :id target-id))]+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users" :description "same")])+ (p/resolve-target-table-ids-in-staging!)+ (p/merge-tables!)+ (let [after-updated-at (:updated_at (t2/select-one :model/Table :id target-id))]+ (is (= original-updated-at after-updated-at)+ "skip-if-unchanged: identical description means no UPDATE fires, updated_at unchanged")))+ (finally (p/clear-staging-tables!)))))++(deftest merge-fires-when-description-differs-test+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"+ :description "v1"}]+ (try+ (t2/query {:update :metabase_table+ :set {:updated_at #t "2020-01-01T00:00:00Z"}+ :where [:= :id target-id]})+ (let [original-updated-at (:updated_at (t2/select-one :model/Table :id target-id))]+ (p/clear-staging-tables!)+ (t2/insert! :metabase_table_import [(staging-row 100 db-name "users" :description "v2")])+ (p/resolve-target-table-ids-in-staging!)+ (p/merge-tables!)+ (let [after (t2/select-one :model/Table :id target-id)]+ (is (= "v2" (:description after)))+ (is (not= original-updated-at (:updated_at after))+ "description differs → UPDATE fires → updated_at bumped")))+ (finally (p/clear-staging-tables!)))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/wire_format_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/wire_format_test.cljnew file mode 100644index 00000000000..661b97ef9b5--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/wire_format_test.clj@@ -0,0 +1,132 @@+(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.wire-format-test+ "Contract test between the metadata export and import sides. The importer's+ Malli schemas describe what the importer expects on the wire; this test+ exercises the export pipeline against a small hand-built fixture and+ validates every emitted row against the corresponding import schema. If the+ export starts emitting a column the import schema doesn't know about — or+ stops emitting a required one — the test fails.++ NOT `^:parallel`: each test does an `mt/with-temp [:model/Database :model/Table+ :model/Field …]` whose `:after-insert` chain hits `metabase_data_permissions`+ + the cluster lock. Concurrent tests in this ns reliably deadlock on MariaDB."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.export :as export]+ [metabase-enterprise.serialization.metadata-file-import.schemas :as schemas]+ [metabase.test :as mt]+ [metabase.util.json :as json]+ [metabase.util.malli.registry :as mr])+ (:import+ (java.io ByteArrayOutputStream)))++(set! *warn-on-reflection* true)++(use-fixtures :once+ (fn [thunk]+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.+ (mt/db)+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (thunk))))++(defn- export-as-json!+ "Run the export against `opts`, decode JSON to keyword-keyed maps."+ [opts]+ (let [bos (ByteArrayOutputStream.)]+ (export/export-metadata! bos opts)+ (json/decode+kw (.toString bos "UTF-8"))))++(defmacro ^:private with-tiny-fixture!+ "Bind a hand-built fixture covering each wire-format field shape:+ - flat root field (no parent, no nfc_path, no FK)+ - Dictionary parent (has children pointing at it)+ - nested leaf (parent_id + nfc_path)+ - unfolded leaf (nfc_path only, no parent_id)+ - FK source (fk_target_field_id pointing at another table's field)+ - FK target (referenced from another field via fk_target_field_id)++ Yields `[db-id table-1-id table-2-id]` in scope of `body`."+ [bindings & body]+ (let [[db table1 table2] bindings]+ `(mt/with-temp+ [:model/Database {~db :id} {:engine :h2}+ :model/Table {~table1 :id} {:db_id ~db :schema "PUBLIC" :name "users"}+ :model/Table {~table2 :id} {:db_id ~db :schema "PUBLIC" :name "accounts"}+ :model/Field _flat# {:table_id ~table1 :name "user_id" :base_type :type/Integer}+ :model/Field {parent-id# :id} {:table_id ~table1 :name "data" :base_type :type/Dictionary}+ :model/Field _nested-leaf# {:table_id ~table1 :name "value"+ :base_type :type/Text+ :parent_id parent-id#+ :nfc_path ["data" "value"]}+ :model/Field _unfolded-leaf# {:table_id ~table1 :name "embedded → leaf"+ :base_type :type/Text+ :nfc_path ["embedded" "leaf"]}+ :model/Field {target-id# :id} {:table_id ~table2 :name "id" :base_type :type/Integer}+ :model/Field _fk-source# {:table_id ~table1+ :name "account_fk"+ :base_type :type/Integer+ :fk_target_field_id target-id#}]+ ~@body)))++(defn- export-fixture!+ "Run the export scoped to the fixture's database id."+ [db-id]+ (export-as-json! {:user-info {:user-id (mt/user->id :crowberto) :is-superuser? true}+ :with-databases true+ :with-tables true+ :with-fields true+ :database-ids [db-id]}))++;;; ============================== Per-section tests ==============================++(deftest database-rows-validate-against-import-schema-test+ (mt/with-premium-features #{:serialization}+ (with-tiny-fixture! [db-id _t1 _t2]+ (let [{:keys [databases]} (export-fixture! db-id)]+ (testing "fixture surfaces at least one database row"+ (is (= 1 (count databases))))+ (doseq [row databases]+ (testing (str "row validates against ::schemas/database-info — " (pr-str row))+ (is (mr/validate ::schemas/database-info row))))))))++(deftest table-rows-validate-against-import-schema-test+ (mt/with-premium-features #{:serialization}+ (with-tiny-fixture! [db-id _t1 _t2]+ (let [{:keys [tables]} (export-fixture! db-id)]+ (testing "fixture surfaces both tables"+ (is (= 2 (count tables))))+ (doseq [row tables]+ (testing (str "row validates against ::schemas/table-info — " (pr-str row))+ (is (mr/validate ::schemas/table-info row))))))))++(deftest field-rows-validate-against-import-schema-test+ (mt/with-premium-features #{:serialization}+ (with-tiny-fixture! [db-id _t1 _t2]+ (let [{:keys [fields]} (export-fixture! db-id)]+ (testing "fixture surfaces all six field shapes"+ (is (= 6 (count fields))))+ (doseq [row fields]+ (testing (str "row validates against ::schemas/field-info — " (pr-str row))+ (is (mr/validate ::schemas/field-info row))))))))++;;; ============================== Per-shape coverage assertions ==============================+;;; Belt-and-suspenders: assert the fixture actually exercised each variant.+;;; If the fixture stops producing a particular shape (e.g., a refactor of+;;; mt/with-temp's defaults), the per-section tests still pass, but these+;;; coverage tests catch the gap.++(deftest field-shape-coverage-test+ (mt/with-premium-features #{:serialization}+ (with-tiny-fixture! [db-id _t1 _t2]+ (let [{:keys [fields]} (export-fixture! db-id)]+ (testing "exactly one nested leaf (parent_id + nfc_path)"+ (is (= 1 (count (filter #(and (:parent_id %) (:nfc_path %)) fields)))))+ (testing "exactly one unfolded leaf (nfc_path without parent_id)"+ (is (= 1 (count (filter #(and (:nfc_path %) (not (:parent_id %))) fields)))))+ (testing "exactly one FK source (fk_target_field_id present)"+ (is (= 1 (count (filter :fk_target_field_id fields)))))+ (testing "at least one flat field (no parent_id, no nfc_path, no FK)"+ (is (pos? (count (filter #(and (not (:parent_id %))+ (not (:nfc_path %))+ (not (:fk_target_field_id %)))+ fields)))))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import_test.cljnew file mode 100644index 00000000000..e7ce33b978e--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import_test.clj@@ -0,0 +1,256 @@+(ns metabase-enterprise.serialization.metadata-file-import-test+ "End-to-end orchestrator tests for [[metabase-enterprise.serialization.metadata-file-import/import-metadata-file!]].+ Each test writes a temp JSON file in the wire format, sets up matching+ `mt/with-temp` Database/Table/Field rows where needed, runs the loader, and+ asserts on the appdb state. See+ [[metabase-enterprise.serialization.metadata-file-import.schemas]] for the+ wire-format primer."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.metadata-file-import :as loader]+ [metabase-enterprise.serialization.metadata-file-import.processors :as processors]+ [metabase.test :as mt]+ [metabase.util.json :as json]+ [metabase.warehouse-schema.models.table :as schema.table]+ [toucan2.core :as t2])+ (:import+ (java.io File)))++(set! *warn-on-reflection* true)++;; NOT ^:parallel — test-helpers-set-global-values! asserts non-parallel.+;; See note in concurrency_test.clj fixture — intentional opt-out from the+;; parallel-safe contract that kondo's `validate-deftest` enforces.+#_{:clj-kondo/ignore [:metabase/validate-deftest]}+(use-fixtures :once+ (fn [thunk]+ ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with+ ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.+ (mt/db)+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (mt/test-helpers-set-global-values!+ (thunk)))))++;;; ============================== Test helpers ==============================++(defn- temp-file ^File [suffix content]+ (let [f (File/createTempFile "loader-test-" suffix)]+ (.deleteOnExit f)+ (spit f content)+ f))++(defn- json-file ^File [data]+ (temp-file ".json" (json/encode data)))++;;; ============================== Happy paths ==============================++(deftest end-to-end-happy-path-with-json-file-test+ (testing "given a target Database that matches by (name, engine) and a metadata file+ describing one table with one root field and one nested field, the loader+ inserts the missing rows and updates initial_sync_status on the matched DB"+ (mt/with-temp [:model/Database {tgt-db :id} {:name "happy-path-db" :engine :postgres+ :initial_sync_status "incomplete"}]+ (let [meta-file (json-file+ {:databases [{:id 7 :name "happy-path-db" :engine "postgres"}]+ :tables [{:id 100 :db_id 7 :schema "public" :name "orders"}]+ :fields [{:id 1000 :table_id 100 :name "id"+ :base_type "type/Integer" :database_type "integer"}+ {:id 1001 :table_id 100 :name "address"+ :base_type "type/Structured" :database_type "json"}+ {:id 1002 :table_id 100 :name "zip"+ :parent_id 1001 :nfc_path ["address"]+ :base_type "type/Text" :database_type "text"}]})]+ (loader/import-metadata-file! meta-file)+ (testing "the matched Database's initial_sync_status was flipped to complete"+ (is (= "complete" (:initial_sync_status (t2/select-one :model/Database :id tgt-db)))))+ (testing "the table was inserted, attached to the target DB"+ (let [tbl (t2/select-one :model/Table :db_id tgt-db :name "orders")]+ (is (some? tbl))+ (is (= "public" (:schema tbl)))))+ (testing "the root and nested fields were inserted with correct parent linkage"+ (let [tbl-id (:id (t2/select-one :model/Table :db_id tgt-db :name "orders"))+ root (t2/select-one :model/Field :table_id tbl-id :name "id")+ addr (t2/select-one :model/Field :table_id tbl-id :name "address")+ zip (t2/select-one :model/Field :table_id tbl-id :name "zip")]+ (is (some? root))+ (is (some? addr))+ (is (some? zip))+ (is (nil? (:parent_id root)))+ (is (nil? (:parent_id addr)))+ (is (= (:id addr) (:parent_id zip))+ "the nested field's parent_id is the address field's target id")+ (is (= ["address"] (:nfc_path zip)))))))))++;;; ============================== No-match skipping ==============================++(deftest unmatched-source-database-skips-its-tables-and-fields-test+ (testing "a source database with no (name, engine) match in target: WARN logged,+ its tables/fields are silently dropped during merge"+ (let [meta-file (json-file+ {:databases [{:id 7 :name "no-such-db" :engine "postgres"}]+ :tables [{:id 100 :db_id 7 :schema "public" :name "ghost"}]+ :fields [{:id 1000 :table_id 100 :name "x"+ :base_type "type/Integer" :database_type "int"}]})]+ (loader/import-metadata-file! meta-file)+ (is (zero? (t2/count :model/Table :name "ghost"))+ "no metabase_table row was created for the unmatched-DB's table")+ (is (zero? (t2/count :model/Field :name "x"))+ "no metabase_field row either"))))++;;; ============================== FK resolution ==============================++(deftest fk-target-field-id-resolved-after-import-test+ (testing "two tables, one with a field that has fk_target_field_id pointing at the+ other table's id field — after import, target appdb's fk_target_field_id+ points at the correctly-resolved target field id"+ (mt/with-temp [:model/Database {tgt-db :id} {:name "fk-test-db" :engine :postgres}]+ (let [meta-file (json-file+ {:databases [{:id 7 :name "fk-test-db" :engine "postgres"}]+ :tables [{:id 100 :db_id 7 :schema "public" :name "users"}+ {:id 101 :db_id 7 :schema "public" :name "orders"}]+ :fields [{:id 1000 :table_id 100 :name "id"+ :base_type "type/Integer" :database_type "integer"}+ {:id 1001 :table_id 101 :name "user_id"+ :base_type "type/Integer" :database_type "integer"+ :fk_target_field_id 1000}]})]+ (loader/import-metadata-file! meta-file)+ (let [users-tbl (:id (t2/select-one :model/Table :db_id tgt-db :name "users"))+ orders-tbl (:id (t2/select-one :model/Table :db_id tgt-db :name "orders"))+ users-id (:id (t2/select-one :model/Field :table_id users-tbl :name "id"))+ orders-fk (t2/select-one :model/Field :table_id orders-tbl :name "user_id")]+ (is (= users-id (:fk_target_field_id orders-fk))+ "orders.user_id's fk_target_field_id is users.id's resolved target id"))))))++;;; ============================== Per-table default permissions ==============================++(deftest newly-imported-tables-get-default-permissions-test+ (testing "Tables inserted by the importer flow through `set-new-tables-permissions!`+ in batch, mirroring what the `:after-insert` hook on `:model/Table`+ would do for single-row inserts (just in one call instead of N).+ Verified via a spy on the bulk grant function — asserting which+ rows actually land in `data_permissions` would couple the test to+ permission-graph semantics (db-level perms cover new tables ⇒ no+ table-level rows), which isn't what we're checking here."+ (mt/with-temp [:model/Database {tgt-db :id} {:name "perms-bulk-db" :engine :postgres}]+ (let [meta-file (json-file+ {:databases [{:id 7 :name "perms-bulk-db" :engine "postgres"}]+ :tables [{:id 100 :db_id 7 :schema "public" :name "alpha"}+ {:id 101 :db_id 7 :schema "public" :name "beta"}]+ :fields [{:id 1000 :table_id 100 :name "id"+ :base_type "type/Integer" :database_type "integer"}+ {:id 1001 :table_id 101 :name "id"+ :base_type "type/Integer" :database_type "integer"}]})+ spy (atom [])+ real-bulk @#'schema.table/set-new-tables-permissions!]+ (with-redefs [schema.table/set-new-tables-permissions!+ (fn [db-id rows]+ (swap! spy conj {:db-id db-id :rows rows})+ (real-bulk db-id rows))]+ (loader/import-metadata-file! meta-file))+ (let [calls @spy]+ (is (= 1 (count calls)) "set-new-tables-permissions! fired exactly once (per db_id)")+ (let [{:keys [db-id rows]} (first calls)]+ (is (= tgt-db db-id) "called with the target db_id")+ (is (= 2 (count rows)) "passed both newly-inserted tables")+ (is (= #{"alpha" "beta"}+ (set (map :name+ (t2/select [:model/Table :name] :id [:in (map :id rows)]))))+ "rows correspond to alpha and beta")))))))++;;; ============================== Idempotence ==============================++(deftest re-import-is-idempotent-test+ (testing "running the same file twice produces the same final appdb state+ — no duplicate rows, no error"+ (mt/with-temp [:model/Database {tgt-db :id} {:name "idem-db" :engine :postgres}]+ (let [meta-file (json-file+ {:databases [{:id 7 :name "idem-db" :engine "postgres"}]+ :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]+ :fields [{:id 1000 :table_id 100 :name "x"+ :base_type "type/Integer" :database_type "int"+ :description "v1"}]})]+ (loader/import-metadata-file! meta-file)+ (loader/import-metadata-file! meta-file)+ (let [tbl-id (:id (t2/select-one :model/Table :db_id tgt-db :name "t"))]+ (is (= 1 (t2/count :model/Table :db_id tgt-db :name "t")))+ (is (= 1 (t2/count :model/Field :table_id tbl-id :name "x")))+ (is (= "v1" (:description (t2/select-one :model/Field :table_id tbl-id :name "x")))))))))++;;; ============================== Pre-flight orphan bail ==============================++(deftest pre-flight-orphan-bail-test+ (testing "a file whose field references a parent_id not in the same file is rejected+ with :file-incomplete; no live writes happen"+ (mt/with-temp [:model/Database {} {:name "orphan-bail-db" :engine :postgres}]+ (let [meta-file (json-file+ {:databases [{:id 7 :name "orphan-bail-db" :engine "postgres"}]+ :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]+ :fields [{:id 1000 :table_id 100 :name "x"+ :base_type "type/Integer" :database_type "int"+ :parent_id 9999}]}) ; bad ref — no such source_id in file+ thrown (try (loader/import-metadata-file! meta-file) nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (some? thrown))+ (is (= :file-incomplete (:kind (ex-data thrown))))+ (is (zero? (t2/count :model/Table :name "t"))+ "no live writes happened — pre-flight ran before any txn")))))++(deftest cycle-bail-test+ (testing "a file with a cycle in source_parent_id is rejected with+ :cycle-in-field-graph; no live writes happen"+ (mt/with-temp [:model/Database {} {:name "cycle-db" :engine :postgres}]+ (let [meta-file (json-file+ {:databases [{:id 7 :name "cycle-db" :engine "postgres"}]+ :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]+ :fields [{:id 1000 :table_id 100 :name "a"+ :base_type "type/Integer" :database_type "int"+ :parent_id 1001}+ {:id 1001 :table_id 100 :name "b"+ :base_type "type/Integer" :database_type "int"+ :parent_id 1000}]})+ thrown (try (loader/import-metadata-file! meta-file) nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (some? thrown))+ (is (= :cycle-in-field-graph (:kind (ex-data thrown))))+ (is (zero? (t2/count :model/Table :name "t")))))))++;;; ============================== Atomic rollback ==============================++(deftest atomic-rollback-on-mid-merge-throw-test+ (testing "if anything inside the merge transaction throws, all live writes are rolled+ back — neither tables nor fields land in the appdb"+ (mt/with-temp [:model/Database {tgt-db :id} {:name "rollback-db" :engine :postgres}]+ (let [meta-file (json-file+ {:databases [{:id 7 :name "rollback-db" :engine "postgres"}]+ :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]+ :fields [{:id 1000 :table_id 100 :name "x"+ :base_type "type/Integer" :database_type "int"}]})]+ ;; Inject a throw into merge-fields-by-depth! — runs after merge-tables! has+ ;; written to metabase_table.+ (with-redefs [processors/merge-fields-by-depth!+ (fn [] (throw (ex-info "boom mid-merge" {:kind ::injected})))]+ (let [thrown (try (loader/import-metadata-file! meta-file) nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (some? thrown))+ (is (= ::injected (:kind (ex-data thrown))))))+ (is (zero? (t2/count :model/Table :db_id tgt-db :name "t"))+ "the table that merge-tables! 'inserted' was rolled back")))))++;;; ============================== field-shape coverage ==============================++(deftest unfolded-leaf-imports-correctly-test+ (testing "an unfolded leaf (nfc_path set, no parent_id) imports as a leaf field+ with nfc_path preserved and parent_id NULL"+ (mt/with-temp [:model/Database {tgt-db :id} {:name "unfolded-leaf-db" :engine :postgres}]+ (let [meta-file (json-file+ {:databases [{:id 7 :name "unfolded-leaf-db" :engine "postgres"}]+ :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]+ :fields [{:id 1000 :table_id 100 :name "data.user.zip"+ :base_type "type/Text" :database_type "text"+ :nfc_path ["data" "user" "zip"]}]})]+ (loader/import-metadata-file! meta-file)+ (let [tbl-id (:id (t2/select-one :model/Table :db_id tgt-db :name "t"))+ leaf (t2/select-one :model/Field :table_id tbl-id :name "data.user.zip")]+ (is (some? leaf))+ (is (nil? (:parent_id leaf)))+ (is (= ["data" "user" "zip"] (:nfc_path leaf))))))))diff --git a/resources/migrations/062/20260506_metadata_import_staging_tables.yaml b/resources/migrations/062/20260506_metadata_import_staging_tables.yamlnew file mode 100644index 00000000000..72f606c7ebc--- /dev/null+++ b/resources/migrations/062/20260506_metadata_import_staging_tables.yaml@@ -0,0 +1,235 @@+## Staging tables for the metadata file importer.+##+## Two scratch tables drained into during an import, then merged into+## metabase_table / metabase_field inside a single short transaction so+## live-data writes are atomic. Both tables live permanently in the+## schema; the importer wipes them on entry and exit (try/finally) so a+## crashed prior attempt cannot leak rows into the next run.+##+## Wire format uses source-side integer IDs (the source appdb's+## metabase_database / metabase_table / metabase_field surrogate keys).+## They are internally consistent within one file and meaningless across+## files. Staging carries `source_*_id` columns for the wire identity+## and `target_*_id` columns populated by the resolve step against the+## live appdb.+##+## Indexes:+## - PK on `source_id` — probed by every depth-walk EXISTS subquery+## - `target_id` — keys the merge UPDATE's clobber subqueries+## - `depth` — per-level scoping for the depth-walk merge++databaseChangeLog:+ - logicalFilePath: migrations/062_update_migrations.yaml++ - changeSet:+ id: v62.2026-05-08T00:00:00+ author: galdre+ comment: Create metabase_table_import staging table+ changes:+ - createTable:+ tableName: metabase_table_import+ remarks: Staging table for the metadata file importer's tables phase. Drained outside the merge txn, merged into metabase_table inside it.+ columns:+ - column:+ name: source_id+ type: int+ constraints:+ nullable: false+ primaryKey: true+ primaryKeyName: pk_metabase_table_import+ remarks: Wire :id — source appdb's metabase_table.id. Unique within one file.+ - column:+ name: source_db_id+ type: int+ constraints:+ nullable: false+ remarks: Wire :db_id — references the source-side database row in :databases[].+ - column:+ name: db_name+ type: varchar(254)+ constraints:+ nullable: false+ remarks: Denormalized from :databases[] at drain time, used for natural-key match against target.+ - column:+ name: schema+ type: varchar(254)+ constraints:+ nullable: true+ remarks: Mirrors metabase_table.schema nullability.+ - column:+ name: name+ type: varchar(254)+ constraints:+ nullable: false+ remarks: Wire :name — table's natural-key name component.+ - column:+ name: description+ type: ${text.type}+ constraints:+ nullable: true+ remarks: Wire :description — clobbered onto metabase_table.description on merge.+ - column:+ name: display_name+ type: varchar(254)+ constraints:+ nullable: true+ remarks: Pre-humanized at drain time so the merge SQL doesn't replicate the algorithm in three dialects.+ - column:+ name: target_id+ type: int+ constraints:+ nullable: true+ remarks: Resolved by table-resolve step; → metabase_table.id. NULL means no live row matches; row is an insert candidate.++ - changeSet:+ id: v62.2026-05-08T00:00:00.1+ author: galdre+ comment: Index metabase_table_import.target_id for the merge-tables clobber subqueries+ changes:+ - createIndex:+ tableName: metabase_table_import+ indexName: idx_mti_target_id+ columns:+ - column:+ name: target_id++ - changeSet:+ id: v62.2026-05-08T00:00:01+ author: galdre+ comment: Create metabase_field_import staging table+ changes:+ - createTable:+ tableName: metabase_field_import+ remarks: Staging table for the metadata file importer's fields phase. Drained outside the merge txn, depth-tagged, then merged into metabase_field level by level inside the txn.+ columns:+ # --- wire identity ---+ - column:+ name: source_id+ type: int+ constraints:+ nullable: false+ primaryKey: true+ primaryKeyName: pk_metabase_field_import+ remarks: Wire :id — source appdb's metabase_field.id. Unique within one file.+ - column:+ name: source_table_id+ type: int+ constraints:+ nullable: false+ remarks: Wire :table_id — references metabase_table_import.source_id within the same file.+ - column:+ name: source_parent_id+ type: int+ constraints:+ nullable: true+ remarks: Wire :parent_id — references another metabase_field_import.source_id within the same file. NULL for top-level fields.+ - column:+ name: source_fk_target_id+ type: int+ constraints:+ nullable: true+ remarks: Wire :fk_target_field_id — references another metabase_field_import.source_id. NULL for non-FK fields.+ # --- clobber payload ---+ - column:+ name: name+ type: varchar(254)+ constraints:+ nullable: false+ remarks: Wire :name — field's natural-key name component (within its parent + table).+ - column:+ name: base_type+ type: varchar(254)+ constraints:+ nullable: false+ remarks: Wire :base_type — clobbered onto metabase_field.base_type on merge.+ - column:+ name: database_type+ type: varchar(254)+ constraints:+ nullable: false+ remarks: Wire :database_type — clobbered onto metabase_field.database_type on merge.+ - column:+ name: effective_type+ type: varchar(254)+ constraints:+ nullable: true+ remarks: NULL when wire row omitted it (≡ base_type). Importer treats NULL as base_type on INSERT; leaves target alone on UPDATE.+ - column:+ name: semantic_type+ type: varchar(254)+ constraints:+ nullable: true+ remarks: Wire :semantic_type — clobbered onto metabase_field.semantic_type on merge.+ - column:+ name: coercion_strategy+ type: varchar(254)+ constraints:+ nullable: true+ remarks: Wire :coercion_strategy — clobbered onto metabase_field.coercion_strategy on merge.+ - column:+ name: description+ type: ${text.type}+ constraints:+ nullable: true+ remarks: Wire :description — clobbered onto metabase_field.description on merge.+ - column:+ name: nfc_path+ type: ${text.type}+ constraints:+ nullable: true+ remarks: NULL for empty/absent paths, JSON-encoded array string otherwise — same encoding as metabase_field.nfc_path.+ # --- resolve outputs ---+ - column:+ name: depth+ type: int+ constraints:+ nullable: true+ remarks: Populated once after drain by the depth-tagging step. 0 = root (no parent, no fk_target). Higher = max(parent depth, fk_target depth) + 1. NULL after depth-tagging means a cycle in the file's reference graph.+ - column:+ name: target_id+ type: int+ constraints:+ nullable: true+ remarks: Resolved per-depth-level by natural-key match against metabase_field. NULL means no live row matches; row is an insert candidate at this depth.+ - column:+ name: target_table_id+ type: int+ constraints:+ nullable: true+ remarks: Populated from the table-resolve step before the depth walk starts. Joined-through from metabase_table_import.target_id.+ - column:+ name: target_parent_id+ type: int+ constraints:+ nullable: true+ remarks: Populated per-depth-level by self-join on source_parent_id → another row's target_id.+ - column:+ name: target_fk_target_id+ type: int+ constraints:+ nullable: true+ remarks: Populated per-depth-level by self-join on source_fk_target_id → another row's target_id.++ - changeSet:+ id: v62.2026-05-08T00:00:01.1+ author: galdre+ comment: Index metabase_field_import.depth for per-level scoping in the depth-walk merge+ changes:+ - createIndex:+ tableName: metabase_field_import+ indexName: idx_mfi_depth+ columns:+ - column:+ name: depth++ - changeSet:+ id: v62.2026-05-08T00:00:01.2+ author: galdre+ comment: Index metabase_field_import.target_id for the merge-fields clobber subqueries+ changes:+ - createIndex:+ tableName: metabase_field_import+ indexName: idx_mfi_target_id+ columns:+ - column:+ name: target_iddiff --git a/src/metabase/permissions/core.clj b/src/metabase/permissions/core.cljindex 6952ac06c48..1b770465944 100644--- a/src/metabase/permissions/core.clj+++ b/src/metabase/permissions/core.clj@@ -70,6 +70,7 @@set-default-group-permissions!set-default-database-permissions!set-default-table-permissions!+ set-default-table-permissions-bulk!with-global-permissions-lockwith-db-scoped-permissions-lock][metabase.permissions.models.data-permissions.sqldiff --git a/src/metabase/permissions/models/data_permissions.clj b/src/metabase/permissions/models/data_permissions.cljindex 7d3b9ad30a1..f1ef476939b 100644--- a/src/metabase/permissions/models/data_permissions.clj+++ b/src/metabase/permissions/models/data_permissions.clj@@ -1123,109 +1123,112 @@groups)](batch-insert-permissions! perm-rows))))-(defn set-default-table-permissions!- "Bulk-sets default permissions for a newly-created table across all relevant groups.- Handles three cases per (group, perm-type):- - Group has DB-level perm matching default → no-op (DB-level covers it)- - Group has DB-level perm but new table needs :blocked → going-granular (expand to per-table rows)- - Group has no DB-level perm (already table-granular) → simple insert for the new table+(defn- mk-perm-row [group-id perm-type perm-value db-id table-id schema]+ {:perm_type perm-type :group_id group-id :perm_value perm-value+ :db_id db-id :table_id table-id :schema_name schema})++(defn- load-perm-context+ "Load the introspection state needed to classify each `(group, perm-type)`+ for new tables on `db-id`."+ [db-id group-ids perm-types]+ (let [qn (mapv u/qualified-name perm-types)+ db-level (t2/select :model/DataPermissions+ {:where [:and [:= :db_id db-id] [:= :table_id nil]+ [:in :group_id group-ids] [:in :perm_type qn]]})+ table-level (t2/select :model/DataPermissions+ {:where [:and [:= :db_id db-id] [:not= :table_id nil]+ [:in :group_id group-ids] [:in :perm_type qn]]})]+ {:db-id db-id+ :db-level-idx (into {} (map (juxt (juxt :group_id :perm_type) identity)) db-level)+ :schema-vals-idx (reduce (fn [acc {:keys [group_id perm_type schema_name perm_value]}]+ (update-in acc [group_id perm_type schema_name] (fnil conj #{}) perm_value))+ {} table-level)+ :all-db-tables (t2/select [:model/Table :id :db_id :schema] :db_id db-id :active true)+ :view-data-levels (new-table-view-data-permission-levels db-id group-ids)}))++(defn- compute-actual-value+ "Per-entry resolution: enterprise view-data override, then schema-consistency+ if all existing tables in the schema agree, else the caller's default."+ [{:keys [view-data-levels schema-vals-idx]}+ {:keys [group-id perm-type default-value table]}]+ (or (when (= perm-type :perms/view-data)+ (get view-data-levels group-id))+ (let [sv (get-in schema-vals-idx [group-id perm-type (:schema table)])]+ (when (and (seq sv) (= (count sv) 1))+ (first sv)))+ default-value))++(defn- classify-key+ "For one `(group-id, perm-type)`, return `{:deletes [id?] :rows [perm-row...]}`.+ Three branches mirror the original cond: going-granular, simple-insert, no-op."+ [{:keys [db-id db-level-idx all-db-tables batch-table-ids]}+ [[group-id perm-type] entries]]+ (let [db-perm (get db-level-idx [group-id perm-type])+ any-blocked? (and db-perm (some #(= :blocked (:actual-value %)) entries))+ mk (fn [v table-id schema]+ (mk-perm-row group-id perm-type v db-id table-id schema))]+ (cond+ ;; Going-granular fires once for the whole `(group, perm-type)`:+ ;; delete the DB-level row, expand non-batch tables to the old value,+ ;; then write each batch table at its actual-value.+ any-blocked?+ (let [expansion-value (case (:perm_value db-perm)+ :query-builder-and-native :query-builder+ (:perm_value db-perm))]+ {:deletes [(:id db-perm)]+ :rows (concat+ (for [t all-db-tables :when (not (contains? batch-table-ids (:id t)))]+ (mk expansion-value (:id t) (:schema t)))+ (for [{:keys [actual-value table]} entries]+ (mk actual-value (u/the-id table) (:schema table))))})+ (nil? db-perm)+ {:rows (for [{:keys [actual-value table]} entries]+ (mk actual-value (u/the-id table) (:schema table)))}+ ;; DB-level covers the batch tables — no-op.+ :else nil)))++(defn set-default-table-permissions-bulk!+ "Bulk-set default permissions for many newly-created tables on the same+ database. Issues the introspection SELECTs once for the whole batch and+ a single DELETE + INSERT, deduping going-granular triggers across the+ batch.++ Caller must hold [[with-db-scoped-permissions-lock]] for `db-id`.+ `tables+defaults` is a seq of `[table group-perm-defaults]` pairs."+ [db-id tables+defaults]+ (when (seq tables+defaults)+ (let [batch-table-ids (set (map (fn [[t _]] (u/the-id t)) tables+defaults))+ all-defaults (mapcat (fn [[t defs]] (map #(assoc % :table t) defs))+ tables+defaults)+ group-ids (distinct (map :group-id all-defaults))+ perm-types (distinct (map :perm-type all-defaults))+ ctx (assoc (load-perm-context db-id group-ids perm-types)+ :batch-table-ids batch-table-ids)+ annotated (mapv (fn [d] (assoc d :actual-value (compute-actual-value ctx d)))+ all-defaults)+ results (->> annotated+ (group-by (juxt :group-id :perm-type))+ (keep #(classify-key ctx %)))+ to-delete (mapcat :deletes results)+ to-insert (mapcat :rows results)]+ (when (seq to-delete) (batch-delete-permissions! to-delete))+ (when (seq to-insert) (batch-insert-permissions! to-insert)))))- `group-perm-defaults` is a seq of {:group-id G :perm-type PT :default-value V} triples."+(defn set-default-table-permissions!+ "Set default permissions for a newly-created table across all relevant+ groups. Handles three cases per `(group, perm-type)`:+ - Group has DB-level perm matching default → no-op+ - Group has DB-level perm but new table needs `:blocked` → going-granular+ - Group has no DB-level perm → simple insert++ `group-perm-defaults` is a seq of `{:group-id :perm-type :default-value}`+ triples. Thin wrapper over [[set-default-table-permissions-bulk!]] for+ the single-table case."[table group-perm-defaults]- (when (seq group-perm-defaults)- (let [table (if (map? table)- table- (t2/select-one [:model/Table :id :db_id :schema] :id table))- db-id (:db_id table)- table-id (u/the-id table)- schema (:schema table)- group-ids (distinct (map :group-id group-perm-defaults))- perm-types (distinct (map :perm-type group-perm-defaults))- ;; Batch SELECT #1: all DB-level permissions for this DB across all relevant groups + perm-types- db-level-perms (t2/select :model/DataPermissions- {:where [:and- [:= :db_id db-id]- [:= :table_id nil]- [:in :group_id group-ids]- [:in :perm_type (mapv u/qualified-name perm-types)]]})- ;; Index: {[group_id perm_type] → db-level-perm-row}- db-level-idx (into {} (map (fn [p] [[(:group_id p) (:perm_type p)] p]) db-level-perms))- ;; Batch SELECT #2: all existing table-level permissions for this DB- ;; (needed for schema-permission-value logic and going-granular expansion)- table-perms (t2/select :model/DataPermissions- {:where [:and- [:= :db_id db-id]- [:not= :table_id nil]- [:in :group_id group-ids]- [:in :perm_type (mapv u/qualified-name perm-types)]]})- ;; Index: {[group_id perm_type schema_name] → set of values}- schema-vals-idx (reduce (fn [acc {:keys [group_id perm_type schema_name perm_value]}]- (update-in acc [group_id perm_type schema_name] (fnil conj #{}) perm_value))- {}- table-perms)- ;; Batch SELECT #3: all tables in this DB (for going-granular expansion)- all-db-tables (t2/select [:model/Table :id :db_id :schema] :db_id db-id :active true)- ;; Batch-fetch view-data levels for all groups at once- view-data-levels (new-table-view-data-permission-levels db-id group-ids)- ;; Classify each (group, perm-type) triple- {:keys [simple-perms going-granular db-rows-to-delete]}- (reduce- (fn [acc {:keys [group-id perm-type default-value]}]- (let [;; Enterprise hook override for view-data- actual-value (or (when (= perm-type :perms/view-data)- (get view-data-levels group-id))- ;; Schema-level consistency: if all tables in schema have same value, use it- (let [sv (get-in schema-vals-idx [group-id perm-type schema])]- (when (and (seq sv) (= (count sv) 1))- (first sv)))- default-value)- db-level-perm (get db-level-idx [group-id perm-type])- new-perm {:perm_type perm-type- :group_id group-id- :perm_value actual-value- :db_id db-id- :table_id table-id- :schema_name schema}]- (cond- ;; Group has DB-level perm and new table needs :blocked → going-granular- (and db-level-perm (= actual-value :blocked))- (-> acc- (update :going-granular conj {:group-id group-id :perm-type perm-type- :new-perm new-perm :db-perm db-level-perm})- (update :db-rows-to-delete conj (:id db-level-perm)))-- ;; Group has no DB-level perm (already table-granular) → simple insert- (not db-level-perm)- (update acc :simple-perms conj new-perm)-- ;; Group has DB-level perm matching or compatible → no-op- :else- acc)))- {:simple-perms [] :going-granular [] :db-rows-to-delete []}- group-perm-defaults)]- ;; Bulk DELETE: DB-level rows that need going-granular expansion- (when (seq db-rows-to-delete)- (batch-delete-permissions! db-rows-to-delete))- ;; Bulk INSERT: expansion rows for going-granular groups + simple insert rows- (let [expansion-rows (mapcat- (fn [{:keys [group-id perm-type new-perm db-perm]}]- (let [db-perm-value (:perm_value db-perm)- ;; Build per-table rows for all existing tables (with the old DB-level value)- existing-table-rows- (keep (fn [t]- (when (not= (:id t) table-id)- {:perm_type perm-type- :group_id group-id- :perm_value (case db-perm-value- :query-builder-and-native :query-builder- db-perm-value)- :db_id db-id- :table_id (:id t)- :schema_name (:schema t)}))- all-db-tables)]- (cons new-perm existing-table-rows)))- going-granular)]- (batch-insert-permissions! (concat expansion-rows simple-perms))))))+ (let [table (if (map? table)+ table+ (t2/select-one [:model/Table :id :db_id :schema] :id table))]+ (set-default-table-permissions-bulk! (:db_id table) [[table group-perm-defaults]])))(defenterprise download-perms-level"Return the download permission for the query that the given user has. OSS returns :full"diff --git a/src/metabase/sync/analyze.clj b/src/metabase/sync/analyze.cljindex 9aa73c07eb7..b6fdb725e8c 100644--- a/src/metabase/sync/analyze.clj+++ b/src/metabase/sync/analyze.clj@@ -115,15 +115,29 @@#(sync.interestingness/score-fields-for-db! % log-fn)interestingness-summary)])+(mu/defn- analyze-db!*+ "Shared core of [[analyze-db!]] and [[analyze-db-explicit!]]: the analysis work, without the+ surrounding `*-sync-operation` wrapper (which is what applies the eligibility gating)."+ [database :- i/DatabaseInstance]+ (sync-util/with-emoji-progress-bar [emoji-progress-bar (inc (* 4 (sync-util/sync-tables-count database)))]+ (u/prog1 (sync-util/run-sync-operation "analyze" database (make-analyze-steps (maybe-log-progress emoji-progress-bar)))+ (update-fields-last-analyzed-for-db! database))))+(mu/defn analyze-db!"Perform in-depth analysis on the data for all Tables in a given `database`. This is dependent on what each databasedriver supports, but includes things like cardinality testing and table row counting. This also updates the- `:last_analyzed` value for each affected Field."+ `:last_analyzed` value for each affected Field. Subject to the `disable-auto-sync` setting; for an+ explicit user-requested analysis use [[analyze-db-explicit!]]."[database :- i/DatabaseInstance](sync-util/sync-operation :analyze database (format "Analyze data for %s" (sync-util/name-for-logging database))- (sync-util/with-emoji-progress-bar [emoji-progress-bar (inc (* 4 (sync-util/sync-tables-count database)))]- (u/prog1 (sync-util/run-sync-operation "analyze" database (make-analyze-steps (maybe-log-progress emoji-progress-bar)))- (update-fields-last-analyzed-for-db! database)))))+ (analyze-db!* database)))++(mu/defn analyze-db-explicit!+ "Like [[analyze-db!]], but for an explicit, user-requested sync (e.g. the Sync-now button): runs even+ when the `disable-auto-sync` setting is enabled."+ [database :- i/DatabaseInstance]+ (sync-util/explicit-sync-operation :analyze database (format "Analyze data for %s" (sync-util/name-for-logging database))+ (analyze-db!* database)))(mu/defn refingerprint-db!"Refingerprint a subset of tables in a given `database`. This will re-fingerprint tables up to a threshold amount ofdiff --git a/src/metabase/sync/core.clj b/src/metabase/sync/core.cljindex 5f9acd2527b..4617e6f4833 100644--- a/src/metabase/sync/core.clj+++ b/src/metabase/sync/core.clj@@ -20,7 +20,8 @@(p/import-vars[metabase.sync.analyze- analyze-db!]+ analyze-db!+ analyze-db-explicit!][metabase.sync.field-valuesupdate-field-values!update-field-values-for-table!]@@ -29,7 +30,8 @@sync-database!sync-table!][metabase.sync.sync-metadata- sync-db-metadata!]+ sync-db-metadata!+ sync-db-metadata-explicit!][metabase.sync.sync-metadata.fieldssync-fields-for-table!][metabase.sync.sync-metadata.tablesdiff --git a/src/metabase/sync/events/sync_database.clj b/src/metabase/sync/events/sync_database.cljindex ef369d2d9ae..dc1a0e984cb 100644--- a/src/metabase/sync/events/sync_database.clj+++ b/src/metabase/sync/events/sync_database.clj@@ -6,6 +6,7 @@[metabase.util :as u][metabase.util.log :as log][metabase.util.quick-task :as quick-task]+ [metabase.warehouses.core :as warehouses][methodical.core :as methodical]))(derive ::event :metabase/event)@@ -16,7 +17,7 @@[topic {database :object :as _event}];; try/catch here to prevent individual topic processing exceptions from bubbling up. better to handle them here.(try- (when database+ (when (and database (not (warehouses/disable-auto-sync)));; just kick off a sync on another thread(quick-task/submit-task!(fn []diff --git a/src/metabase/sync/sync_metadata.clj b/src/metabase/sync/sync_metadata.cljindex 5dfb4c4a175..ee99278028e 100644--- a/src/metabase/sync/sync_metadata.clj+++ b/src/metabase/sync/sync_metadata.clj@@ -60,16 +60,31 @@;; Sync the metabase metadata table if it exists.(sync-util/create-sync-step "sync-metabase-metadata" #(metabase-metadata/sync-metabase-metadata! % db-metadata))])+(mu/defn- sync-db-metadata!*+ "Shared core of [[sync-db-metadata!]] and [[sync-db-metadata-explicit!]]: the metadata sync work,+ without the surrounding `*-sync-operation` wrapper (which is what applies the eligibility gating)."+ [database :- i/DatabaseInstance]+ (let [db-metadata (tracing/with-span :sync "sync.metadata.fetch-metadata" {:db/id (:id database)}+ (fetch-metadata/db-metadata database))]+ (u/prog1 (sync-util/run-sync-operation "sync" database (make-sync-steps db-metadata))+ (if (some sync-util/abandon-sync? (map second (:steps <>)))+ (sync-util/set-initial-database-sync-aborted! database)+ (sync-util/set-initial-database-sync-complete! database)))))+(mu/defn sync-db-metadata!- "Sync the metadata for a Metabase `database`. This makes sure child Table & Field objects are synchronized."+ "Sync the metadata for a Metabase `database`. This makes sure child Table & Field objects are synchronized.+ Subject to the `disable-auto-sync` setting; for an explicit user-requested sync use+ [[sync-db-metadata-explicit!]]."[database :- i/DatabaseInstance](sync-util/sync-operation :sync-metadata database (format "Sync metadata for %s" (sync-util/name-for-logging database))- (let [db-metadata (tracing/with-span :sync "sync.metadata.fetch-metadata" {:db/id (:id database)}- (fetch-metadata/db-metadata database))]- (u/prog1 (sync-util/run-sync-operation "sync" database (make-sync-steps db-metadata))- (if (some sync-util/abandon-sync? (map second (:steps <>)))- (sync-util/set-initial-database-sync-aborted! database)- (sync-util/set-initial-database-sync-complete! database))))))+ (sync-db-metadata!* database)))++(mu/defn sync-db-metadata-explicit!+ "Like [[sync-db-metadata!]], but for an explicit, user-requested sync (e.g. the Sync-now button):+ runs even when the `disable-auto-sync` setting is enabled."+ [database :- i/DatabaseInstance]+ (sync-util/explicit-sync-operation :sync-metadata database (format "Sync metadata for %s" (sync-util/name-for-logging database))+ (sync-db-metadata!* database)))(mu/defn sync-table-metadata!"Sync the metadata for an individual `table` -- make sure Fields and FKs are up-to-date."diff --git a/src/metabase/sync/task/sync_databases.clj b/src/metabase/sync/task/sync_databases.cljindex 2790b2fa7f9..0cf6ff5bf26 100644--- a/src/metabase/sync/task/sync_databases.clj+++ b/src/metabase/sync/task/sync_databases.clj@@ -28,6 +28,7 @@[metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]+ [metabase.warehouses.core :as warehouses][toucan2.core :as t2])(:import(org.quartz@@ -107,7 +108,11 @@"The sync and analyze database job, as a function that can be used in a test"[job-context](when-let [database-id (job-context->database-id job-context)]- (if (= audit/audit-db-id database-id)+ (cond+ (warehouses/disable-auto-sync)+ (log/debugf "Skipping scheduled sync for Database %d: disable-auto-sync is on." database-id)++ (= audit/audit-db-id database-id)(do(log/warn "Cannot sync Database: It is the audit db.")(when-not config/is-prod?@@ -115,6 +120,8 @@{:database-id database-id:raw-job-context job-context:job-context (pr-str job-context)}))))++ :else(sync-and-analyze-database*! database-id))))(task/defjob ^{org.quartz.DisallowConcurrentExecution true@@ -126,14 +133,17 @@"The update field values job, as a function that can be used in a test"[job-context](when-let [database-id (job-context->database-id job-context)]- (log/infof "Update Field values task triggered for Database %d." database-id)- (when-let [database (or (t2/select-one :model/Database :id database-id)- (do- (unschedule-tasks-for-db! (mi/instance :model/Database {:id database-id}))- (log/warnf "Cannot update Field values for Database %d: Database does not exist." database-id)))]- (if (:is_full_sync database)- (sync.field-values/update-field-values! database)- (log/infof "Skipping update, automatic Field value updates are disabled for Database %d." database-id)))))+ (if (warehouses/disable-auto-sync)+ (log/debugf "Skipping scheduled field-values update for Database %d: disable-auto-sync is on." database-id)+ (do+ (log/infof "Update Field values task triggered for Database %d." database-id)+ (when-let [database (or (t2/select-one :model/Database :id database-id)+ (do+ (unschedule-tasks-for-db! (mi/instance :model/Database {:id database-id}))+ (log/warnf "Cannot update Field values for Database %d: Database does not exist." database-id)))]+ (if (:is_full_sync database)+ (sync.field-values/update-field-values! database)+ (log/infof "Skipping update, automatic Field value updates are disabled for Database %d." database-id)))))))(task/defjob ^{org.quartz.DisallowConcurrentExecution true:doc "Update field values"}diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.cljindex 4e6d0390044..1e72a6134ea 100644--- a/src/metabase/sync/util.clj+++ b/src/metabase/sync/util.clj@@ -91,6 +91,19 @@;; setups in the future(defonce ^:private operation->db-ids (atom {}))+;; In-JVM only, same scope as `operation->db-ids`.+(defonce ^:private external-busy-predicates (atom []))++(defn register-busy-predicate!+ "Register a 0-arity `pred` to be checked before each `do-sync-operation`.+ `pred` returns nil to permit the sync, or `{:reason \"...\"}` to skip it+ (logged at WARN by the caller)."+ [pred]+ (swap! external-busy-predicates conj pred))++(defn- external-busy []+ (some (fn [p] (p)) @external-busy-predicates))+(defn with-duplicate-ops-prevented"Run `f` in a way that will prevent it from simultaneously being ran more for a single database more than once for agiven `operation`. This prevents duplicate sync-like operations from taking place for a given DB, e.g. if a user@@ -259,13 +272,17 @@:analyze :fingerprint:refingerprint :fingerprint})-(mu/defn do-sync-operation- "Internal implementation of [[sync-operation]]; use that instead of calling this directly."+(mu/defn- do-sync-operation*+ "Shared core of [[do-sync-operation]] and [[do-explicit-sync-operation]]. Runs the sync work `f`+ wrapped in the surrounding machinery (task-history, events, logging, duplicate-op prevention).+ Performs no eligibility gating itself — the public wrappers decide whether to call it."[operation :- :keyword ; something like `:sync-metadata` or `:refingerprint`database :- (ms/InstanceOf :model/Database)message :- ms/NonBlankStringf :- fn?]- (when (database/should-sync? database)+ (if-let [busy (external-busy)]+ (log/warnf "Skipping %s for database %d: %s"+ (name operation) (u/the-id database) (:reason busy))(let [run-type (operation->run-type operation)](task-history/with-task-run (when run-type{:run_type run-type@@ -285,14 +302,46 @@(analytics/inc! :metabase-sync/failures {:driver (name (:engine database))}))result)))))+(mu/defn do-sync-operation+ "Internal implementation of [[sync-operation]]; use that instead of calling this directly. Runs+ only when the database is eligible for *automatic* sync (see+ [[metabase.warehouses.models.database/should-auto-sync?]]) — i.e. it is suppressed when the+ `disable-auto-sync` setting is on."+ [operation :- :keyword+ database :- (ms/InstanceOf :model/Database)+ message :- ms/NonBlankString+ f :- fn?]+ (when (database/should-auto-sync? database)+ (do-sync-operation* operation database message f)))++(mu/defn do-explicit-sync-operation+ "Internal implementation of [[explicit-sync-operation]]; use that instead of calling this directly.+ For explicit, user-requested syncs (e.g. the Sync-now button): runs whenever the database is+ syncable at all (see [[metabase.warehouses.models.database/should-sync?]]), ignoring the+ `disable-auto-sync` setting, which suppresses only automatically-triggered syncs."+ [operation :- :keyword+ database :- (ms/InstanceOf :model/Database)+ message :- ms/NonBlankString+ f :- fn?]+ (when (database/should-sync? database)+ (do-sync-operation* operation database message f)))+(defmacro sync-operation"Perform the operations in `body` as a sync operation, which wraps the code in several special macros that do thingslike error handling, logging, duplicate operation prevention, and event publishing. Intended for use with the- various top-level sync operations, such as `sync-metadata` or `analyze`."+ various top-level sync operations, such as `sync-metadata` or `analyze`. Suppressed when+ `disable-auto-sync` is on; for explicit user-requested syncs use [[explicit-sync-operation]]."{:style/indent 3}[operation database message & body]`(do-sync-operation ~operation ~database ~message (fn [] ~@body)))+(defmacro explicit-sync-operation+ "Like [[sync-operation]], but for explicit, user-requested syncs: the work runs even when the+ `disable-auto-sync` setting is enabled (that setting suppresses only automatically-triggered syncs)."+ {:style/indent 3}+ [operation database message & body]+ `(do-explicit-sync-operation ~operation ~database ~message (fn [] ~@body)))+;;; +----------------------------------------------------------------------------------------------------------------+;;; | EMOJI PROGRESS METER |;;; +----------------------------------------------------------------------------------------------------------------+diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex cde9cf2d314..e2ae2fb72e9 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -264,6 +264,24 @@table(group-perm-defaults table all-users-group non-magic-groups non-admin-groups)))))+(defn set-new-tables-permissions!+ "Batch variant of [[set-new-table-permissions!]] for many newly-inserted+ tables on the same database. Acquires the cluster lock once and delegates+ to [[perms/set-default-table-permissions-bulk!]]. All `tables` must share+ `db-id`."+ [db-id tables]+ (when (seq tables)+ (perms/with-db-scoped-permissions-lock db-id+ (let [all-users-group (perms/all-users-group)+ non-magic-groups (perms/non-magic-groups)+ non-admin-groups (conj non-magic-groups all-users-group)+ tables+defaults (mapv (fn [t]+ [t (group-perm-defaults t all-users-group+ non-magic-groups+ non-admin-groups)])+ tables)]+ (perms/set-default-table-permissions-bulk! db-id tables+defaults)))))+(t2/define-after-insert :model/Table[table](u/prog1 tablediff --git a/src/metabase/warehouses/core.clj b/src/metabase/warehouses/core.cljindex fa2767153b8..9c3c49d718a 100644--- a/src/metabase/warehouses/core.clj+++ b/src/metabase/warehouses/core.clj@@ -13,7 +13,8 @@[metabase.warehouses.provider-detectiondetect-provider-from-database][metabase.warehouses.settings- cloud-gateway-ips]+ cloud-gateway-ips+ disable-auto-sync][metabase.warehouses.utilget-databasetest-connection-detailsdiff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 2ddb7c60c42..0f81de35354 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -28,6 +28,7 @@[metabase.util.malli :as mu][metabase.util.quick-task :as quick-task][metabase.warehouses.provider-detection :as provider-detection]+ [metabase.warehouses.settings :as warehouses.settings][methodical.core :as methodical][toucan2.core :as t2][toucan2.pipeline :as t2.pipeline]@@ -204,16 +205,27 @@(boolean (:router_database_id db)))(defn should-sync?- "Should this database be synced?"+ "Should this database be synced at all? Destination (router-child) databases are never synced.+ This is the gate for *explicit*, user-requested syncs (e.g. the Sync-now button), which run+ regardless of the `disable-auto-sync` setting. For automatically-triggered syncs use+ [[should-auto-sync?]]."[db](not (is-destination? db)))+(defn should-auto-sync?+ "Should this database be synced *automatically* — scheduled sync/analyze, the post-add initial+ sync, and Quartz trigger registration? False when the database isn't syncable at all, or when+ auto-sync is globally suppressed via the `disable-auto-sync` setting."+ [db]+ (and (should-sync? db)+ (not (warehouses.settings/disable-auto-sync))))+(defn- check-and-schedule-tasks-for-db!"(Re)schedule sync operation tasks for `database`. (Existing scheduled tasks will be deleted first.)"[database](try;; this is done this way to avoid circular dependencies- (when (should-sync? database)+ (when (should-auto-sync? database)((requiring-resolve 'metabase.sync.task.sync-databases/check-and-schedule-tasks-for-db!) database))(catch Throwable e(log/error e "Error scheduling tasks for DB"))))diff --git a/src/metabase/warehouses/settings.clj b/src/metabase/warehouses/settings.cljindex adbef52d48b..dca974722c5 100644--- a/src/metabase/warehouses/settings.clj+++ b/src/metabase/warehouses/settings.clj@@ -5,6 +5,20 @@[metabase.settings.core :as setting :refer [defsetting]][metabase.util.i18n :refer [deferred-tru]]))+(defsetting disable-auto-sync+ (deferred-tru+ (str "When true, suppresses automatically-triggered syncs: the scheduled sync-and-analyze and "+ "update-field-values jobs do not run (and new triggers are not registered), and adding a "+ "new database does not kick off an initial sync. "+ "Syncs originating from an explicit request — the Sync-now REST endpoints, or a transform "+ "finalizing its output table — are unaffected. "+ "For deployments that load database metadata from disk at startup and should not have "+ "Metabase re-discover it."))+ :type :boolean+ :default false+ :visibility :internal+ :export? false)+(defsetting cloud-gateway-ips(deferred-tru "Metabase Cloud gateway IP addresses, to configure connections to DBs behind firewalls"):visibility :publicdiff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.cljindex 901bfec98cf..4344884c988 100644--- a/src/metabase/warehouses_rest/api.clj+++ b/src/metabase/warehouses_rest/api.clj@@ -1152,8 +1152,9 @@(quick-task/submit-task!(fn [](database-routing/with-database-routing-off- (sync/sync-db-metadata! db)- (sync/analyze-db! db))))+ ;; explicit, user-requested sync — runs even when `disable-auto-sync` is enabled+ (sync/sync-db-metadata-explicit! db)+ (sync/analyze-db-explicit! db)))){:status :ok}))));; TODO (Cam 10/28/25) -- fix this endpoint route to use kebab-case for consistency with the rest of our REST APIdiff --git a/test/metabase/sync/events/sync_database_test.clj b/test/metabase/sync/events/sync_database_test.cljnew file mode 100644index 00000000000..e4ba7c1cf41--- /dev/null+++ b/test/metabase/sync/events/sync_database_test.clj@@ -0,0 +1,27 @@+(ns metabase.sync.events.sync-database-test+ (:require+ [clojure.test :refer :all]+ [metabase.events.core :as events]+ [metabase.sync.sync :as sync]+ [metabase.sync.sync-metadata :as sync-metadata]+ [metabase.test :as mt]+ [metabase.util.quick-task :as quick-task]))++(set! *warn-on-reflection* true)++(deftest database-create-event-respects-disable-auto-sync-test+ (testing "The :event/database-create handler kicks off a sync on a new thread by default"+ (mt/with-temp [:model/Database {:as db} {:is_full_sync true}]+ (let [calls (atom 0)]+ (with-redefs [quick-task/submit-task! (fn [task] (task))+ sync/sync-database! (fn [_] (swap! calls inc))+ sync-metadata/sync-db-metadata! (fn [_] (swap! calls inc))]+ (testing "default (disable-auto-sync=false): a sync entry point is invoked"+ (reset! calls 0)+ (events/publish-event! :event/database-create {:object db :user-id (mt/user->id :crowberto)})+ (is (= 1 @calls)))+ (testing "with disable-auto-sync=true: no sync entry point is invoked"+ (reset! calls 0)+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (events/publish-event! :event/database-create {:object db :user-id (mt/user->id :crowberto)}))+ (is (zero? @calls))))))))diff --git a/test/metabase/sync/task/sync_databases_test.clj b/test/metabase/sync/task/sync_databases_test.cljindex 203bd602787..06288b07f7f 100644--- a/test/metabase/sync/task/sync_databases_test.clj+++ b/test/metabase/sync/task/sync_databases_test.clj@@ -7,6 +7,7 @@[clojure.test :refer :all][clojurewerkz.quartzite.conversion :as qc][java-time.api :as t]+ [metabase.sync.field-values :as sync.field-values][metabase.sync.schedules :as sync.schedules][metabase.sync.task.sync-databases :as task.sync-databases][metabase.task.core :as task]@@ -172,6 +173,34 @@(from-job-data [this](.getMergedJobDataMap this)))+(deftest scheduled-jobs-respect-disable-auto-sync-test+ (testing "When disable-auto-sync=true, scheduled job fns no-op even if a stale trigger fires"+ (mt/with-temp [:model/Database {db-id :id} {:is_full_sync true}]+ (testing "SyncAndAnalyzeDatabase: inner sync orchestrator is skipped when the flag is on"+ (let [calls (atom 0)]+ (with-redefs [task.sync-databases/sync-and-analyze-database*! (fn [_] (swap! calls inc))]+ (testing "default (flag=false): job proceeds and calls the inner orchestrator"+ (reset! calls 0)+ (#'task.sync-databases/sync-and-analyze-database! (MockJobExecutionContext. {"db-id" db-id}))+ (is (= 1 @calls)))+ (testing "flag=true: job returns early; the inner orchestrator is not called"+ (reset! calls 0)+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (#'task.sync-databases/sync-and-analyze-database! (MockJobExecutionContext. {"db-id" db-id})))+ (is (zero? @calls))))))+ (testing "UpdateFieldValues: field-values update is skipped when the flag is on"+ (let [calls (atom 0)]+ (with-redefs [sync.field-values/update-field-values! (fn [_] (swap! calls inc))]+ (testing "default (flag=false): job proceeds and calls update-field-values!"+ (reset! calls 0)+ (#'task.sync-databases/update-field-values! (MockJobExecutionContext. {"db-id" db-id}))+ (is (= 1 @calls)))+ (testing "flag=true: job returns early; update-field-values! is not called"+ (reset! calls 0)+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (#'task.sync-databases/update-field-values! (MockJobExecutionContext. {"db-id" db-id})))+ (is (zero? @calls)))))))))+(deftest check-orphaned-jobs-removed-test(testing "jobs for orphaned databases are removed during sync run"(with-scheduler-setup!diff --git a/test/metabase/transforms_base/util_test.clj b/test/metabase/transforms_base/util_test.cljindex 640a6e83dfa..86be4dcf9c3 100644--- a/test/metabase/transforms_base/util_test.clj+++ b/test/metabase/transforms_base/util_test.clj@@ -3,6 +3,7 @@[clojure.test :refer :all][java-time.api :as t][metabase.config.core :as config]+ [metabase.sync.sync :as sync][metabase.test :as mt][metabase.transforms-base.util :as transforms-base.u]))@@ -26,6 +27,21 @@{:name "Routing transform"}(mt/db)))))))))+(deftest activate-table-syncs-despite-disable-auto-sync-test+ (testing "disable-auto-sync gates *automatic* syncs only; a transform finalizing its"+ (testing "output table still calls sync/sync-table! so the new table's fields are populated."+ (let [calls (atom 0)]+ (mt/with-temp [:model/Table _ {:db_id (mt/id)+ :schema nil+ :name "disable_auto_sync_target"}]+ (with-redefs [sync/sync-table! (fn [_] (swap! calls inc))]+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (transforms-base.u/activate-table-and-mark-computed!+ (mt/db)+ {:type "table" :schema nil :name "disable_auto_sync_target"}))+ (is (= 1 @calls)+ "Expected the transform path to run sync/sync-table! exactly once with disable-auto-sync on.")))))))+(deftest ^:parallel full-incremental-run?-test(testing "true for an incremental transform with no checkpoint yet"(is (true? (transforms-base.u/full-incremental-run?diff --git a/test/metabase/warehouses/models/database_test.clj b/test/metabase/warehouses/models/database_test.cljindex 35bb4596c27..a216717826e 100644--- a/test/metabase/warehouses/models/database_test.clj+++ b/test/metabase/warehouses/models/database_test.clj@@ -34,6 +34,30 @@(use-fixtures :once (fixtures/initialize :db :plugins :test-drivers))+(deftest should-auto-sync?-respects-disable-auto-sync-test+ (testing "should-auto-sync? gates automatically-triggered syncs on the disable-auto-sync setting,"+ (testing "so that newly inserted or edited databases skip Quartz trigger registration."+ (mt/with-temp [:model/Database db {}]+ (testing "default (disable-auto-sync=false): a regular database is auto-sync-eligible"+ (is (true? (database/should-auto-sync? db))))+ (testing "with disable-auto-sync=true: a regular database is not auto-sync-eligible"+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (is (false? (database/should-auto-sync? db)))))))+ (testing "but should-sync? (the gate for explicit, user-requested syncs) ignores the setting"+ (mt/with-temp [:model/Database db {}]+ (is (true? (database/should-sync? db)))+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (is (true? (database/should-sync? db))+ "an explicit Sync-now must remain available even when auto-sync is disabled")))))+ (testing "destination databases (router children) are excluded from both, regardless of the flag"+ (mt/with-temp [:model/Database router {}+ :model/Database dest {:router_database_id (:id router)}]+ (is (false? (database/should-sync? dest)))+ (is (false? (database/should-auto-sync? dest)))+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (is (false? (database/should-sync? dest)))+ (is (false? (database/should-auto-sync? dest)))))))+(defn- trigger-for-db [db-id](some (fn [{trigger-key :key, :as trigger}](when (str/ends-with? trigger-key (str \. db-id))diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex 271389ac1e2..9662d7b4077 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -1529,10 +1529,11 @@analyze-called? (promise)](mt/with-premium-features #{:audit-app}(mt/with-temp [:model/Database {db-id :id} {:engine "h2", :details (:details (mt/db))}]- ;; redefine quick-task/submit-task! so as not to depend on the capacity of the quick-task executor- (with-redefs [quick-task/submit-task! future-call- sync-metadata/sync-db-metadata! (deliver-when-db sync-called? db-id)- analyze/analyze-db! (deliver-when-db analyze-called? db-id)]+ ;; redefine quick-task/submit-task! so as not to depend on the capacity of the quick-task executor.+ ;; The Sync-now endpoint dispatches to the *explicit* sync fns (which bypass disable-auto-sync).+ (with-redefs [quick-task/submit-task! future-call+ sync-metadata/sync-db-metadata-explicit! (deliver-when-db sync-called? db-id)+ analyze/analyze-db-explicit! (deliver-when-db analyze-called? db-id)](snowplow-test/with-fake-snowplow-collector(mt/user-http-request :crowberto :post 200 (format "database/%d/sync_schema" db-id));; Block waiting for the promises from sync and analyze to be delivered. Should be delivered instantly,@@ -1551,13 +1552,39 @@{"event" "database_manual_sync", "target_id" db-id}(:data (last (snowplow-test/pop-event-data-and-user-id!)))))))))))))+(deftest manual-sync-schema-runs-despite-disable-auto-sync-test+ (testing (str "POST /api/database/:id/sync_schema is an explicit-request sync and must actually run "+ "(not merely dispatch) when disable-auto-sync=true — the setting suppresses only "+ "automatically-triggered syncs.")+ ;; Resolve the test-data H2 connection details OUTSIDE the disable-auto-sync window, so loading the+ ;; reference DB isn't itself suppressed. Create the target DB INSIDE the window so its creation event+ ;; doesn't auto-sync it — then the only thing that can flip initial_sync_status to "complete" is the+ ;; explicit Sync-now request under test. We deliberately do NOT stub sync-db-metadata!/analyze-db!:+ ;; the real sync body must execute so the should-sync? gate inside do-sync-operation is genuinely+ ;; exercised. The submit-task! redef runs the sync synchronously (so we can assert its effect) with+ ;; H2 connections permitted on whatever thread the endpoint dispatches to.+ (let [details (:details (mt/db))]+ (mt/with-temporary-setting-values [disable-auto-sync true]+ (mt/with-temp [:model/Database {db-id :id} {:engine "h2"+ :details details+ :initial_sync_status "incomplete"}]+ (with-redefs [quick-task/submit-task! (fn [f]+ (binding [driver.settings/*allow-testing-h2-connections* true]+ (f)))]+ (mt/user-http-request :crowberto :post 200 (format "database/%d/sync_schema" db-id)))+ (testing "the explicit sync actually ran, not merely dispatched"+ (is (= "complete" (t2/select-one-fn :initial_sync_status :model/Database :id db-id))+ "Sync-now must complete the sync even when disable-auto-sync is on")+ (is (pos? (t2/count :model/Table :db_id db-id))+ "Sync-now must populate tables even when disable-auto-sync is on")))))))+(deftest sync-schema-executes-when-executor-busy-test(testing "POST /api/database/:id/sync_schema should execute sync even when quick-task executor is busy (GHY-3254)"(let [sync-called? (promise)blocker-latch (CountDownLatch. 1)](mt/with-temp [:model/Database {db-id :id} {:engine "h2" :details (:details (mt/db))}]- (with-redefs [sync-metadata/sync-db-metadata! (deliver-when-db sync-called? db-id)- analyze/analyze-db! (constantly nil)]+ (with-redefs [sync-metadata/sync-db-metadata-explicit! (deliver-when-db sync-called? db-id)+ analyze/analyze-db-explicit! (constantly nil)];; Submit a blocking task with a 1-second timeout so it gets cancelled quickly.;; This simulates a stuck sync (e.g., hanging JDBC connection) that exceeds;; the quick-task timeout and gets evicted.