The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GHY-3974] Fix Usage Analytics breaking after upgrade (two failure modes) (#76551)
Eric Normand0605e0352f78
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 93b7fec0e88..79423882756 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2873,7 +2873,6 @@app-dbaudit-appclassloader- configdriverlibmodelsdiff --git a/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj b/enterprise/backend/src/metabase_enterprise/audit_app/audit.cljindex 1e637f14a68..7385331bc72 100644--- a/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj+++ b/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj@@ -3,11 +3,13 @@[babashka.fs :as fs][clojure.java.io :as io][clojure.string :as str]+ [medley.core :as m][metabase-enterprise.audit-app.settings :as audit-app.settings][metabase-enterprise.serialization.cmd :as serialization.cmd]+ [metabase.app-db.cluster-lock :as cluster-lock][metabase.app-db.core :as mdb][metabase.audit-app.core :as audit]- [metabase.config.core :as config]+ [metabase.lib.core :as lib][metabase.plugins.core :as plugins][metabase.premium-features.core :refer [defenterprise]][metabase.sync.core :as sync]@@ -250,6 +252,13 @@[skip-checksum-flag skip-checksum-flag][last-checksum (analytics-checksum)])))+(defn- audit-db-in-source-state?+ "True when *active* audit tables are stuck at the postgres \"source\" schema (`public`) on a non-postgres host — the+ half-applied state left by an interrupted `adjust-audit-db-to-host!`."+ [audit-db-id]+ (and (not= :postgres (mdb/db-type))+ (t2/exists? :model/Table :db_id audit-db-id :schema "public" :active true)))+(defn- maybe-load-analytics-content!"Loads serialized audit content from the classpath if its checksum has changed.@@ -262,25 +271,30 @@(boolean(when analytics-dir-resource(ia-content->plugins (plugins/plugins-dir))- (let [[last-checksum current-checksum] (get-last-and-current-checksum)]- (when (should-load-audit? (audit-app.settings/load-analytics-content) last-checksum current-checksum)+ (let [[last-checksum current-checksum] (get-last-and-current-checksum)+ load? (audit-app.settings/load-analytics-content)]+ (when (or (should-load-audit? load? last-checksum current-checksum)+ (and load? (audit-db-in-source-state? (:id audit-db))))(adjust-audit-db-to-source! audit-db)(log/info (str "Loading Analytics Content from: " (instance-analytics-plugin-dir (plugins/plugins-dir))));; The EE token might not have :serialization enabled, but audit features should still be able to use it.- (let [report (log/with-no-logs- (serialization.cmd/v2-load-internal! (str (instance-analytics-plugin-dir (plugins/plugins-dir)))- {:backfill? false}- :token-check? false- :require-initialized-db? false))]- (if (not-empty (:errors report))- (log/info (str "Error Loading Analytics Content: " (pr-str report)))- (do- (log/info (str "Loading Analytics Content Complete (" (count (:seen report)) ") entities loaded."))- (audit/last-analytics-checksum! current-checksum))))- (when-let [{:keys [engine] :as audit-db} (t2/select-one :model/Database :is_audit true)]- (let [original-engine engine]- (adjust-audit-db-to-host! audit-db)- (not= original-engine (mdb/db-type)))))))))+ (let [report (log/with-no-logs+ (serialization.cmd/v2-load-internal! (str (instance-analytics-plugin-dir (plugins/plugins-dir)))+ {:backfill? false}+ :token-check? false+ :require-initialized-db? false))+ loaded? (empty? (:errors report))]+ (if loaded?+ (log/info (str "Loading Analytics Content Complete (" (count (:seen report)) ") entities loaded."))+ (log/info (str "Error Loading Analytics Content: " (pr-str report))))+ (when-let [{:keys [engine] :as audit-db} (t2/select-one :model/Database :is_audit true)]+ (let [original-engine engine]+ (adjust-audit-db-to-host! audit-db)+ ;; GHY-3974 Mode B: advance the checksum only after the host-adjust completes, so an+ ;; interrupted host-adjust leaves the checksum behind and the next boot re-runs the load.+ (when loaded?+ (audit/last-analytics-checksum! current-checksum))+ (not= original-engine (mdb/db-type))))))))))(defn- maybe-install-audit-db![]@@ -318,40 +332,168 @@(directory-content-checksum views-dir ".sql"))))(defn- maybe-sync-audit-db!- "One-shot `:scan :schema` sync of the audit DB. Fires when either trigger is true:+ "One-shot synchronous `:scan :schema` sync of the audit DB. Fires when either trigger is true:- `engine-changed?` — `maybe-load-analytics-content!` swapped the audit DB engine andfield metadata needs to be refreshed for the new dialect.- the `instance_analytics_views` SQL files have changed since the last successful sync,meaning a migration may have added a new view that isn't yet in `metabase_table`.- The two triggers share one sync because they both want the same operation; doing them- sequentially would race two parallel syncs against the same DB on a cold boot."+ The two triggers share one sync because they both want the same operation. Runs synchronously+ (not in a background future) so it stays inside the caller's cross-node cluster lock and+ transaction — a sync on another thread would escape the lock and, on a transactional appdb,+ deadlock against the caller's uncommitted `metabase_table` writes."[audit-db engine-changed?](let [current (views-checksum)views-stale? (and current (not= current (audit-app.settings/last-analytics-views-checksum)))](when (or engine-changed? views-stale?)(log/infof "Syncing Audit DB schema (engine-changed? %s, views-stale? %s)"engine-changed? views-stale?)- (let [sync-future (future- (try- (log/with-no-logs (sync/sync-database! audit-db {:scan :schema}))- (when current- (audit-app.settings/last-analytics-views-checksum! current))- (log/info "Audit DB sync complete.")- (catch Exception e- (log/error e "Audit DB sync failed."))))]- (when config/is-test?- ;; Tests need the sync to complete before they run- @sync-future)))))+ (try+ (log/with-no-logs (sync/sync-database! audit-db {:scan :schema}))+ (when current+ (audit-app.settings/last-analytics-views-checksum! current))+ (log/info "Audit DB sync complete.")+ (catch Exception e+ (log/error e "Audit DB sync failed."))))))++(defn- host-canonical-table+ "The `[name schema]` an audit-DB `metabase_table` row should use for the host engine, matching the+ conventions `adjust-audit-db-to-host!` applies."+ [table-name]+ (case (mdb/db-type)+ :mysql [(u/lower-case-en table-name) nil]+ :h2 [(u/upper-case-en table-name) "PUBLIC"]+ [(u/lower-case-en table-name) "public"]))++(defn- orphan->survivor-field-ids+ "Map of `orphan-table-id`'s field ids to `survivor-table-id`'s field ids, matched by lower-cased field name.+ Orphan fields with no same-named survivor field are omitted."+ [orphan-table-id survivor-table-id]+ (let [survivor-by-name (into {}+ (map (juxt (comp u/lower-case-en :name) :id))+ (t2/select [:model/Field :id :name] :table_id survivor-table-id))]+ (into {}+ (keep (fn [{:keys [id name]}]+ (when-let [survivor-field-id (survivor-by-name (u/lower-case-en name))]+ [id survivor-field-id])))+ (t2/select [:model/Field :id :name] :table_id orphan-table-id))))++(defn- remap-result-metadata-ref+ "Remap the Field ID of a single result-metadata `:field_ref` via `field-id-remap`. Handles legacy refs+ (`[:field id opts]`, `[:field-id id]`, id second) and pMBQL refs (`[:field opts id]`, id last). Non-field refs and+ ids with no remapping are returned unchanged."+ [field-id-remap ref]+ (if (and (vector? ref) (#{:field :field-id} (first ref)))+ (let [idx (if (map? (second ref)) 2 1)+ id (nth ref idx nil)]+ (if (field-id-remap id)+ (assoc ref idx (field-id-remap id))+ ref))+ ref))++(defn- remap-result-metadata+ "Remap orphan Field IDs onto survivor fields in a card's `result_metadata` columns — the `:id`,+ `:fk_target_field_id`, and `:field_ref` of each column — via `field-id-remap`. Result metadata is stored, not a+ Lib query, so it's handled here rather than by the Lib query helpers."+ [field-id-remap result-metadata]+ (let [remap-id (fn [id] (or (field-id-remap id) id))]+ (mapv (fn [col]+ (-> col+ (m/update-existing :id remap-id)+ (m/update-existing :fk_target_field_id remap-id)+ (m/update-existing :field_ref #(remap-result-metadata-ref field-id-remap %))))+ result-metadata)))++(defn- repoint-cards-to-survivor!+ "Move every card that references `orphan-table-id` onto `survivor-table-id` by rewriting its query and result+ metadata. A card's `table_id` is derived from its `dataset_query` by `populate-query-fields`, so updating the+ column alone is silently reverted — and the orphan's FK is ON DELETE CASCADE, so a card left behind is destroyed+ when the orphan is deleted. Field ids are remapped onto the survivor's same-named fields."+ [orphan-table-id survivor-table-id]+ (let [field-id-remap (orphan->survivor-field-ids orphan-table-id survivor-table-id)]+ (doseq [card (t2/select :model/Card :table_id orphan-table-id)]+ (t2/update! :model/Card (:id card)+ (cond-> {:dataset_query (-> (:dataset_query card)+ (lib/replace-table-ids {orphan-table-id survivor-table-id})+ (lib/replace-field-ids field-id-remap))}+ (:result_metadata card)+ (assoc :result_metadata (remap-result-metadata field-id-remap (:result_metadata card))))))))++(defn- reconcile-audit-db-duplicates!+ "Collapse duplicate `metabase_table` rows for the same audit view into a single active row at the host-canonical+ name/schema.++ Idempotent: a no-op when there are no duplicates.++ Dedupe by choosing the row other content points at, otherwise the active row, otherwise the lowest id. Content on+ the rows being removed is moved onto the survivor (query source-table and field ids rewritten) before they are+ deleted, so no card is lost to the orphan's ON DELETE CASCADE."+ [audit-db-id]+ ;; order by id in the query so every selection below (including the `referenced?` tiebreak) is deterministic;+ ;; group-by preserves this order within each group+ (let [groups (->> (t2/select [:model/Table :id :name :schema :active] :db_id audit-db-id+ {:order-by [[:id :asc]]})+ (group-by (comp u/lower-case-en :name))+ (filter (fn [[_ rows]] (> (count rows) 1))))]+ (doseq [[_ rows] groups]+ (let [referenced-ids (into #{}+ (map :table_id)+ (t2/query {:select-distinct [:table_id]+ :from [(t2/table-name :model/Card)]+ :where [:in :table_id (map :id rows)]}))+ survivor (or (first (filter (comp referenced-ids :id) rows))+ (first (filter :active rows))+ (first rows))+ [c-name c-schema] (host-canonical-table (:name survivor))+ orphans (remove #(= (:id %) (:id survivor)) rows)]+ ;; move content off each orphan, then delete it before re-canonicalizing the survivor so the survivor's+ ;; new (name, schema) can't collide with an orphan still sitting at that slot on idx_unique_table+ (doseq [{orphan-id :id} orphans]+ (repoint-cards-to-survivor! orphan-id (:id survivor))+ (t2/delete! :model/Table orphan-id))+ ;; clear is_defective_duplicate so the survivor re-enters idx_unique_table (its unique_table_helper is+ ;; NULL — and thus excluded from the index — while the flag is set)+ (t2/update! :model/Table (:id survivor)+ {:active true :name c-name :schema c-schema :is_defective_duplicate false})+ (log/infof "Reconciled %d duplicate audit view row(s) onto table id %s"+ (count orphans) (:id survivor))))))++(def ^:private audit-db-cluster-lock+ "Cluster lock serializing the audit DB load/adjust/sync/reconcile across nodes (GHY-3974 1b)."+ ::audit-db-lock)++(defn- audit-lock-contention?+ "True only for the exception `with-cluster-lock` throws when it could not acquire *our* audit lock. A+ different cluster lock's acquisition failure raised from inside the locked body (e.g. the permissions+ lock taken when serdes inserts a Database) also carries `:lock-names`, but must propagate rather than be+ swallowed as audit contention."+ [e]+ (let [audit-lock-name (str (namespace audit-db-cluster-lock) "/" (name audit-db-cluster-lock))]+ (boolean (some #{audit-lock-name} (:lock-names (ex-data e))))))(defenterprise ensure-audit-db-installed!"EE implementation of `ensure-db-installed!`. Installs audit db if it does not already exist, and loads auditcontent if it is available.":feature :none[]- (u/prog1 (maybe-install-audit-db!)- (when-let [audit-db (t2/select-one :model/Database :is_audit true)]- ;; prevent sync while loading- ((sync-util/with-duplicate-ops-prevented- :sync-database audit-db- (fn []- (maybe-sync-audit-db! audit-db (maybe-load-analytics-content! audit-db))))))))+ ;; serialize install+adjust+load+sync+reconcile across nodes so a rolling upgrade can't run an adjust or sync+ ;; against a half-adjusted schema (`with-duplicate-ops-prevented` is per-process only). The install runs inside the+ ;; lock too, so a node that acquires it after the installer committed sees the DB already exists and falls through+ ;; to a no-op rather than colliding on the audit DB primary key. The sync runs synchronously inside the lock so it+ ;; stays in the lock's transaction. If another node already holds the lock it is doing this same work, so we skip+ ;; rather than fail the boot.+ (try+ (cluster-lock/with-cluster-lock {:lock audit-db-cluster-lock :timeout-seconds 5 :retry-config {:max-retries 2}}+ (u/prog1 (maybe-install-audit-db!)+ (when-let [audit-db (t2/select-one :model/Database :is_audit true)]+ ((sync-util/with-duplicate-ops-prevented+ :sync-database audit-db+ (fn []+ (maybe-sync-audit-db! audit-db (maybe-load-analytics-content! audit-db))+ ;; GHY-3974 Mode A: runs every boot so already-corrupted instances (whose checksum already+ ;; matches, so the load/sync above are skipped) still self-heal.+ (reconcile-audit-db-duplicates! (:id audit-db))))))))+ (catch Throwable e+ (if (audit-lock-contention? e)+ (u/prog1 ::skipped-locked+ (log/info "Another node holds the audit DB lock; skipping audit DB install/load on this node."))+ (throw e)))))diff --git a/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj b/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.cljindex 3fc69f1a4fc..8d3bb8765c4 100644--- a/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj+++ b/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj@@ -10,6 +10,7 @@[metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill][metabase.audit-app.core :as audit][metabase.core.core :as mbc]+ [metabase.lib.core :as lib][metabase.models.serialization :as serdes][metabase.permissions-rest.data-permissions.graph :as data-perms.graph][metabase.permissions.models.permissions-group :as perms-group]@@ -318,3 +319,137 @@(testing "Fields without lowercase versions should be converted to lowercase"(is (= "product"(t2/select-one-fn :name :model/Field :id single-field-id)))))))++(defn- audit-view-table+ "The single active `metabase_table` row for audit view `view-name` (a lower-cased name), or nil.+ Matched case-insensitively so it finds the same view across host engines whose name casing differs."+ [view-name]+ (t2/select-one [:model/Table :id :name :schema :active]+ :db_id audit/audit-db-id+ :active true+ {:where [:= [:lower :name] view-name]}))++(deftest audit-db-self-heals-duplicate-rows-from-stale-schema-test+ ;; GHY-3974 Mode A: when an interrupted or raced `adjust-audit-db-to-host!` leaves an audit+ ;; `metabase_table` row at a schema the host driver does not report, the next schema sync sees the+ ;; driver-reported tuple as new (insert) and the existing row as old (retire) — duplicating the view+ ;; and breaking customer content that still points at the now-inactive row. `ensure-audit-db-installed!`+ ;; must reconcile back to a single active row per view and leave dependent content on an active table.+ (mt/test-drivers #{:postgres :h2 :mysql}+ (with-audit-db-restoration!+ (ee-audit/ensure-audit-db-installed!)+ (let [v-content (audit-view-table "v_content")+ orig-id (:id v-content)]+ (is (some? orig-id) "expected a v_content table after install")+ (mt/with-temp [:model/Card {card-id :id}+ {:database_id audit/audit-db-id+ :table_id orig-id+ :dataset_query {:database audit/audit-db-id+ :type :query+ :query {:source-table orig-id}}}]+ (testing "precondition: an interrupted host-adjust + schema sync duplicates the view row"+ ;; flip the schema to a value the host driver will not report, so the sync diff mismatches+ (t2/update! :model/Table orig-id {:schema (if (nil? (:schema v-content)) "public" nil)})+ (sync/sync-database! (t2/select-one :model/Database :is_audit true) {:scan :schema})+ (is (= 2 (t2/count :model/Table :db_id audit/audit-db-id :name (:name v-content)))+ "stale-schema sync should produce a duplicate pair")+ (is (false? (t2/select-one-fn :active :model/Table :id orig-id))+ "the row the customer card points at is retired"))+ (testing "ensure-audit-db-installed! self-heals the duplicate"+ (ee-audit/ensure-audit-db-installed!)+ (let [rows (t2/select [:model/Table :id :name :active]+ :db_id audit/audit-db-id+ {:where [:= [:lower :name] "v_content"]})]+ (is (= 1 (count rows)) "exactly one metabase_table row per view after heal")+ (is (every? :active rows) "the surviving row is active"))+ (testing "the customer card still points at an active table"+ (is (true? (t2/select-one-fn :active :model/Table+ :id (t2/select-one-fn :table_id :model/Card :id card-id)))+ "card's table_id must reference an active table after heal"))))))))++(deftest audit-db-reconcile-preserves-content-on-the-deleted-duplicate-test+ ;; GHY-3974 Mode A, destructive path: the sibling test above only covers a card pointing at the row that+ ;; becomes the *survivor*. But in real Mode A there are *two* referenced rows — Metabase's own bundled audit+ ;; cards reference the original (now retired) row, so any customer card that landed on the freshly-synced active+ ;; row points at the row reconcile chooses to delete. A naive `(t2/update! :model/Card {:table_id orphan} ...)`+ ;; repoint is a no-op (Card's before-update re-derives `table_id` from the unchanged `dataset_query`), and the+ ;; orphan's FK is ON DELETE CASCADE, so deleting it would destroy the customer card. A correct heal must rewrite+ ;; the card's query/field refs onto the survivor and keep it resolving.+ (mt/test-drivers #{:postgres :h2 :mysql}+ (with-audit-db-restoration!+ (ee-audit/ensure-audit-db-installed!)+ (let [v-content (audit-view-table "v_content")+ orig-id (:id v-content)]+ (is (some? orig-id) "expected a v_content table after install")+ ;; a card on the ORIGINAL row stands in for bundled content, forcing orig to win as survivor+ (mt/with-temp [:model/Card _bundled+ {:database_id audit/audit-db-id+ :table_id orig-id+ :dataset_query {:database audit/audit-db-id :type :query :query {:source-table orig-id}}}]+ (testing "precondition: a real Mode A duplicate pair, customer card on the new/active row"+ (t2/update! :model/Table orig-id {:schema (if (nil? (:schema v-content)) "public" nil)})+ (sync/sync-database! (t2/select-one :model/Database :is_audit true) {:scan :schema})+ (let [rows (t2/select [:model/Table :id :active] :db_id audit/audit-db-id+ {:where [:= [:lower :name] "v_content"]})+ new-id (:id (first (filter :active rows)))]+ (is (= 2 (count rows)) "stale-schema sync should produce a duplicate pair")+ (is (some? new-id) "a freshly-synced active row exists")+ (let [a-field (t2/select-one [:model/Field :id :name :base_type] :table_id new-id)+ a-field-id (:id a-field)]+ (mt/with-temp [:model/Card {card-id :id}+ {:database_id audit/audit-db-id+ :table_id new-id+ :dataset_query {:database audit/audit-db-id+ :type :query+ :query (cond-> {:source-table new-id}+ a-field-id (assoc :fields [[:field a-field-id nil]]))}+ ;; legacy-style result_metadata (id at position 1), as the QP stores it+ :result_metadata (when a-field-id+ [{:name (:name a-field)+ :base_type (:base_type a-field)+ :id a-field-id+ :field_ref [:field a-field-id nil]}])}]+ (testing "ensure-audit-db-installed! self-heals without destroying the customer card"+ (ee-audit/ensure-audit-db-installed!)+ (is (true? (t2/exists? :model/Card :id card-id))+ "customer card must still exist after heal (must not be cascade-deleted)")+ (let [healed (t2/select-one :model/Card :id card-id)+ healed-tid (:table_id healed)+ ref-field-id (-> healed :dataset_query :stages first :fields first lib/field-ref-id)]+ (is (some? healed-tid) "card's table_id must not be nil after heal")+ (is (true? (t2/select-one-fn :active :model/Table :id healed-tid))+ "card's table_id must reference an active table after heal")+ (when a-field-id+ (is (some? ref-field-id) "card's field ref survived the rewrite")+ (is (= healed-tid (t2/select-one-fn :table_id :model/Field :id ref-field-id))+ "card's field ref must resolve to a field on the survivor table")+ (testing "result_metadata (legacy field_ref) is remapped onto the survivor"+ (let [col (-> healed :result_metadata first)]+ (is (= healed-tid (t2/select-one-fn :table_id :model/Field :id (:id col)))+ "result_metadata :id resolves to a survivor field")+ (is (= (:id col) (nth (:field_ref col) 1))+ "result_metadata legacy :field_ref id matches the remapped :id")))))))))))))))++(deftest checksum-not-advanced-until-host-adjust-completes-test+ ;; GHY-3974 Mode B: `last-analytics-checksum` must not advance until the audit DB is fully back at the+ ;; host-canonical schema. Today the checksum is written right after the serdes load but *before*+ ;; `adjust-audit-db-to-host!`; a process death in that window leaves rows at the postgres "source"+ ;; convention (schema="public", postgres types) while the checksum already says "up to date", so+ ;; `should-load-audit?` returns false forever and the host-adjust never re-runs. If the host-adjust+ ;; is interrupted, the checksum must stay put so the next boot re-runs the load and the adjust.+ (mt/test-drivers #{:postgres :h2 :mysql}+ (try+ (t2/delete! :model/Database :is_audit true)+ (audit/last-analytics-checksum! 0)+ (testing "an interrupted adjust-audit-db-to-host! does not advance the checksum"+ (mt/with-dynamic-fn-redefs [serialization.cmd/v2-load-internal! (fn [& _] {:seen [] :errors []})+ ee-audit/adjust-audit-db-to-host! (fn [& _] (throw (ex-info "host-adjust interrupted" {})))]+ (is (thrown-with-msg? Exception #"host-adjust interrupted"+ (ee-audit/ensure-audit-db-installed!)))+ (is (= 0 (audit/last-analytics-checksum))+ "checksum stays 0 so the next boot re-runs the load and host-adjust")))+ (finally+ ;; restore a clean, installed audit DB for sibling tests regardless of how this test exits+ (t2/delete! :model/Database :is_audit true)+ (audit/last-analytics-checksum! 0)+ (mbc/ensure-audit-db-installed!)))))diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex 84912ecf3e7..00d077e4354 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -1579,7 +1579,8 @@[lib.reffield-ref-idfield-ref-name- ref]+ ref+ with-field-ref-id][lib.referenced-columnsreferenced-columns][lib.remove-replace@@ -1663,7 +1664,9 @@all-template-tags-mapall-template-tags-id->field-idsany-native-stage?- any-native-stage-not-introduced-by-sandbox?])+ any-native-stage-not-introduced-by-sandbox?+ replace-field-ids+ replace-table-ids])#?(:clj(defmacro with-card-clean-hookdiff --git a/src/metabase/lib/walk/util.cljc b/src/metabase/lib/walk/util.cljcindex fe9b100c4e3..4827f2c1c91 100644--- a/src/metabase/lib/walk/util.cljc+++ b/src/metabase/lib/walk/util.cljc@@ -9,6 +9,7 @@[metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.template-tag :as lib.schema.template-tag]+ [metabase.lib.util :as lib.util][metabase.lib.walk :as lib.walk][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]@@ -321,3 +322,49 @@:measure measure-ids:segment segment-ids:snippet template-tag-snippet-ids})))++(defn- remap-field-clause-ids+ "Remap the Field ID and any `:source-field`/`:fk-field-id` option of a single `:field` clause via `id->new-id`+ (a `nil` result leaves a value unchanged). Non-`:field` clauses and non-clauses are returned unchanged."+ [id->new-id clause]+ (if (lib.util/clause-of-type? clause :field)+ (let [[tag opts id-or-name] clause+ remap (fn [x] (if (pos-int? x) (or (id->new-id x) x) x))]+ [tag+ (cond-> opts+ (:source-field opts) (update :source-field remap)+ (:fk-field-id opts) (update :fk-field-id remap))+ (remap id-or-name)])+ clause))++(mu/defn replace-field-ids :- ::lib.schema/query+ "Return `query` with every Field ID reference remapped by `id->new-id` (a function; a map works). Covers `:field`+ refs everywhere they appear (filters, aggregations, breakouts, expressions, order-by, join conditions and fields,+ nested stages), the `:source-field`/`:fk-field-id` options inside `:field` refs, and native `:template-tags`+ `:dimension` refs. A field id for which `id->new-id` returns `nil` is left unchanged; name-based refs are untouched."+ [query :- ::lib.schema/query+ id->new-id :- ifn?]+ (let [remap #(remap-field-clause-ids id->new-id %)]+ (-> query+ (lib.walk/walk-clauses (fn [_query _path-type _path clause]+ (remap clause)))+ (lib.walk/walk-stages (fn [_query _path stage]+ (cond-> stage+ (:template-tags stage)+ (update :template-tags update-vals+ (fn [tag]+ (cond-> tag+ (:dimension tag) (update :dimension remap))))))))))++(mu/defn replace-table-ids :- ::lib.schema/query+ "Return `query` with every source Table ID remapped by `id->new-id` (a function; a map works). Covers+ `:source-table` in every stage and join. A table id for which `id->new-id` returns `nil` is left unchanged;+ `:source-card` refs are untouched."+ [query :- ::lib.schema/query+ id->new-id :- ifn?]+ (lib.walk/walk-stages+ query+ (fn [_query _path stage]+ (if-let [table-id (:source-table stage)]+ (assoc stage :source-table (or (id->new-id table-id) table-id))+ stage))))