Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[remote sync] add `on-success` callback, set git :version on revisions (#74031)

Braden Shepherdson32db915c8a2d
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj
index 5f4daa603b9..c25b9bc89e3 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj
@@ -19,6 +19,16 @@
(set! *warn-on-reflection* true)
+(defn- publish-sync-event!
+ "Publishes an audit-log event for a completed remote-sync task. Called from
+ the `:on-success` callback so the task already has its version set."
+ [topic task-id branch user-id]
+ (let [task (t2/select-one :model/RemoteSyncTask task-id)]
+ (events/publish-event! topic
+ {:object task
+ :details {:branch branch}
+ :user-id user-id})))
+
(api.macros/defendpoint :post "/import" :- remote-sync.schema/ImportResponse
"Import Metabase content from configured Remote Sync source.
@@ -37,11 +47,11 @@
(api/check-superuser)
(api/check-400 (settings/remote-sync-enabled) "Remote sync is not configured.")
(let [branch-name (or branch (settings/remote-sync-branch))
- {task-id :id :as task} (impl/async-import! branch-name force {})]
- (events/publish-event! :event/remote-sync-import
- {:object task
- :details {:branch branch-name}
- :user-id api/*current-user-id*})
+ user-id api/*current-user-id*
+ {task-id :id}
+ (impl/async-import!
+ branch-name force {}
+ :on-success #(publish-sync-event! :event/remote-sync-import %1 branch-name user-id))]
{:status :success
:task_id task-id
:message (when-not task-id "No changes since last import")}))
@@ -104,13 +114,13 @@
(api/check-400 (settings/remote-sync-enabled) "Remote sync is not configured.")
(api/check-400 (= (settings/remote-sync-type) :read-write) "Exports are only allowed when remote-sync-type is set to 'read-write'")
(let [branch-name (or branch (settings/remote-sync-branch))
- {task-id :id :as task} (impl/async-export! branch-name
- (or force false)
- (or message "Exported from Metabase"))]
- (events/publish-event! :event/remote-sync-export
- {:object task
- :details {:branch branch-name}
- :user-id api/*current-user-id*})
+ user-id api/*current-user-id*
+ {task-id :id}
+ (impl/async-export!
+ branch-name
+ (or force false)
+ (or message "Exported from Metabase")
+ :on-success #(publish-sync-event! :event/remote-sync-export %1 branch-name user-id))]
{:message "Export task started"
:task_id task-id}))
@@ -234,11 +244,9 @@
(api/check-400 (= (settings/remote-sync-type) :read-write) "Stash is only allowed when remote-sync-type is set to 'read-write'")
(api/check-400 (source/source-from-settings) "Source not configured")
(try
- (let [{task-id :id :as task} (impl/stash! new-branch message)]
- (events/publish-event! :event/remote-sync-stash
- {:object task
- :details {:branch new-branch}
- :user-id api/*current-user-id*})
+ (let [user-id api/*current-user-id*
+ {task-id :id} (impl/stash! new-branch message
+ :on-success #(publish-sync-event! :event/remote-sync-stash %1 new-branch user-id))]
{:status "success"
:message (str "Stashing to " new-branch)
:task_id task-id})
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index f4215fcf234..bbb5a62457b 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -497,37 +497,43 @@
(defn- run-async!
"Executes a remote sync task asynchronously in a virtual thread.
- Takes a task-type string ('import' or 'export'), a branch name to update in settings upon completion, and a
- sync-fn function that takes a task-id and performs the sync operation. Creates a new task (or errors if one is
- already running), then executes the sync function in a virtual thread with a timeout.
+ Takes a task-type string ('import' or 'export'), a branch name to update in settings upon completion, a
+ sync-fn function that takes a task-id and performs the sync operation, and an optional :on-success callback
+ that receives [task-id result] after a successful sync. Creates a new task (or errors if one is already
+ running), then executes the sync function in a virtual thread with a timeout.
Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 if a sync task is already in progress."
- [task-type branch sync-fn]
+ [task-type branch sync-fn & {:keys [on-success]}]
(let [{task-id :id existing? :existing? :as task} (create-task-with-lock! task-type)]
(api/check-400 (not existing?) "Remote sync in progress")
(u.jvm/in-virtual-thread*
(dh/with-timeout {:interrupt? true
:timeout-ms (* (settings/remote-sync-task-time-limit-ms) 10)}
- (handle-task-result!
- (try
- (sync-fn task-id)
- (catch Exception e
- (log/error e "Remote sync task failed")
- {:status :error
- :message (source-error-message e)}))
- task-id branch)))
+ (let [result (try
+ (sync-fn task-id)
+ (catch Exception e
+ (log/error e "Remote sync task failed")
+ {:status :error
+ :message (source-error-message e)}))]
+ (handle-task-result! result task-id branch)
+ (when (and on-success (= :success (:status result)))
+ (try
+ (on-success task-id result)
+ (catch Exception e
+ (log/error e "Remote sync task :on-success function failed")))))))
task))
(defn async-import!
"Imports remote-synced collections from a remote source repository asynchronously.
Takes a branch name to import from, a force? boolean (if true, imports even if there are unsaved changes or conflicts),
- and an import-args map of additional arguments to pass to the import function. Checks for dirty changes and throws an
+ and an import-args map of additional arguments to pass to the import function. Optionally accepts an :on-success
+ callback that receives [task-id result] after a successful import. Checks for dirty changes and throws an
exception if force? is false and changes exist.
Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 and :conflicts true if there
are unsaved changes and force? is false."
- [branch force? import-args]
+ [branch force? import-args & {:keys [on-success]}]
(guards/ensure-no-active-task!)
(let [pre-task-branch (settings/remote-sync-branch)
source (source/source-from-settings branch)
@@ -541,18 +547,20 @@
(import! (source.p/snapshot source) task-id
(assoc import-args
:force? force?
- :pre-task-branch pre-task-branch))))))
+ :pre-task-branch pre-task-branch)))
+ :on-success on-success)))
(defn async-export!
"Exports the remote-synced collections to the remote source repository asynchronously.
Takes a branch name to export to, a force? boolean (if true, exports even if there are new changes in the remote
- branch), and a commit message string. Checks if the remote branch has changed since the last sync and throws an
+ branch), and a commit message string. Optionally accepts an :on-success callback that receives [task-id result]
+ after a successful export. Checks if the remote branch has changed since the last sync and throws an
exception if force? is false and changes exist.
Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 and :conflicts true if there
are new remote changes and force? is false."
- [branch force? message]
+ [branch force? message & {:keys [on-success]}]
(guards/ensure-no-active-task!)
(let [pre-task-branch (settings/remote-sync-branch)
source (source/source-from-settings branch)
@@ -565,7 +573,9 @@
:conflicts true})))
(run-async! "export" branch
(fn [task-id]
- (export! snapshot task-id message :pre-task-branch pre-task-branch)))))
+ (export! snapshot task-id message
+ :pre-task-branch pre-task-branch))
+ :on-success on-success)))
(defn create-branch!
"Creates a new remote branch from `base-branch` and switches `remote-sync-branch`
@@ -580,11 +590,11 @@
(defn stash!
"Creates a new remote branch from the current `remote-sync-branch` and starts an
async export to it. Returns the resulting RemoteSyncTask. Does not publish events."
- [new-branch message]
+ [new-branch message & {:keys [on-success]}]
(guards/ensure-no-active-task!)
(let [source (source/source-from-settings)]
(source.p/create-branch source new-branch (settings/remote-sync-branch))
- (async-export! new-branch false message)))
+ (async-export! new-branch false message :on-success on-success)))
(defn finish-remote-config!
"Based on the current configuration, fill in any missing settings and finalize remote sync setup.
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
index d8563e399f6..32dab03c085 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
@@ -834,7 +834,7 @@
:status "updated"
:status_changed_at (java.time.OffsetDateTime/now)}]
(with-redefs [source/source-from-settings (constantly mock-source)
- impl/async-export! (fn [_ _ _] (assoc remote-sync :calls (swap! export-calls inc)))]
+ impl/async-export! (fn [_ _ _ & _opts] (assoc remote-sync :calls (swap! export-calls inc)))]
(is (=? {:status "success"
:message "Stashing to feature-branch"}
(mt/user-http-request :crowberto :post 200 "ee/remote-sync/stash"
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index 200a6f613b1..a503ad458dd 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -722,6 +722,7 @@
{:name "Test Collection"
:is_remote_synced true
:entity_id "test-collection-1xxxx"
+ :type "library-data"
:location "/"}
;; Table must have collection_id and is_published to be included as Collection descendant
:model/Table {table-id :id} {:name "test-table"
diff --git a/test/metabase/audit_app/events/audit_log_test.clj b/test/metabase/audit_app/events/audit_log_test.clj
index 59cfaf9dd78..40886eddfc0 100644
--- a/test/metabase/audit_app/events/audit_log_test.clj
+++ b/test/metabase/audit_app/events/audit_log_test.clj
@@ -11,7 +11,8 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.notification.test-util :as notification.tu]
[metabase.test :as mt]
- [metabase.util :as u]))
+ [metabase.util :as u]
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -696,6 +697,40 @@
:model "Glossary"}
(mt/latest-audit-log-entry :glossary-delete (:id glossary))))))))
+(deftest ^:synchronized remote-sync-event-version-after-completion-test
+ (testing "remote-sync audit log entries include version when event is published after task completion (#73329)"
+ (mt/when-ee-evailable
+ (mt/with-current-user (mt/user->id :rasta)
+ (mt/with-temp [:model/RemoteSyncTask {task-id :id}
+ {:sync_task_type "import"
+ :version nil
+ :initiated_by (mt/user->id :rasta)
+ :started_at (t/offset-date-time 2025 9 30 14 0 0)
+ :ended_at (t/offset-date-time 2025 9 30 14 0 0)}]
+ (testing "event published with re-fetched task after version is set"
+ (t2/update! :model/RemoteSyncTask task-id {:version "abc123"})
+ (let [completed-task (t2/select-one :model/RemoteSyncTask task-id)]
+ (events/publish-event! :event/remote-sync-import
+ {:object completed-task
+ :details {:branch "main"}
+ :user-id (mt/user->id :rasta)}))
+ (is (= "abc123"
+ (:version (:details (mt/latest-audit-log-entry :remote-sync-import task-id))))
+ "version should be the git hash from the completed task"))
+ (testing "event published with original task (before version set) has nil version"
+ (mt/with-temp [:model/RemoteSyncTask {task-id-2 :id :as task-2}
+ {:sync_task_type "import"
+ :version nil
+ :initiated_by (mt/user->id :rasta)
+ :started_at (t/offset-date-time 2025 9 30 14 0 0)
+ :ended_at (t/offset-date-time 2025 9 30 14 0 0)}]
+ (events/publish-event! :event/remote-sync-import
+ {:object task-2
+ :details {:branch "main"}
+ :user-id (mt/user->id :rasta)})
+ (is (nil? (:version (:details (mt/latest-audit-log-entry :remote-sync-import task-id-2))))
+ "version is nil when event is published before task sets it"))))))))
+
(deftest remote-sync-events-test
(mt/when-ee-evailable
(mt/with-current-user (mt/user->id :rasta)