The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[remote sync] add `on-success` callback, set git :version on revisions (#74031)
Braden Shepherdson32db915c8a2d
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/api.cljindex 5f4daa603b9..c25b9bc89e3 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj@@ -19,6 +19,16 @@(set! *warn-on-reflection* true)+(defn- publish-sync-event!+ "Publishes an audit-log event for a completed remote-sync task. Called from+ the `:on-success` callback so the task already has its version set."+ [topic task-id branch user-id]+ (let [task (t2/select-one :model/RemoteSyncTask task-id)]+ (events/publish-event! topic+ {:object task+ :details {:branch branch}+ :user-id user-id})))+(api.macros/defendpoint :post "/import" :- remote-sync.schema/ImportResponse"Import Metabase content from configured Remote Sync source.@@ -37,11 +47,11 @@(api/check-superuser)(api/check-400 (settings/remote-sync-enabled) "Remote sync is not configured.")(let [branch-name (or branch (settings/remote-sync-branch))- {task-id :id :as task} (impl/async-import! branch-name force {})]- (events/publish-event! :event/remote-sync-import- {:object task- :details {:branch branch-name}- :user-id api/*current-user-id*})+ user-id api/*current-user-id*+ {task-id :id}+ (impl/async-import!+ branch-name force {}+ :on-success #(publish-sync-event! :event/remote-sync-import %1 branch-name user-id))]{:status :success:task_id task-id:message (when-not task-id "No changes since last import")}))@@ -104,13 +114,13 @@(api/check-400 (settings/remote-sync-enabled) "Remote sync is not configured.")(api/check-400 (= (settings/remote-sync-type) :read-write) "Exports are only allowed when remote-sync-type is set to 'read-write'")(let [branch-name (or branch (settings/remote-sync-branch))- {task-id :id :as task} (impl/async-export! branch-name- (or force false)- (or message "Exported from Metabase"))]- (events/publish-event! :event/remote-sync-export- {:object task- :details {:branch branch-name}- :user-id api/*current-user-id*})+ user-id api/*current-user-id*+ {task-id :id}+ (impl/async-export!+ branch-name+ (or force false)+ (or message "Exported from Metabase")+ :on-success #(publish-sync-event! :event/remote-sync-export %1 branch-name user-id))]{:message "Export task started":task_id task-id}))@@ -234,11 +244,9 @@(api/check-400 (= (settings/remote-sync-type) :read-write) "Stash is only allowed when remote-sync-type is set to 'read-write'")(api/check-400 (source/source-from-settings) "Source not configured")(try- (let [{task-id :id :as task} (impl/stash! new-branch message)]- (events/publish-event! :event/remote-sync-stash- {:object task- :details {:branch new-branch}- :user-id api/*current-user-id*})+ (let [user-id api/*current-user-id*+ {task-id :id} (impl/stash! new-branch message+ :on-success #(publish-sync-event! :event/remote-sync-stash %1 new-branch user-id))]{:status "success":message (str "Stashing to " new-branch):task_id task-id})diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex f4215fcf234..bbb5a62457b 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -497,37 +497,43 @@(defn- run-async!"Executes a remote sync task asynchronously in a virtual thread.- Takes a task-type string ('import' or 'export'), a branch name to update in settings upon completion, and a- sync-fn function that takes a task-id and performs the sync operation. Creates a new task (or errors if one is- already running), then executes the sync function in a virtual thread with a timeout.+ Takes a task-type string ('import' or 'export'), a branch name to update in settings upon completion, a+ sync-fn function that takes a task-id and performs the sync operation, and an optional :on-success callback+ that receives [task-id result] after a successful sync. Creates a new task (or errors if one is already+ running), then executes the sync function in a virtual thread with a timeout.Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 if a sync task is already in progress."- [task-type branch sync-fn]+ [task-type branch sync-fn & {:keys [on-success]}](let [{task-id :id existing? :existing? :as task} (create-task-with-lock! task-type)](api/check-400 (not existing?) "Remote sync in progress")(u.jvm/in-virtual-thread*(dh/with-timeout {:interrupt? true:timeout-ms (* (settings/remote-sync-task-time-limit-ms) 10)}- (handle-task-result!- (try- (sync-fn task-id)- (catch Exception e- (log/error e "Remote sync task failed")- {:status :error- :message (source-error-message e)}))- task-id branch)))+ (let [result (try+ (sync-fn task-id)+ (catch Exception e+ (log/error e "Remote sync task failed")+ {:status :error+ :message (source-error-message e)}))]+ (handle-task-result! result task-id branch)+ (when (and on-success (= :success (:status result)))+ (try+ (on-success task-id result)+ (catch Exception e+ (log/error e "Remote sync task :on-success function failed")))))))task))(defn async-import!"Imports remote-synced collections from a remote source repository asynchronously.Takes a branch name to import from, a force? boolean (if true, imports even if there are unsaved changes or conflicts),- and an import-args map of additional arguments to pass to the import function. Checks for dirty changes and throws an+ and an import-args map of additional arguments to pass to the import function. Optionally accepts an :on-success+ callback that receives [task-id result] after a successful import. Checks for dirty changes and throws anexception if force? is false and changes exist.Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 and :conflicts true if thereare unsaved changes and force? is false."- [branch force? import-args]+ [branch force? import-args & {:keys [on-success]}](guards/ensure-no-active-task!)(let [pre-task-branch (settings/remote-sync-branch)source (source/source-from-settings branch)@@ -541,18 +547,20 @@(import! (source.p/snapshot source) task-id(assoc import-args:force? force?- :pre-task-branch pre-task-branch))))))+ :pre-task-branch pre-task-branch)))+ :on-success on-success)))(defn async-export!"Exports the remote-synced collections to the remote source repository asynchronously.Takes a branch name to export to, a force? boolean (if true, exports even if there are new changes in the remote- branch), and a commit message string. Checks if the remote branch has changed since the last sync and throws an+ branch), and a commit message string. Optionally accepts an :on-success callback that receives [task-id result]+ after a successful export. Checks if the remote branch has changed since the last sync and throws anexception if force? is false and changes exist.Returns a RemoteSyncTask. Throws ExceptionInfo with status 400 and :conflicts true if thereare new remote changes and force? is false."- [branch force? message]+ [branch force? message & {:keys [on-success]}](guards/ensure-no-active-task!)(let [pre-task-branch (settings/remote-sync-branch)source (source/source-from-settings branch)@@ -565,7 +573,9 @@:conflicts true})))(run-async! "export" branch(fn [task-id]- (export! snapshot task-id message :pre-task-branch pre-task-branch)))))+ (export! snapshot task-id message+ :pre-task-branch pre-task-branch))+ :on-success on-success)))(defn create-branch!"Creates a new remote branch from `base-branch` and switches `remote-sync-branch`@@ -580,11 +590,11 @@(defn stash!"Creates a new remote branch from the current `remote-sync-branch` and starts anasync export to it. Returns the resulting RemoteSyncTask. Does not publish events."- [new-branch message]+ [new-branch message & {:keys [on-success]}](guards/ensure-no-active-task!)(let [source (source/source-from-settings)](source.p/create-branch source new-branch (settings/remote-sync-branch))- (async-export! new-branch false message)))+ (async-export! new-branch false message :on-success on-success)))(defn finish-remote-config!"Based on the current configuration, fill in any missing settings and finalize remote sync setup.diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.cljindex d8563e399f6..32dab03c085 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj@@ -834,7 +834,7 @@:status "updated":status_changed_at (java.time.OffsetDateTime/now)}](with-redefs [source/source-from-settings (constantly mock-source)- impl/async-export! (fn [_ _ _] (assoc remote-sync :calls (swap! export-calls inc)))]+ impl/async-export! (fn [_ _ _ & _opts] (assoc remote-sync :calls (swap! export-calls inc)))](is (=? {:status "success":message "Stashing to feature-branch"}(mt/user-http-request :crowberto :post 200 "ee/remote-sync/stash"diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex 200a6f613b1..a503ad458dd 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -722,6 +722,7 @@{:name "Test Collection":is_remote_synced true:entity_id "test-collection-1xxxx"+ :type "library-data":location "/"};; Table must have collection_id and is_published to be included as Collection descendant:model/Table {table-id :id} {:name "test-table"diff --git a/test/metabase/audit_app/events/audit_log_test.clj b/test/metabase/audit_app/events/audit_log_test.cljindex 59cfaf9dd78..40886eddfc0 100644--- a/test/metabase/audit_app/events/audit_log_test.clj+++ b/test/metabase/audit_app/events/audit_log_test.clj@@ -11,7 +11,8 @@[metabase.lib.schema.id :as lib.schema.id][metabase.notification.test-util :as notification.tu][metabase.test :as mt]- [metabase.util :as u]))+ [metabase.util :as u]+ [toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -696,6 +697,40 @@:model "Glossary"}(mt/latest-audit-log-entry :glossary-delete (:id glossary))))))))+(deftest ^:synchronized remote-sync-event-version-after-completion-test+ (testing "remote-sync audit log entries include version when event is published after task completion (#73329)"+ (mt/when-ee-evailable+ (mt/with-current-user (mt/user->id :rasta)+ (mt/with-temp [:model/RemoteSyncTask {task-id :id}+ {:sync_task_type "import"+ :version nil+ :initiated_by (mt/user->id :rasta)+ :started_at (t/offset-date-time 2025 9 30 14 0 0)+ :ended_at (t/offset-date-time 2025 9 30 14 0 0)}]+ (testing "event published with re-fetched task after version is set"+ (t2/update! :model/RemoteSyncTask task-id {:version "abc123"})+ (let [completed-task (t2/select-one :model/RemoteSyncTask task-id)]+ (events/publish-event! :event/remote-sync-import+ {:object completed-task+ :details {:branch "main"}+ :user-id (mt/user->id :rasta)}))+ (is (= "abc123"+ (:version (:details (mt/latest-audit-log-entry :remote-sync-import task-id))))+ "version should be the git hash from the completed task"))+ (testing "event published with original task (before version set) has nil version"+ (mt/with-temp [:model/RemoteSyncTask {task-id-2 :id :as task-2}+ {:sync_task_type "import"+ :version nil+ :initiated_by (mt/user->id :rasta)+ :started_at (t/offset-date-time 2025 9 30 14 0 0)+ :ended_at (t/offset-date-time 2025 9 30 14 0 0)}]+ (events/publish-event! :event/remote-sync-import+ {:object task-2+ :details {:branch "main"}+ :user-id (mt/user->id :rasta)})+ (is (nil? (:version (:details (mt/latest-audit-log-entry :remote-sync-import task-id-2))))+ "version is nil when event is published before task sets it"))))))))+(deftest remote-sync-events-test(mt/when-ee-evailable(mt/with-current-user (mt/user->id :rasta)