Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Correctly sync routed databases in task (#65441)

dpsuttond5e562665f90
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index fcec2323a92..efadcc17bcc 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1271,6 +1271,7 @@
app-db
audit-app
config
+ database-routing
driver
events
lib
diff --git a/enterprise/backend/test/metabase_enterprise/database_routing/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/database_routing/e2e_test.clj
index a20cc006874..be3a4fc8983 100644
--- a/enterprise/backend/test/metabase_enterprise/database_routing/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/database_routing/e2e_test.clj
@@ -1,7 +1,9 @@
(ns ^:mb/driver-tests metabase-enterprise.database-routing.e2e-test
(:require
[clojure.java.jdbc :as jdbc]
+ [clojure.set :as set]
[clojure.test :refer [deftest is testing]]
+ [clojurewerkz.quartzite.conversion :as qc]
[metabase-enterprise.test :as met]
[metabase.app-db.core :as mdb]
[metabase.driver :as driver]
@@ -9,6 +11,7 @@
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.query-processor :as qp]
[metabase.sync.core :as sync]
+ [metabase.sync.task.sync-databases :as task.sync-databases]
[metabase.test :as mt]
[metabase.test.data :as data]
[metabase.test.data.interface :as tx]
@@ -310,7 +313,23 @@
(is (= [[1 "routed-foo"] [2 "routed-bar"]]
(->> (mt/query t)
(mt/process-query)
- (mt/formatted-rows [int str])))))))))))))))))
+ (mt/formatted-rows [int str]))))))
+ (testing "sync task can see database"
+ (let [job-data (reify qc/JobDataMapConversion
+ ;; i'm doing this at the "job-data" level so it's as close to what runs in
+ ;; the task itself without actually hitting scheduler stuff.
+ (from-job-data [_] {"db-id" (u/the-id router)})
+ (to-job-data [_]))
+ results (#'task.sync-databases/sync-and-analyze-database! job-data)
+ step-with-name (fn [step-name]
+ (->> results :metadata-results :steps
+ (some (fn [step] (when (= step-name (first step)) (second step))))))]
+ (is (set/subset? #{"sync-fields" "sync-tables"}
+ (->> results :metadata-results :steps (map first) set)))
+ ;; this is usually 1 total tables, but some cloud dbs put multiple databases inside of a
+ ;; single catalog
+ (is (=? {:updated-tables 0 :total-tables pos-int?}
+ (step-with-name "sync-tables"))))))))))))))))
(deftest athena-region-bucket-routing-test
(mt/test-driver :athena
diff --git a/src/metabase/sync/task/sync_databases.clj b/src/metabase/sync/task/sync_databases.clj
index 27c1315ece9..ec8d46cb206 100644
--- a/src/metabase/sync/task/sync_databases.clj
+++ b/src/metabase/sync/task/sync_databases.clj
@@ -11,6 +11,7 @@
[java-time.api :as t]
[metabase.audit-app.core :as audit]
[metabase.config.core :as config]
+ [metabase.database-routing.core :as database-routing]
[metabase.driver.settings :as driver.settings]
[metabase.driver.util :as driver.u]
[metabase.lib.schema.id :as lib.schema.id]
@@ -86,13 +87,16 @@
(catch Throwable e
e))]
(log/warnf ex "Cannot sync Database %s: %s" (:name database) (ex-message ex))
- (do
- (sync-metadata/sync-db-metadata! database)
- ;; only run analysis if this is a "full sync" database
- (when (:is_full_sync database)
- (let [results (analyze/analyze-db! database)]
- (when (and (:refingerprint database) (should-refingerprint-fields? results))
- (analyze/refingerprint-db! database))))))))
+ (database-routing/with-database-routing-off
+ (let [metadata-results (sync-metadata/sync-db-metadata! database)
+ analyze-results (when (:is_full_sync database)
+ (analyze/analyze-db! database))
+ refingerprint-results (when (and (:refingerprint database)
+ (should-refingerprint-fields? analyze-results))
+ (analyze/refingerprint-db! database))]
+ (cond-> {:metadata-results metadata-results}
+ analyze-results (assoc :analyze-results analyze-results)
+ refingerprint-results (assoc :refingerprint-results refingerprint-results)))))))
(defn- sync-and-analyze-database!
"The sync and analyze database job, as a function that can be used in a test"