Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GHY-3274: Initial sync should not attempt to create field values (#82963)

Eric Normande8f85e28e06f
diff --git a/e2e/test/scenarios/monitor/tools/tools.cy.spec.ts b/e2e/test/scenarios/monitor/tools/tools.cy.spec.ts
index 3f5da30c7b1..806d8bdcc25 100644
--- a/e2e/test/scenarios/monitor/tools/tools.cy.spec.ts
+++ b/e2e/test/scenarios/monitor/tools/tools.cy.spec.ts
@@ -170,6 +170,8 @@ describe("issue 14636", () => {
const filteredTask = createMockTask({ task });
// Keep this test independent of background tasks created asynchronously by H.restore().
+ // The task picker only shows a value that is in the unique task list.
+ cy.intercept("GET", "/api/task/unique-tasks", [task]);
// Register the less-specific routes first because Cypress matches intercepts in reverse order.
stubFilteredResponse({
status: "success",
diff --git a/src/metabase/sync/sync.clj b/src/metabase/sync/sync.clj
index 73d2470f974..e5519f256ae 100644
--- a/src/metabase/sync/sync.clj
+++ b/src/metabase/sync/sync.clj
@@ -40,9 +40,16 @@
:analyze analyze/analyze-db!
:field-values sync.field-values/update-field-values!})
-(defn- scan-phases [scan]
- (if (not= :full scan)
+(defn- scan-phases [scan initial-sync-complete?]
+ (cond
+ (not= :full scan)
[:metadata]
+
+ ;; FieldValues are only created on demand, so a Database on its first sync has none to update or clear.
+ (not initial-sync-complete?)
+ [:metadata :analyze]
+
+ :else
[:metadata :analyze :field-values]))
(defn- do-phase! [database phase]
@@ -61,7 +68,8 @@
"Perform all the different sync operations synchronously for `database`.
By default, does a `:full` sync that performs all the different sync operations consecutively. You may instead
- specify only a `:schema` sync that will sync just the schema but skip analysis.
+ specify only a `:schema` sync that will sync just the schema but skip analysis. A `:full` sync skips the FieldValues
+ scan when the `:initial_sync_status` of `database` is not `\"complete\"`.
Please note that this function is *not* what is called by the scheduled tasks; those call different steps
independently. This function is called when a Database is first added."
@@ -73,7 +81,7 @@
[:scan {:optional true} [:maybe [:enum :schema :full]]]]]]
(tracing/with-span :sync "sync.database" {:db/id (:id database)}
(sync-util/sync-operation :sync database (format "Sync %s" (sync-util/name-for-logging database))
- (->> (scan-phases scan)
+ (->> (scan-phases scan (= "complete" (:initial_sync_status database)))
(keep (partial do-phase! database))
(doall))))))
diff --git a/test/metabase/driver_test.clj b/test/metabase/driver_test.clj
index 1a510e52843..716f99bfc9e 100644
--- a/test/metabase/driver_test.clj
+++ b/test/metabase/driver_test.clj
@@ -19,6 +19,7 @@
[metabase.test.data.interface :as tx]
[metabase.util :as u]
[metabase.util.json :as json]
+ [metabase.util.quick-task :as quick-task]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -161,8 +162,11 @@
(testing "sense check 1: sync-and-analyze-database! should not log a warning"
(is (false? (cant-sync-logged?))))
(testing "sense check 2: triggering the sync via the POST /api/database/:id/sync_schema endpoint should succeed"
- (is (= {:status "ok"}
- (mt/user-http-request :crowberto :post 200 (str "/database/" (u/the-id db) "/sync_schema"))))))
+ ;; run the endpoint's sync synchronously: if it is still running in the background, its connection
+ ;; can hold the database and make the destroy below fail
+ (mt/with-dynamic-fn-redefs [quick-task/submit-task! (fn [f] (f))]
+ (is (= {:status "ok"}
+ (mt/user-http-request :crowberto :post 200 (str "/database/" (u/the-id db) "/sync_schema")))))))
;; release db resources like connection pools so we don't have to wait to finish syncing before destroying the db
(driver/notify-database-updated driver/*driver* db)
;; destroy the db
diff --git a/test/metabase/sync/sync_test.clj b/test/metabase/sync/sync_test.clj
index d2ba281f179..c1f109f1d9f 100644
--- a/test/metabase/sync/sync_test.clj
+++ b/test/metabase/sync/sync_test.clj
@@ -253,7 +253,7 @@
(mt/with-temp [:model/Database db {:engine ::sync-test}]
(let [results (sync/sync-database! db)]
(testing "Returns results from sync-database step"
- (is (= ["metadata" "analyze" "field-values"]
+ (is (= ["metadata" "analyze"]
(map :name results)))))
(let [[movie studio] (mapv table-details (t2/select :model/Table :db_id (u/the-id db) {:order-by [:name]}))
;; a full sync runs the analyze step, which scores dimension_interestingness
@@ -325,13 +325,13 @@
(testing "New values come in after sync"
(binding [*execute-response* (fn [_query respond] (respond {:cols [{:name "field"}]}
(partition-all 1 ["d" "e" "f"])))]
- (sync/sync-database! db))
+ (sync/sync-database! (t2/select-one :model/Database (u/the-id db))))
(is (=? {:f {:name "title" :has_field_values :auto-list}
:fv {:values ["d" "e" "f"] :has_more_values false}}
(query-field-and-values))))
(testing "After setting to search it should stay search and sync removes field-values"
(t2/update! :model/Field (:id field) {:has_field_values "search"})
- (sync/sync-database! db)
+ (sync/sync-database! (t2/select-one :model/Database (u/the-id db)))
(get-or-create-vals ["x" "y" "z"])
(is (=? {:f {:name "title" :has_field_values :search}
:fv nil}
@@ -350,9 +350,23 @@
(binding [sync-util/*log-exceptions-and-continue?* true]
(let [results (sync/sync-database! db)]
(testing "Skips the metadata step"
- (is (= ["analyze" "field-values"]
+ (is (= ["analyze"]
(map :name results)))))))))
+(deftest sync-database!-skips-field-values-until-initial-sync-complete-test
+ (testing (str "GHY-3274: a full sync skips the FieldValues phase while the Database's initial sync is not "
+ "complete, because FieldValues are only created on demand and a new Database has none")
+ (binding [sync-util/*log-exceptions-and-continue?* false]
+ (mt/with-temp [:model/Database db {:engine ::sync-test, :initial_sync_status "incomplete"}]
+ (testing "the initial sync skips the field-values phase"
+ (is (= ["metadata" "analyze"]
+ (map :name (sync/sync-database! db)))))
+ (testing "a later sync of the Database, with its initial sync complete, runs it"
+ (let [synced-db (t2/select-one :model/Database (u/the-id db))]
+ (is (= "complete" (:initial_sync_status synced-db)))
+ (is (= ["metadata" "analyze" "field-values"]
+ (map :name (sync/sync-database! synced-db))))))))))
+
;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
;; !! !!
;; !! HEY! Your tests probably don't belong in this namespace! Put them in one appropriate to the specific part of !!
diff --git a/test/metabase/test/data/one_off_dbs.clj b/test/metabase/test/data/one_off_dbs.clj
index 6d02d24b2d0..f6a00202ef6 100644
--- a/test/metabase/test/data/one_off_dbs.clj
+++ b/test/metabase/test/data/one_off_dbs.clj
@@ -8,7 +8,8 @@
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.sync.core :as sync]
[metabase.test :as mt]
- [metabase.test.data :as data]))
+ [metabase.test.data :as data]
+ [toucan2.core :as t2]))
(def ^:dynamic *conn*
"Bound to a JDBC connection spec when using one of the `with-db` macros below."
@@ -78,4 +79,5 @@
(insert-rows-and-sync! [0 1 2 3]) ; insert 4 rows"
[values]
(jdbc/execute! *conn* [(insert-sql values)])
- (sync/sync-database! (data/db)))
+ ;; `(data/db)` is the map from before the first sync; its stale `initial_sync_status` would skip the FieldValues scan
+ (sync/sync-database! (t2/select-one :model/Database (data/id))))
diff --git a/test/metabase/warehouse_schema/models/field_values_test.clj b/test/metabase/warehouse_schema/models/field_values_test.clj
index 4d71d64ddf1..1d57e07b00d 100644
--- a/test/metabase/warehouse_schema/models/field_values_test.clj
+++ b/test/metabase/warehouse_schema/models/field_values_test.clj
@@ -200,7 +200,7 @@
(select-keys [:values :human_readable_values])))
(defn- sync-and-find-values! [db field-values-id]
- (sync/sync-database! db)
+ (sync/sync-database! (t2/select-one :model/Database (u/the-id db)))
(find-values field-values-id))
(deftest implicit-deduplication-test
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index f707aeea57f..3bf564bf9f4 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -1834,6 +1834,34 @@
(is (= :SEC_KEY (label "PEOPLE" "PASSWORD")))
(is (= :PUBLIC (label "ORDERS" "TOTAL")))))))))
+(deftest initial-sync-skips-field-values-scan-test
+ (testing (str "GHY-3274: the initial sync of a new Database must not scan FieldValues. A new Database has no "
+ "FieldValues (they are made on demand by the `/values` endpoints), so the scan can only issue a "
+ "useless per-Field clear/update attempt for every Field.")
+ (let [details (:details (mt/db))]
+ (mt/with-model-cleanup [:model/Database]
+ (let [db-id (binding [driver.settings/*allow-testing-h2-connections* true]
+ (mt/with-dynamic-fn-redefs [quick-task/submit-task! (fn [f]
+ (binding [driver.settings/*allow-testing-h2-connections* true]
+ (f)))]
+ (:id (mt/user-http-request :crowberto :post 200 "database"
+ {:name (mt/random-name)
+ :engine "h2"
+ :details details
+ :is_full_sync true}))))
+ task-runs (fn [task-name]
+ (->> (mt/user-http-request :crowberto :get 200 "task/" :task task-name :limit 1000)
+ :data
+ (filter #(= db-id (:db_id %)))))]
+ (testing "the initial sync ran"
+ (is (= "complete" (:initial_sync_status (mt/user-http-request :crowberto :get 200 (format "database/%d" db-id)))))
+ (is (seq (task-runs "sync-fields")))
+ (is (seq (task-runs "fingerprint-fields"))))
+ (testing "the initial sync did not scan FieldValues"
+ (is (empty? (task-runs "field values scanning")))
+ (is (empty? (task-runs "update-field-values")))
+ (is (empty? (task-runs "delete-expired-advanced-field-values")))))))))
+
(deftest sync-schema-executes-when-executor-busy-test
(testing "POST /api/database/:id/sync_schema should execute sync even when quick-task executor is busy (GHY-3254)"
(let [sync-called? (promise)