The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GHY-3274: Initial sync should not attempt to create field values (#82963)
Eric Normande8f85e28e06f
diff --git a/e2e/test/scenarios/monitor/tools/tools.cy.spec.ts b/e2e/test/scenarios/monitor/tools/tools.cy.spec.tsindex 3f5da30c7b1..806d8bdcc25 100644--- a/e2e/test/scenarios/monitor/tools/tools.cy.spec.ts+++ b/e2e/test/scenarios/monitor/tools/tools.cy.spec.ts@@ -170,6 +170,8 @@ describe("issue 14636", () => {const filteredTask = createMockTask({ task });// Keep this test independent of background tasks created asynchronously by H.restore().+ // The task picker only shows a value that is in the unique task list.+ cy.intercept("GET", "/api/task/unique-tasks", [task]);// Register the less-specific routes first because Cypress matches intercepts in reverse order.stubFilteredResponse({status: "success",diff --git a/src/metabase/sync/sync.clj b/src/metabase/sync/sync.cljindex 73d2470f974..e5519f256ae 100644--- a/src/metabase/sync/sync.clj+++ b/src/metabase/sync/sync.clj@@ -40,9 +40,16 @@:analyze analyze/analyze-db!:field-values sync.field-values/update-field-values!})-(defn- scan-phases [scan]- (if (not= :full scan)+(defn- scan-phases [scan initial-sync-complete?]+ (cond+ (not= :full scan)[:metadata]++ ;; FieldValues are only created on demand, so a Database on its first sync has none to update or clear.+ (not initial-sync-complete?)+ [:metadata :analyze]++ :else[:metadata :analyze :field-values]))(defn- do-phase! [database phase]@@ -61,7 +68,8 @@"Perform all the different sync operations synchronously for `database`.By default, does a `:full` sync that performs all the different sync operations consecutively. You may instead- specify only a `:schema` sync that will sync just the schema but skip analysis.+ specify only a `:schema` sync that will sync just the schema but skip analysis. A `:full` sync skips the FieldValues+ scan when the `:initial_sync_status` of `database` is not `\"complete\"`.Please note that this function is *not* what is called by the scheduled tasks; those call different stepsindependently. This function is called when a Database is first added."@@ -73,7 +81,7 @@[:scan {:optional true} [:maybe [:enum :schema :full]]]]]](tracing/with-span :sync "sync.database" {:db/id (:id database)}(sync-util/sync-operation :sync database (format "Sync %s" (sync-util/name-for-logging database))- (->> (scan-phases scan)+ (->> (scan-phases scan (= "complete" (:initial_sync_status database)))(keep (partial do-phase! database))(doall))))))diff --git a/test/metabase/driver_test.clj b/test/metabase/driver_test.cljindex 1a510e52843..716f99bfc9e 100644--- a/test/metabase/driver_test.clj+++ b/test/metabase/driver_test.clj@@ -19,6 +19,7 @@[metabase.test.data.interface :as tx][metabase.util :as u][metabase.util.json :as json]+ [metabase.util.quick-task :as quick-task][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -161,8 +162,11 @@(testing "sense check 1: sync-and-analyze-database! should not log a warning"(is (false? (cant-sync-logged?))))(testing "sense check 2: triggering the sync via the POST /api/database/:id/sync_schema endpoint should succeed"- (is (= {:status "ok"}- (mt/user-http-request :crowberto :post 200 (str "/database/" (u/the-id db) "/sync_schema"))))))+ ;; run the endpoint's sync synchronously: if it is still running in the background, its connection+ ;; can hold the database and make the destroy below fail+ (mt/with-dynamic-fn-redefs [quick-task/submit-task! (fn [f] (f))]+ (is (= {:status "ok"}+ (mt/user-http-request :crowberto :post 200 (str "/database/" (u/the-id db) "/sync_schema")))))));; release db resources like connection pools so we don't have to wait to finish syncing before destroying the db(driver/notify-database-updated driver/*driver* db);; destroy the dbdiff --git a/test/metabase/sync/sync_test.clj b/test/metabase/sync/sync_test.cljindex d2ba281f179..c1f109f1d9f 100644--- a/test/metabase/sync/sync_test.clj+++ b/test/metabase/sync/sync_test.clj@@ -253,7 +253,7 @@(mt/with-temp [:model/Database db {:engine ::sync-test}](let [results (sync/sync-database! db)](testing "Returns results from sync-database step"- (is (= ["metadata" "analyze" "field-values"]+ (is (= ["metadata" "analyze"](map :name results)))))(let [[movie studio] (mapv table-details (t2/select :model/Table :db_id (u/the-id db) {:order-by [:name]}));; a full sync runs the analyze step, which scores dimension_interestingness@@ -325,13 +325,13 @@(testing "New values come in after sync"(binding [*execute-response* (fn [_query respond] (respond {:cols [{:name "field"}]}(partition-all 1 ["d" "e" "f"])))]- (sync/sync-database! db))+ (sync/sync-database! (t2/select-one :model/Database (u/the-id db))))(is (=? {:f {:name "title" :has_field_values :auto-list}:fv {:values ["d" "e" "f"] :has_more_values false}}(query-field-and-values))))(testing "After setting to search it should stay search and sync removes field-values"(t2/update! :model/Field (:id field) {:has_field_values "search"})- (sync/sync-database! db)+ (sync/sync-database! (t2/select-one :model/Database (u/the-id db)))(get-or-create-vals ["x" "y" "z"])(is (=? {:f {:name "title" :has_field_values :search}:fv nil}@@ -350,9 +350,23 @@(binding [sync-util/*log-exceptions-and-continue?* true](let [results (sync/sync-database! db)](testing "Skips the metadata step"- (is (= ["analyze" "field-values"]+ (is (= ["analyze"](map :name results)))))))))+(deftest sync-database!-skips-field-values-until-initial-sync-complete-test+ (testing (str "GHY-3274: a full sync skips the FieldValues phase while the Database's initial sync is not "+ "complete, because FieldValues are only created on demand and a new Database has none")+ (binding [sync-util/*log-exceptions-and-continue?* false]+ (mt/with-temp [:model/Database db {:engine ::sync-test, :initial_sync_status "incomplete"}]+ (testing "the initial sync skips the field-values phase"+ (is (= ["metadata" "analyze"]+ (map :name (sync/sync-database! db)))))+ (testing "a later sync of the Database, with its initial sync complete, runs it"+ (let [synced-db (t2/select-one :model/Database (u/the-id db))]+ (is (= "complete" (:initial_sync_status synced-db)))+ (is (= ["metadata" "analyze" "field-values"]+ (map :name (sync/sync-database! synced-db))))))))))+;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!;; !! !!;; !! HEY! Your tests probably don't belong in this namespace! Put them in one appropriate to the specific part of !!diff --git a/test/metabase/test/data/one_off_dbs.clj b/test/metabase/test/data/one_off_dbs.cljindex 6d02d24b2d0..f6a00202ef6 100644--- a/test/metabase/test/data/one_off_dbs.clj+++ b/test/metabase/test/data/one_off_dbs.clj@@ -8,7 +8,8 @@[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.sync.core :as sync][metabase.test :as mt]- [metabase.test.data :as data]))+ [metabase.test.data :as data]+ [toucan2.core :as t2]))(def ^:dynamic *conn*"Bound to a JDBC connection spec when using one of the `with-db` macros below."@@ -78,4 +79,5 @@(insert-rows-and-sync! [0 1 2 3]) ; insert 4 rows"[values](jdbc/execute! *conn* [(insert-sql values)])- (sync/sync-database! (data/db)))+ ;; `(data/db)` is the map from before the first sync; its stale `initial_sync_status` would skip the FieldValues scan+ (sync/sync-database! (t2/select-one :model/Database (data/id))))diff --git a/test/metabase/warehouse_schema/models/field_values_test.clj b/test/metabase/warehouse_schema/models/field_values_test.cljindex 4d71d64ddf1..1d57e07b00d 100644--- a/test/metabase/warehouse_schema/models/field_values_test.clj+++ b/test/metabase/warehouse_schema/models/field_values_test.clj@@ -200,7 +200,7 @@(select-keys [:values :human_readable_values])))(defn- sync-and-find-values! [db field-values-id]- (sync/sync-database! db)+ (sync/sync-database! (t2/select-one :model/Database (u/the-id db)))(find-values field-values-id))(deftest implicit-deduplication-testdiff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex f707aeea57f..3bf564bf9f4 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -1834,6 +1834,34 @@(is (= :SEC_KEY (label "PEOPLE" "PASSWORD")))(is (= :PUBLIC (label "ORDERS" "TOTAL")))))))))+(deftest initial-sync-skips-field-values-scan-test+ (testing (str "GHY-3274: the initial sync of a new Database must not scan FieldValues. A new Database has no "+ "FieldValues (they are made on demand by the `/values` endpoints), so the scan can only issue a "+ "useless per-Field clear/update attempt for every Field.")+ (let [details (:details (mt/db))]+ (mt/with-model-cleanup [:model/Database]+ (let [db-id (binding [driver.settings/*allow-testing-h2-connections* true]+ (mt/with-dynamic-fn-redefs [quick-task/submit-task! (fn [f]+ (binding [driver.settings/*allow-testing-h2-connections* true]+ (f)))]+ (:id (mt/user-http-request :crowberto :post 200 "database"+ {:name (mt/random-name)+ :engine "h2"+ :details details+ :is_full_sync true}))))+ task-runs (fn [task-name]+ (->> (mt/user-http-request :crowberto :get 200 "task/" :task task-name :limit 1000)+ :data+ (filter #(= db-id (:db_id %)))))]+ (testing "the initial sync ran"+ (is (= "complete" (:initial_sync_status (mt/user-http-request :crowberto :get 200 (format "database/%d" db-id)))))+ (is (seq (task-runs "sync-fields")))+ (is (seq (task-runs "fingerprint-fields"))))+ (testing "the initial sync did not scan FieldValues"+ (is (empty? (task-runs "field values scanning")))+ (is (empty? (task-runs "update-field-values")))+ (is (empty? (task-runs "delete-expired-advanced-field-values")))))))))+(deftest sync-schema-executes-when-executor-busy-test(testing "POST /api/database/:id/sync_schema should execute sync even when quick-task executor is busy (GHY-3254)"(let [sync-called? (promise)