The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fix incremental transform tests (#68470)
Oleksandr Solovyov0880e7297d2c
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.cljindex 8f32f1a2262..b9cf55a5647 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj@@ -183,9 +183,9 @@result (qp/process-query native-query)](some-> result :data :rows first first bigint)))-(defn get-checkpoint-value [transform-id]+(defn get-checkpoint-value [transform](#'transforms.u/next-checkpoint-value- (transforms.u/next-checkpoint transform-id)))+ (transforms.u/next-checkpoint transform)))(defn- compare-checkpoint-values"Compare two checkpoint values with type-appropriate logic. "@@ -258,7 +258,7 @@(is (= field-name (-> transform :source :source-incremental-strategy :checkpoint-filter)))(testing "No checkpoint exists initially"- (is (nil? (get-checkpoint-value (:id transform)))))+ (is (nil? (get-checkpoint-value transform))))(testing "Can retrieve transform via API"(let [retrieved (mt/user-http-request :crowberto :get 200 (format "ee/transform/%d" (:id transform)))]@@ -294,7 +294,7 @@(is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")(testing "Checkpoint is created after first run"- (let [checkpoint (get-checkpoint-value (:id transform))]+ (let [checkpoint (get-checkpoint-value transform)](is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint)(format "Checkpoint should be MAX(%s) from first 10 rows" (:field-name checkpoint-config)))))))@@ -304,7 +304,7 @@distinct-timestamps (get-distinct-timestamp-count target-table)](is (= 16 row-count) "Second run should add remaining 6 rows")(is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")- (let [checkpoint (get-checkpoint-value (:id transform))]+ (let [checkpoint (get-checkpoint-value transform)](is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)(format "Checkpoint should be MAX(%s) from all 16 rows" (:field-name checkpoint-config))))))))))))))))@@ -327,7 +327,7 @@(transforms.tu/wait-for-table target-table 10000)(let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)- checkpoint (get-checkpoint-value (:id transform))]+ checkpoint (get-checkpoint-value transform)](is (= 10 row-count) "Initial run should process first 10 products")(is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")(is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint) "Checkpoint should be created")))@@ -391,7 +391,7 @@(is (= 1 distinct-timestamps) "All rows should have same timestamp from non-incremental run")(testing "No checkpoint exists"- (let [checkpoint (get-checkpoint-value (:id transform))]+ (let [checkpoint (get-checkpoint-value transform)](is (nil? checkpoint) "No checkpoint for non-incremental transform")))))(testing "Switch to incremental via PUT API"@@ -405,7 +405,7 @@(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})(let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)- checkpoint (get-checkpoint-value (:id transform))]+ checkpoint (get-checkpoint-value transform)](is (= 16 row-count) "Should process remaining 6 entries")(is (= 2 distinct-timestamps) "Should have 2 distinct timestamp")(is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint) "Checkpoint should be computed from existing data"))))@@ -421,7 +421,7 @@(let [transform (t2/select-one :model/Transform (:id transform))](execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})(let [row-count (get-table-row-count target-table)- checkpoint (get-checkpoint-value (:id transform))]+ checkpoint (get-checkpoint-value transform)](is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)");; For integer checkpoints, we can verify exact value >= 17;; For float/temporal, just verify checkpoint exists@@ -453,7 +453,7 @@(transforms.execute/execute! transform {:run-method :manual})(transforms.tu/wait-for-table target-table 10000)(let [row-count (get-table-row-count target-table)- checkpoint (get-checkpoint-value (:id transform))]+ checkpoint (get-checkpoint-value transform)](is (= 16 row-count) "First run should process all 16 products")(is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)(format "Checkpoint should be MAX(%s) = %s" (:field-name checkpoint-config) expected-second-checkpoint))))@@ -477,7 +477,7 @@(transforms.execute/execute! transform {:run-method :manual})(let [row-count (get-table-row-count target-table)- checkpoint (get-checkpoint-value (:id transform))]+ checkpoint (get-checkpoint-value transform)](is (= 18 row-count) "Should append 2 new rows (16 + 2 = 18)")(is (some? checkpoint) "Checkpoint should be updated")))))))))))))))@@ -522,7 +522,7 @@(transforms.tu/wait-for-table target-table 10000)(let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)- checkpoint (get-checkpoint-value (:id transform))]+ checkpoint (get-checkpoint-value transform)](is (= 10 row-count) "First run should process the first 10 products")(is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")(is (compare-checkpoint-values :temporal expected-initial-checkpoint checkpoint)@@ -532,7 +532,7 @@(transforms.execute/execute! transform {:run-method :manual})(let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)- checkpoint (get-checkpoint-value (:id transform))]+ checkpoint (get-checkpoint-value transform)](is (= 16 row-count) "Second run should add remaining 6 rows")(is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")(is (compare-checkpoint-values :temporal expected-second-checkpoint checkpoint)@@ -552,7 +552,7 @@(transforms.execute/execute! transform {:run-method :manual})(let [row-count (get-table-row-count target-table)- checkpoint (get-checkpoint-value (:id transform))]+ checkpoint (get-checkpoint-value transform)](is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")(is (some? checkpoint) "Checkpoint should be updated"))))))))))))@@ -582,7 +582,7 @@(testing "sync has picked up table"(is (=? {:name target-table, :fields [{:name "id"}]} (-> (t2/select-one :model/Table :name target-table) (t2/hydrate :fields)))))(testing "checkpoint is recognized"- (is (some? (transforms.u/next-checkpoint (:id transform)))))))))))+ (is (some? (transforms.u/next-checkpoint transform))))))))))(deftest checkpoint-field-does-not-exist-test(mt/test-drivers #{:postgres} ; no db specifics@@ -609,7 +609,7 @@(testing "target table has expected data"(is (= [{:id 42}] (pg-table-rows db-spec target-table))))(testing "checkpoint is not recognized, so transform acts as if no checkpoint"- (is (nil? (transforms.u/next-checkpoint (:id transform)))))+ (is (nil? (transforms.u/next-checkpoint transform))));; Maybe this is unrealistic - you cannot select a column that does not exist;; But the source tables schema can change, e.g. rename: you change event_time to event_ts or something;; so one would have to be careful to disable or delete transforms ahead of a schema change like this