The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Temporarily remove belize test and create-replace-table test in snowflake (#75830)
technomancyb0da250e6970
diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.cljindex 314ebea82f3..54c92b1b2fa 100644--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj@@ -483,184 +483,188 @@[(format "DROP TABLE IF EXISTS \"%s\".\"PUBLIC\".\"%s\";" db-name table-name)]{:transaction? false})))))))))-;; Five related repro phases (v1-v5) share Snowflake connection setup and table-name lifecycle.-;; Splitting would multiply CI Snowflake setup cost. Kondo's warning is acknowledged.-^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}-(deftest ^:sequential ^:synchronized create-or-replace-table-updates-effective-type-test- (mt/test-driver :snowflake- (testing "GHY-3388: when a column's database type changes via CREATE OR REPLACE TABLE+;; marking this as quarantined is not enough; when run in CI it frequently causes+;; other unrelated tests to fail for reasons that are unclear. Disabling this+;; and the Belize test makes the unreliability go away.+(comment+ ;; Five related repro phases (v1-v5) share Snowflake connection setup and table-name lifecycle.+ ;; Splitting would multiply CI Snowflake setup cost. Kondo's warning is acknowledged.+ ^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}+ (deftest ^:sequential ^:synchronized create-or-replace-table-updates-effective-type-test+ (mt/test-driver :snowflake+ (testing "GHY-3388: when a column's database type changes via CREATE OR REPLACE TABLE(e.g. TEXT -> NUMBER via TRY_TO_NUMBER), sync should update effective_type to matchthe new base_type and clear stale coercion_strategy/semantic_type. Reproduces thein-place update path (repro v1 from the Linear issue) where Metabase's Field row ispreserved across the schema change."- (let [db-name (#'driver.snowflake/db-name (mt/db))- table-name (str "ghy_3388_" (u.random/random-name))- run-sql! (fn [stmts]- (sql-jdbc.execute/do-with-connection-with-options- :snowflake- (mt/db)- nil- (fn [^java.sql.Connection conn]- (doseq [stmt stmts]- (jdbc/execute! {:connection conn} [stmt] {:transaction? false})))))- qualified (format "\"%s\".\"PUBLIC\".\"%s\"" db-name table-name)]- (try- ;; step 1: create the table with a TEXT column and insert rows. Quote identifiers so they- ;; are preserved lowercase (Snowflake upper-cases unquoted identifiers).- (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)- (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])- ;; step 2: sync — Metabase creates Table + Field rows for our new table- (sync/sync-database! (mt/db))- (let [original-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)- _ (is (some? original-table) "table should be synced")- original-field (t2/select-one :model/Field :table_id (:id original-table) :name "text_column")]- (testing "sanity check: text_column is text"- (is (=? {:base_type :type/Text- :effective_type :type/Text}- original-field)))- ;; step 3: CREATE OR REPLACE the table, converting text_column to a number via TRY_TO_NUMBER+ (let [db-name (#'driver.snowflake/db-name (mt/db))+ table-name (str "ghy_3388_" (u.random/random-name))+ run-sql! (fn [stmts]+ (sql-jdbc.execute/do-with-connection-with-options+ :snowflake+ (mt/db)+ nil+ (fn [^java.sql.Connection conn]+ (doseq [stmt stmts]+ (jdbc/execute! {:connection conn} [stmt] {:transaction? false})))))+ qualified (format "\"%s\".\"PUBLIC\".\"%s\"" db-name table-name)]+ (try+ ;; step 1: create the table with a TEXT column and insert rows. Quote identifiers so they+ ;; are preserved lowercase (Snowflake upper-cases unquoted identifiers).+ (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)+ (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])+ ;; step 2: sync — Metabase creates Table + Field rows for our new table+ (sync/sync-database! (mt/db))+ (let [original-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)+ _ (is (some? original-table) "table should be synced")+ original-field (t2/select-one :model/Field :table_id (:id original-table) :name "text_column")]+ (testing "sanity check: text_column is text"+ (is (=? {:base_type :type/Text+ :effective_type :type/Text}+ original-field)))+ ;; step 3: CREATE OR REPLACE the table, converting text_column to a number via TRY_TO_NUMBER+ (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "+ "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")+ qualified qualified)])+ ;; step 4: sync again+ (sync/sync-database! (mt/db))+ (let [new-field (t2/select-one :model/Field :id (:id original-field))]+ (testing "after sync, base_type and effective_type both reflect the new numeric column,+ and stale coercion/semantic type are cleared"+ ;; TRY_TO_NUMBER with no precision returns NUMBER(38,0) -> :type/BigInteger+ (is (=? {:base_type :type/BigInteger+ :effective_type :type/BigInteger+ :coercion_strategy nil+ :semantic_type nil}+ new-field))))+ ;; ----- repro v2 (clean state baseline): drop the table, wipe Metabase metadata for it,+ ;; recreate fresh as TEXT, sync, then CREATE OR REPLACE to numeric and sync again. The+ ;; field should end up with effective_type matching base_type. (In the Linear repro the+ ;; user observes the cast feature does not appear here — i.e. effective_type vs base_type+ ;; should be consistent and not stuck as text.)+ (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])+ (t2/delete! :model/Field :table_id (:id original-table))+ (t2/delete! :model/Table :id (:id original-table)))+ (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)+ (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])+ (sync/sync-database! (mt/db))(run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "- "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")+ "SELECT \"id\", TRY_TO_NUMBER(\"text_column\", 38, 2) AS \"text_column\" FROM %s;")qualified qualified)])- ;; step 4: sync again(sync/sync-database! (mt/db))- (let [new-field (t2/select-one :model/Field :id (:id original-field))]- (testing "after sync, base_type and effective_type both reflect the new numeric column,- and stale coercion/semantic type are cleared"- ;; TRY_TO_NUMBER with no precision returns NUMBER(38,0) -> :type/BigInteger- (is (=? {:base_type :type/BigInteger- :effective_type :type/BigInteger+ (let [fresh-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)+ fresh-field (t2/select-one :model/Field :table_id (:id fresh-table) :name "text_column")]+ (testing "after fresh-state CREATE OR REPLACE to numeric, base_type and effective_type+ both reflect the new numeric column"+ ;; TRY_TO_NUMBER(x, 38, 2) returns NUMBER(38,2) -> :type/Number on Snowflake+ (is (=? {:base_type :type/Number+ :effective_type :type/Number:coercion_strategy nil:semantic_type nil}- new-field))))- ;; ----- repro v2 (clean state baseline): drop the table, wipe Metabase metadata for it,- ;; recreate fresh as TEXT, sync, then CREATE OR REPLACE to numeric and sync again. The- ;; field should end up with effective_type matching base_type. (In the Linear repro the- ;; user observes the cast feature does not appear here — i.e. effective_type vs base_type- ;; should be consistent and not stuck as text.)- (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])- (t2/delete! :model/Field :table_id (:id original-table))- (t2/delete! :model/Table :id (:id original-table)))- (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)- (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])- (sync/sync-database! (mt/db))- (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "- "SELECT \"id\", TRY_TO_NUMBER(\"text_column\", 38, 2) AS \"text_column\" FROM %s;")- qualified qualified)])- (sync/sync-database! (mt/db))- (let [fresh-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)- fresh-field (t2/select-one :model/Field :table_id (:id fresh-table) :name "text_column")]- (testing "after fresh-state CREATE OR REPLACE to numeric, base_type and effective_type- both reflect the new numeric column"- ;; TRY_TO_NUMBER(x, 38, 2) returns NUMBER(38,2) -> :type/Number on Snowflake- (is (=? {:base_type :type/Number- :effective_type :type/Number- :coercion_strategy nil- :semantic_type nil}- fresh-field)))- ;; ----- repro v3: simulate the user setting a coercion strategy on the TEXT column- ;; (via the UI cast-toggle) before the underlying type changes. When the DB column- ;; becomes numeric natively, sync should clear the now-meaningless coercion strategy- ;; and reset effective_type to match the new base_type. If sync leaves the coercion- ;; sticky, effective_type will diverge from base_type — the observable GHY-3388 symptom.- (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])- (t2/delete! :model/Field :table_id (:id fresh-table))- (t2/delete! :model/Table :id (:id fresh-table)))- (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)- (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])- (sync/sync-database! (mt/db))- (let [pre-coerce-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)- pre-coerce-field (t2/select-one :model/Field :table_id (:id pre-coerce-table) :name "text_column")]- ;; user enables coercion: this TEXT column is really an integer- (t2/update! :model/Field (:id pre-coerce-field)- {:coercion_strategy :Coercion/String->Integer- :effective_type :type/Integer})- (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "- "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")- qualified qualified)])+ fresh-field)))+ ;; ----- repro v3: simulate the user setting a coercion strategy on the TEXT column+ ;; (via the UI cast-toggle) before the underlying type changes. When the DB column+ ;; becomes numeric natively, sync should clear the now-meaningless coercion strategy+ ;; and reset effective_type to match the new base_type. If sync leaves the coercion+ ;; sticky, effective_type will diverge from base_type — the observable GHY-3388 symptom.+ (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])+ (t2/delete! :model/Field :table_id (:id fresh-table))+ (t2/delete! :model/Table :id (:id fresh-table)))+ (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)+ (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])(sync/sync-database! (mt/db))- (let [after-field (t2/select-one :model/Field :id (:id pre-coerce-field))]- (testing "after the underlying TEXT column becomes a native number, the previously+ (let [pre-coerce-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)+ pre-coerce-field (t2/select-one :model/Field :table_id (:id pre-coerce-table) :name "text_column")]+ ;; user enables coercion: this TEXT column is really an integer+ (t2/update! :model/Field (:id pre-coerce-field)+ {:coercion_strategy :Coercion/String->Integer+ :effective_type :type/Integer})+ (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "+ "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")+ qualified qualified)])+ (sync/sync-database! (mt/db))+ (let [after-field (t2/select-one :model/Field :id (:id pre-coerce-field))]+ (testing "after the underlying TEXT column becomes a native number, the previouslyuser-set coercion strategy should be cleared and effective_type shouldmatch the new base_type"- (is (=? {:base_type :type/BigInteger- :effective_type :type/BigInteger- :coercion_strategy nil- :semantic_type nil}- after-field)))))- ;; ----- repro v4: set coercion the way the PUT /api/field/:id endpoint does:- ;; via upsert-user-settings, which writes to metabase_field_user_settings. That- ;; row is overlaid back onto the field on every t2/update! via the- ;; sync-user-settings before-update hook (merge non-nil user-settings over- ;; field changes). v3 wrote only to metabase_field and so bypassed this- ;; overlay; v4 exercises the real UI cast-toggle path.- (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])- (when-let [stale-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)]- (t2/delete! :model/Field :table_id (:id stale-table))- (t2/delete! :model/Table :id (:id stale-table)))- (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)- (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])- (sync/sync-database! (mt/db))- (let [v4-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)- v4-field (t2/select-one :model/Field :table_id (:id v4-table) :name "text_column")]- (field-user-settings/upsert-user-settings v4-field- {:coercion_strategy :Coercion/String->Integer- :effective_type :type/Integer})- (t2/update! :model/Field (:id v4-field)- {:coercion_strategy :Coercion/String->Integer- :effective_type :type/Integer})- (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "- "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")- qualified qualified)])+ (is (=? {:base_type :type/BigInteger+ :effective_type :type/BigInteger+ :coercion_strategy nil+ :semantic_type nil}+ after-field)))))+ ;; ----- repro v4: set coercion the way the PUT /api/field/:id endpoint does:+ ;; via upsert-user-settings, which writes to metabase_field_user_settings. That+ ;; row is overlaid back onto the field on every t2/update! via the+ ;; sync-user-settings before-update hook (merge non-nil user-settings over+ ;; field changes). v3 wrote only to metabase_field and so bypassed this+ ;; overlay; v4 exercises the real UI cast-toggle path.+ (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])+ (when-let [stale-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)]+ (t2/delete! :model/Field :table_id (:id stale-table))+ (t2/delete! :model/Table :id (:id stale-table)))+ (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)+ (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])(sync/sync-database! (mt/db))- (let [after-field (t2/select-one :model/Field :id (:id v4-field))- after-user-settings (t2/select-one :model/FieldUserSettings :field_id (:id v4-field))]- (testing "after CREATE OR REPLACE to numeric, the user-set coercion stored in+ (let [v4-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)+ v4-field (t2/select-one :model/Field :table_id (:id v4-table) :name "text_column")]+ (field-user-settings/upsert-user-settings v4-field+ {:coercion_strategy :Coercion/String->Integer+ :effective_type :type/Integer})+ (t2/update! :model/Field (:id v4-field)+ {:coercion_strategy :Coercion/String->Integer+ :effective_type :type/Integer})+ (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "+ "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")+ qualified qualified)])+ (sync/sync-database! (mt/db))+ (let [after-field (t2/select-one :model/Field :id (:id v4-field))+ after-user-settings (t2/select-one :model/FieldUserSettings :field_id (:id v4-field))]+ (testing "after CREATE OR REPLACE to numeric, the user-set coercion stored inmetabase_field_user_settings (via upsert-user-settings) should becleared so it doesn't overlay back on top of the synced field"- (is (=? {:base_type :type/BigInteger- :effective_type :type/BigInteger- :coercion_strategy nil- :semantic_type nil}- after-field))- (is (=? {:effective_type :type/BigInteger- :coercion_strategy nil- :semantic_type nil}- after-user-settings)))))- ;; ----- repro v5: same as v1 (in-place CREATE OR REPLACE TEXT -> NUMBER) but trigger- ;; sync via sync/sync-table! — the function the per-table "Sync table now" UI button- ;; calls. Goes through sync-table-metadata! -> sync-fields-for-table! instead of- ;; sync-db-metadata! -> sync-fields!. Both reach the same sync-and-update! core, but- ;; the OP's repro instructions said "Now sync the table" (singular), so this rules- ;; out a code-path-specific bug at the entry point.- (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])- (when-let [stale-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)]- (t2/delete! :model/Field :table_id (:id stale-table))- (t2/delete! :model/Table :id (:id stale-table)))- (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)- (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])- (sync/sync-database! (mt/db))- (let [v5-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)- v5-field (t2/select-one :model/Field :table_id (:id v5-table) :name "text_column")]- (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "- "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")- qualified qualified)])- (sync/sync-table! v5-table)- (let [after-field (t2/select-one :model/Field :id (:id v5-field))]- (testing "sync-table! (the per-table UI button) should also reset effective_type+ (is (=? {:base_type :type/BigInteger+ :effective_type :type/BigInteger+ :coercion_strategy nil+ :semantic_type nil}+ after-field))+ (is (=? {:effective_type :type/BigInteger+ :coercion_strategy nil+ :semantic_type nil}+ after-user-settings)))))+ ;; ----- repro v5: same as v1 (in-place CREATE OR REPLACE TEXT -> NUMBER) but trigger+ ;; sync via sync/sync-table! — the function the per-table "Sync table now" UI button+ ;; calls. Goes through sync-table-metadata! -> sync-fields-for-table! instead of+ ;; sync-db-metadata! -> sync-fields!. Both reach the same sync-and-update! core, but+ ;; the OP's repro instructions said "Now sync the table" (singular), so this rules+ ;; out a code-path-specific bug at the entry point.+ (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])+ (when-let [stale-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)]+ (t2/delete! :model/Field :table_id (:id stale-table))+ (t2/delete! :model/Table :id (:id stale-table)))+ (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)+ (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])+ (sync/sync-database! (mt/db))+ (let [v5-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)+ v5-field (t2/select-one :model/Field :table_id (:id v5-table) :name "text_column")]+ (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "+ "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")+ qualified qualified)])+ (sync/sync-table! v5-table)+ (let [after-field (t2/select-one :model/Field :id (:id v5-field))]+ (testing "sync-table! (the per-table UI button) should also reset effective_typewhen the underlying column type changes"- (is (=? {:base_type :type/BigInteger- :effective_type :type/BigInteger- :coercion_strategy nil- :semantic_type nil}- after-field)))))- (finally- (let [t (t2/select-one :model/Table :db_id (mt/id) :name table-name)]- (when t- (u/ignore-exceptions (t2/delete! :model/Field :table_id (:id t)))- (u/ignore-exceptions (t2/delete! :model/Table :id (:id t)))))- (u/ignore-exceptions- (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)]))))))))+ (is (=? {:base_type :type/BigInteger+ :effective_type :type/BigInteger+ :coercion_strategy nil+ :semantic_type nil}+ after-field)))))+ (finally+ (let [t (t2/select-one :model/Table :db_id (mt/id) :name table-name)]+ (when t+ (u/ignore-exceptions (t2/delete! :model/Field :table_id (:id t)))+ (u/ignore-exceptions (t2/delete! :model/Table :id (:id t)))))+ (u/ignore-exceptions+ (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])))))))))(deftest ^:sequential describe-table-test(mt/test-driver :snowflake@@ -1059,7 +1063,7 @@(is (= [expected](mt/first-row (qp/process-query query))))))))))-(deftest ^:parallel local-date-time-parameter-test+(deftest local-date-time-parameter-test(test-temporal-instance#t "2024-04-25T14:44:00""2024-04-25T14:44:00Z"))@@ -1317,125 +1321,130 @@;;;; (In Belize they know no DST anymore.);;;;-(def ^:private belize-offset (t/zone-offset "-06:00"))--(defn- rows-for-good-datetimes-in-belize- []- (let [number-of-points (* 4 3)- today-dt (t/truncate-to (t/offset-date-time belize-offset) :days)- first-dt-point (t/- today-dt (t/days 2))- dt-points (for [i (range number-of-points)]- (t/+ first-dt-point (t/hours (* 6 i))))- various-offset-strs ["-10:00" "-04:00" "+02:00" "+09:00"]]- (mapv (fn [today-dt offset-str]- (vector (t/with-offset-same-instant today-dt (t/zone-offset "Z"))- (t/with-offset-same-instant today-dt (t/zone-offset offset-str))- (t/local-date-time today-dt)+;; marking this as quarantined is not enough; when run in CI it frequently causes+;; other unrelated tests to fail for reasons that are unclear. Disabling this+;; and the create-or-replace-table-updates-effective-type-test test makes the+;; unreliability go away.+(comment+ (def ^:private belize-offset (t/zone-offset "-06:00"))++ (defn- rows-for-good-datetimes-in-belize+ []+ (let [number-of-points (* 4 3)+ today-dt (t/truncate-to (t/offset-date-time belize-offset) :days)+ first-dt-point (t/- today-dt (t/days 2))+ dt-points (for [i (range number-of-points)]+ (t/+ first-dt-point (t/hours (* 6 i))))+ various-offset-strs ["-10:00" "-04:00" "+02:00" "+09:00"]]+ (mapv (fn [today-dt offset-str]+ (vector (t/with-offset-same-instant today-dt (t/zone-offset "Z"))+ (t/with-offset-same-instant today-dt (t/zone-offset offset-str))+ (t/local-date-time today-dt);;;; Shift local date time for belize offset so timestamp_ltz has same instant as rest!- ;; Even though later in the test the user with America/Belize timezone does the data loading,- ;; the timestamps have to be adjusted.- ;; I believe that it is because of either (1) us hardcoding the :timestamp connection property- ;; to UTC (see the `connection-details->spec :snowflake`) or (2) the fact that the JVM timezone- ;; is set to UTC.- (t/+ (t/local-date-time today-dt) (t/hours 6))))- dt-points- (cycle various-offset-strs))))--;; BEWARE: No cleanup is done atm. It is expected that every CI run creates its own instance of this database.-(mt/defdataset good-datetimes-in-belize- [["GOOD_DATETIMES" [{:field-name "IN_Z_OFFSET"- :base-type {:native "timestamptz"}}- {:field-name "IN_VARIOUS_OFFSETS"- :base-type {:native "timestamptz"}}- {:field-name "JUST_NTZ"- :base-type {:native "timestampntz"}}- {:field-name "JUST_LTZ"- :base-type {:native "timestampltz"}}]- (rows-for-good-datetimes-in-belize)]])--(deftest ^:parallel sync-datetime-types- (mt/test-driver- :snowflake- (mt/dataset- good-datetimes-in-belize- (is (= [["id" "NUMBER" :type/BigInteger 0]- ["IN_Z_OFFSET" "TIMESTAMPTZ" :type/DateTimeWithLocalTZ 1]- ["IN_VARIOUS_OFFSETS" "TIMESTAMPTZ" :type/DateTimeWithLocalTZ 2]- ["JUST_NTZ" "TIMESTAMPNTZ" :type/DateTime 3]- ["JUST_LTZ" "TIMESTAMPLTZ" :type/DateTimeWithTZ 4]]- (sort-by last- (into []- (map (juxt :name :database-type :base-type :database-position))- (fetch-metadata/fields-metadata (mt/db)))))))))--;; The test needs user with no report timezone set and database timezone other than UTC. That's the reason for redefs-;; prior to dataset generation.-(deftest ^:synchronized correct-timestamp-type-querying-test- (mt/test-driver- :snowflake- (let [original-set-current-user-timezone! @#'test.data.snowflake/set-current-user-timezone!- original-dbdef->connection-details (get-method tx/dbdef->connection-details :snowflake)]- ;; load dataset as the default user before we switch to the belize-based- ;; user who may not have permission to create the dataset- (mt/dataset good-datetimes-in-belize- (mt/id :GOOD_DATETIMES))- (with-redefs [test.data.snowflake/set-current-user-timezone!- (fn [_timezone]- (original-set-current-user-timezone! "America/Belize"))- tx/dbdef->connection-details- (fn [driver connection-type database-definition]- (-> (original-dbdef->connection-details driver connection-type database-definition)- (assoc :user "BELIZE_PERSON")))]- (mt/with-temporary-setting-values [report-timezone "America/Belize"]- (mt/dataset- good-datetimes-in-belize- (testing "Expected data is returned using yesterday filter"- (let [yesterday-first (t/- (t/truncate-to (t/offset-date-time belize-offset) :days) (t/days 1))- yesterday-last (t/+ yesterday-first (t/hours 18))- yesterday-first-str (t/format :iso-offset-date-time yesterday-first)- yesterday-last-str (t/format :iso-offset-date-time yesterday-last)]- (doseq [[tested-field-kw base-type database-type]- [[:IN_Z_OFFSET :type/DateTimeWithLocalTZ "timestamptz"]- [:IN_VARIOUS_OFFSETS :type/DateTimeWithLocalTZ "timestamptz"]- [:JUST_NTZ :type/DateTime "timestampntz"]- [:JUST_LTZ :type/DateTimeWithTZ "timestampltz"]]- :let [tested-field [:field (mt/id :GOOD_DATETIMES tested-field-kw) {:base-type base-type}]]]- (testing (str "on column type " database-type)- (let [rows (mt/rows (qp/process-query- {:database (mt/id)- :type :query- :query {:source-table (mt/id :GOOD_DATETIMES)- :fields [tested-field]- :filter [:time-interval tested-field -1 :day]- :order-by [[tested-field :asc]]}}))]- (testing "Correct rows count returned"- (is (= 4 (count rows))))- (testing "First row has expected values"- (is (= yesterday-first-str- (ffirst rows))))- (testing "Last row has expected values"- (is (= yesterday-last-str- (ffirst (reverse rows)))))))- (testing "Rows should be properly allocated to days"- (let [tested-day (assoc-in tested-field [2 :temporal-unit] :day)- tested-minute (assoc-in tested-field [2 :temporal-unit] :minute)]- (is (=? [[string? 4] [string? 4] [string? 4]]- (mt/rows- (qp/process-query- (mt/mbql-query nil- {:source-table (mt/id :GOOD_DATETIMES)- :aggregation [[:count]]- :breakout [tested-day]})))))- (is (= [[yesterday-first-str 4 yesterday-first-str yesterday-last-str]]- (mt/rows- (qp/process-query- (mt/mbql-query nil- {:source-table (mt/id :GOOD_DATETIMES)- :aggregation [[:count]- [:min tested-minute]- [:max tested-minute]]- :breakout [tested-day]- :filter [:= tested-field (t/format :iso-local-date yesterday-last)]}))))))))))))))))+ ;; Even though later in the test the user with America/Belize timezone does the data loading,+ ;; the timestamps have to be adjusted.+ ;; I believe that it is because of either (1) us hardcoding the :timestamp connection property+ ;; to UTC (see the `connection-details->spec :snowflake`) or (2) the fact that the JVM timezone+ ;; is set to UTC.+ (t/+ (t/local-date-time today-dt) (t/hours 6))))+ dt-points+ (cycle various-offset-strs))))++ ;; BEWARE: No cleanup is done atm. It is expected that every CI run creates its own instance of this database.+ (mt/defdataset good-datetimes-in-belize+ [["GOOD_DATETIMES" [{:field-name "IN_Z_OFFSET"+ :base-type {:native "timestamptz"}}+ {:field-name "IN_VARIOUS_OFFSETS"+ :base-type {:native "timestamptz"}}+ {:field-name "JUST_NTZ"+ :base-type {:native "timestampntz"}}+ {:field-name "JUST_LTZ"+ :base-type {:native "timestampltz"}}]+ (rows-for-good-datetimes-in-belize)]])++ (deftest ^:parallel sync-datetime-types+ (mt/test-driver+ :snowflake+ (mt/dataset+ good-datetimes-in-belize+ (is (= [["id" "NUMBER" :type/BigInteger 0]+ ["IN_Z_OFFSET" "TIMESTAMPTZ" :type/DateTimeWithLocalTZ 1]+ ["IN_VARIOUS_OFFSETS" "TIMESTAMPTZ" :type/DateTimeWithLocalTZ 2]+ ["JUST_NTZ" "TIMESTAMPNTZ" :type/DateTime 3]+ ["JUST_LTZ" "TIMESTAMPLTZ" :type/DateTimeWithTZ 4]]+ (sort-by last+ (into []+ (map (juxt :name :database-type :base-type :database-position))+ (fetch-metadata/fields-metadata (mt/db)))))))))++ ;; The test needs user with no report timezone set and database timezone other than UTC. That's the reason for redefs+ ;; prior to dataset generation.+ (deftest ^:synchronized correct-timestamp-type-querying-test+ (mt/test-driver+ :snowflake+ (let [original-set-current-user-timezone! @#'test.data.snowflake/set-current-user-timezone!+ original-dbdef->connection-details (get-method tx/dbdef->connection-details :snowflake)]+ ;; load dataset as the default user before we switch to the belize-based+ ;; user who may not have permission to create the dataset+ (mt/dataset good-datetimes-in-belize+ (mt/id :GOOD_DATETIMES))+ (with-redefs [test.data.snowflake/set-current-user-timezone!+ (fn [_timezone]+ (original-set-current-user-timezone! "America/Belize"))+ tx/dbdef->connection-details+ (fn [driver connection-type database-definition]+ (-> (original-dbdef->connection-details driver connection-type database-definition)+ (assoc :user "BELIZE_PERSON")))]+ (mt/with-temporary-setting-values [report-timezone "America/Belize"]+ (mt/dataset+ good-datetimes-in-belize+ (testing "Expected data is returned using yesterday filter"+ (let [yesterday-first (t/- (t/truncate-to (t/offset-date-time belize-offset) :days) (t/days 1))+ yesterday-last (t/+ yesterday-first (t/hours 18))+ yesterday-first-str (t/format :iso-offset-date-time yesterday-first)+ yesterday-last-str (t/format :iso-offset-date-time yesterday-last)]+ (doseq [[tested-field-kw base-type database-type]+ [[:IN_Z_OFFSET :type/DateTimeWithLocalTZ "timestamptz"]+ [:IN_VARIOUS_OFFSETS :type/DateTimeWithLocalTZ "timestamptz"]+ [:JUST_NTZ :type/DateTime "timestampntz"]+ [:JUST_LTZ :type/DateTimeWithTZ "timestampltz"]]+ :let [tested-field [:field (mt/id :GOOD_DATETIMES tested-field-kw) {:base-type base-type}]]]+ (testing (str "on column type " database-type)+ (let [rows (mt/rows (qp/process-query+ {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :GOOD_DATETIMES)+ :fields [tested-field]+ :filter [:time-interval tested-field -1 :day]+ :order-by [[tested-field :asc]]}}))]+ (testing "Correct rows count returned"+ (is (= 4 (count rows))))+ (testing "First row has expected values"+ (is (= yesterday-first-str+ (ffirst rows))))+ (testing "Last row has expected values"+ (is (= yesterday-last-str+ (ffirst (reverse rows)))))))+ (testing "Rows should be properly allocated to days"+ (let [tested-day (assoc-in tested-field [2 :temporal-unit] :day)+ tested-minute (assoc-in tested-field [2 :temporal-unit] :minute)]+ (is (=? [[string? 4] [string? 4] [string? 4]]+ (mt/rows+ (qp/process-query+ (mt/mbql-query nil+ {:source-table (mt/id :GOOD_DATETIMES)+ :aggregation [[:count]]+ :breakout [tested-day]})))))+ (is (= [[yesterday-first-str 4 yesterday-first-str yesterday-last-str]]+ (mt/rows+ (qp/process-query+ (mt/mbql-query nil+ {:source-table (mt/id :GOOD_DATETIMES)+ :aggregation [[:count]+ [:min tested-minute]+ [:max tested-minute]]+ :breakout [tested-day]+ :filter [:= tested-field (t/format :iso-local-date yesterday-last)]})))))))))))))))))(deftest snowflake-all-auth-combos-test(mt/test-driver