Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Temporarily remove belize test and create-replace-table test in snowflake (#75830)

technomancyb0da250e6970
diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
index 314ebea82f3..54c92b1b2fa 100644
--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
@@ -483,184 +483,188 @@
[(format "DROP TABLE IF EXISTS \"%s\".\"PUBLIC\".\"%s\";" db-name table-name)]
{:transaction? false})))))))))
-;; Five related repro phases (v1-v5) share Snowflake connection setup and table-name lifecycle.
-;; Splitting would multiply CI Snowflake setup cost. Kondo's warning is acknowledged.
-^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
-(deftest ^:sequential ^:synchronized create-or-replace-table-updates-effective-type-test
- (mt/test-driver :snowflake
- (testing "GHY-3388: when a column's database type changes via CREATE OR REPLACE TABLE
+;; marking this as quarantined is not enough; when run in CI it frequently causes
+;; other unrelated tests to fail for reasons that are unclear. Disabling this
+;; and the Belize test makes the unreliability go away.
+(comment
+ ;; Five related repro phases (v1-v5) share Snowflake connection setup and table-name lifecycle.
+ ;; Splitting would multiply CI Snowflake setup cost. Kondo's warning is acknowledged.
+ ^{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
+ (deftest ^:sequential ^:synchronized create-or-replace-table-updates-effective-type-test
+ (mt/test-driver :snowflake
+ (testing "GHY-3388: when a column's database type changes via CREATE OR REPLACE TABLE
(e.g. TEXT -> NUMBER via TRY_TO_NUMBER), sync should update effective_type to match
the new base_type and clear stale coercion_strategy/semantic_type. Reproduces the
in-place update path (repro v1 from the Linear issue) where Metabase's Field row is
preserved across the schema change."
- (let [db-name (#'driver.snowflake/db-name (mt/db))
- table-name (str "ghy_3388_" (u.random/random-name))
- run-sql! (fn [stmts]
- (sql-jdbc.execute/do-with-connection-with-options
- :snowflake
- (mt/db)
- nil
- (fn [^java.sql.Connection conn]
- (doseq [stmt stmts]
- (jdbc/execute! {:connection conn} [stmt] {:transaction? false})))))
- qualified (format "\"%s\".\"PUBLIC\".\"%s\"" db-name table-name)]
- (try
- ;; step 1: create the table with a TEXT column and insert rows. Quote identifiers so they
- ;; are preserved lowercase (Snowflake upper-cases unquoted identifiers).
- (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)
- (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])
- ;; step 2: sync — Metabase creates Table + Field rows for our new table
- (sync/sync-database! (mt/db))
- (let [original-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)
- _ (is (some? original-table) "table should be synced")
- original-field (t2/select-one :model/Field :table_id (:id original-table) :name "text_column")]
- (testing "sanity check: text_column is text"
- (is (=? {:base_type :type/Text
- :effective_type :type/Text}
- original-field)))
- ;; step 3: CREATE OR REPLACE the table, converting text_column to a number via TRY_TO_NUMBER
+ (let [db-name (#'driver.snowflake/db-name (mt/db))
+ table-name (str "ghy_3388_" (u.random/random-name))
+ run-sql! (fn [stmts]
+ (sql-jdbc.execute/do-with-connection-with-options
+ :snowflake
+ (mt/db)
+ nil
+ (fn [^java.sql.Connection conn]
+ (doseq [stmt stmts]
+ (jdbc/execute! {:connection conn} [stmt] {:transaction? false})))))
+ qualified (format "\"%s\".\"PUBLIC\".\"%s\"" db-name table-name)]
+ (try
+ ;; step 1: create the table with a TEXT column and insert rows. Quote identifiers so they
+ ;; are preserved lowercase (Snowflake upper-cases unquoted identifiers).
+ (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)
+ (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])
+ ;; step 2: sync — Metabase creates Table + Field rows for our new table
+ (sync/sync-database! (mt/db))
+ (let [original-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)
+ _ (is (some? original-table) "table should be synced")
+ original-field (t2/select-one :model/Field :table_id (:id original-table) :name "text_column")]
+ (testing "sanity check: text_column is text"
+ (is (=? {:base_type :type/Text
+ :effective_type :type/Text}
+ original-field)))
+ ;; step 3: CREATE OR REPLACE the table, converting text_column to a number via TRY_TO_NUMBER
+ (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "
+ "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")
+ qualified qualified)])
+ ;; step 4: sync again
+ (sync/sync-database! (mt/db))
+ (let [new-field (t2/select-one :model/Field :id (:id original-field))]
+ (testing "after sync, base_type and effective_type both reflect the new numeric column,
+ and stale coercion/semantic type are cleared"
+ ;; TRY_TO_NUMBER with no precision returns NUMBER(38,0) -> :type/BigInteger
+ (is (=? {:base_type :type/BigInteger
+ :effective_type :type/BigInteger
+ :coercion_strategy nil
+ :semantic_type nil}
+ new-field))))
+ ;; ----- repro v2 (clean state baseline): drop the table, wipe Metabase metadata for it,
+ ;; recreate fresh as TEXT, sync, then CREATE OR REPLACE to numeric and sync again. The
+ ;; field should end up with effective_type matching base_type. (In the Linear repro the
+ ;; user observes the cast feature does not appear here — i.e. effective_type vs base_type
+ ;; should be consistent and not stuck as text.)
+ (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])
+ (t2/delete! :model/Field :table_id (:id original-table))
+ (t2/delete! :model/Table :id (:id original-table)))
+ (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)
+ (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])
+ (sync/sync-database! (mt/db))
(run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "
- "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")
+ "SELECT \"id\", TRY_TO_NUMBER(\"text_column\", 38, 2) AS \"text_column\" FROM %s;")
qualified qualified)])
- ;; step 4: sync again
(sync/sync-database! (mt/db))
- (let [new-field (t2/select-one :model/Field :id (:id original-field))]
- (testing "after sync, base_type and effective_type both reflect the new numeric column,
- and stale coercion/semantic type are cleared"
- ;; TRY_TO_NUMBER with no precision returns NUMBER(38,0) -> :type/BigInteger
- (is (=? {:base_type :type/BigInteger
- :effective_type :type/BigInteger
+ (let [fresh-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)
+ fresh-field (t2/select-one :model/Field :table_id (:id fresh-table) :name "text_column")]
+ (testing "after fresh-state CREATE OR REPLACE to numeric, base_type and effective_type
+ both reflect the new numeric column"
+ ;; TRY_TO_NUMBER(x, 38, 2) returns NUMBER(38,2) -> :type/Number on Snowflake
+ (is (=? {:base_type :type/Number
+ :effective_type :type/Number
:coercion_strategy nil
:semantic_type nil}
- new-field))))
- ;; ----- repro v2 (clean state baseline): drop the table, wipe Metabase metadata for it,
- ;; recreate fresh as TEXT, sync, then CREATE OR REPLACE to numeric and sync again. The
- ;; field should end up with effective_type matching base_type. (In the Linear repro the
- ;; user observes the cast feature does not appear here — i.e. effective_type vs base_type
- ;; should be consistent and not stuck as text.)
- (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])
- (t2/delete! :model/Field :table_id (:id original-table))
- (t2/delete! :model/Table :id (:id original-table)))
- (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)
- (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])
- (sync/sync-database! (mt/db))
- (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "
- "SELECT \"id\", TRY_TO_NUMBER(\"text_column\", 38, 2) AS \"text_column\" FROM %s;")
- qualified qualified)])
- (sync/sync-database! (mt/db))
- (let [fresh-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)
- fresh-field (t2/select-one :model/Field :table_id (:id fresh-table) :name "text_column")]
- (testing "after fresh-state CREATE OR REPLACE to numeric, base_type and effective_type
- both reflect the new numeric column"
- ;; TRY_TO_NUMBER(x, 38, 2) returns NUMBER(38,2) -> :type/Number on Snowflake
- (is (=? {:base_type :type/Number
- :effective_type :type/Number
- :coercion_strategy nil
- :semantic_type nil}
- fresh-field)))
- ;; ----- repro v3: simulate the user setting a coercion strategy on the TEXT column
- ;; (via the UI cast-toggle) before the underlying type changes. When the DB column
- ;; becomes numeric natively, sync should clear the now-meaningless coercion strategy
- ;; and reset effective_type to match the new base_type. If sync leaves the coercion
- ;; sticky, effective_type will diverge from base_type — the observable GHY-3388 symptom.
- (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])
- (t2/delete! :model/Field :table_id (:id fresh-table))
- (t2/delete! :model/Table :id (:id fresh-table)))
- (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)
- (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])
- (sync/sync-database! (mt/db))
- (let [pre-coerce-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)
- pre-coerce-field (t2/select-one :model/Field :table_id (:id pre-coerce-table) :name "text_column")]
- ;; user enables coercion: this TEXT column is really an integer
- (t2/update! :model/Field (:id pre-coerce-field)
- {:coercion_strategy :Coercion/String->Integer
- :effective_type :type/Integer})
- (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "
- "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")
- qualified qualified)])
+ fresh-field)))
+ ;; ----- repro v3: simulate the user setting a coercion strategy on the TEXT column
+ ;; (via the UI cast-toggle) before the underlying type changes. When the DB column
+ ;; becomes numeric natively, sync should clear the now-meaningless coercion strategy
+ ;; and reset effective_type to match the new base_type. If sync leaves the coercion
+ ;; sticky, effective_type will diverge from base_type — the observable GHY-3388 symptom.
+ (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])
+ (t2/delete! :model/Field :table_id (:id fresh-table))
+ (t2/delete! :model/Table :id (:id fresh-table)))
+ (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)
+ (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])
(sync/sync-database! (mt/db))
- (let [after-field (t2/select-one :model/Field :id (:id pre-coerce-field))]
- (testing "after the underlying TEXT column becomes a native number, the previously
+ (let [pre-coerce-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)
+ pre-coerce-field (t2/select-one :model/Field :table_id (:id pre-coerce-table) :name "text_column")]
+ ;; user enables coercion: this TEXT column is really an integer
+ (t2/update! :model/Field (:id pre-coerce-field)
+ {:coercion_strategy :Coercion/String->Integer
+ :effective_type :type/Integer})
+ (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "
+ "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")
+ qualified qualified)])
+ (sync/sync-database! (mt/db))
+ (let [after-field (t2/select-one :model/Field :id (:id pre-coerce-field))]
+ (testing "after the underlying TEXT column becomes a native number, the previously
user-set coercion strategy should be cleared and effective_type should
match the new base_type"
- (is (=? {:base_type :type/BigInteger
- :effective_type :type/BigInteger
- :coercion_strategy nil
- :semantic_type nil}
- after-field)))))
- ;; ----- repro v4: set coercion the way the PUT /api/field/:id endpoint does:
- ;; via upsert-user-settings, which writes to metabase_field_user_settings. That
- ;; row is overlaid back onto the field on every t2/update! via the
- ;; sync-user-settings before-update hook (merge non-nil user-settings over
- ;; field changes). v3 wrote only to metabase_field and so bypassed this
- ;; overlay; v4 exercises the real UI cast-toggle path.
- (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])
- (when-let [stale-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)]
- (t2/delete! :model/Field :table_id (:id stale-table))
- (t2/delete! :model/Table :id (:id stale-table)))
- (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)
- (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])
- (sync/sync-database! (mt/db))
- (let [v4-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)
- v4-field (t2/select-one :model/Field :table_id (:id v4-table) :name "text_column")]
- (field-user-settings/upsert-user-settings v4-field
- {:coercion_strategy :Coercion/String->Integer
- :effective_type :type/Integer})
- (t2/update! :model/Field (:id v4-field)
- {:coercion_strategy :Coercion/String->Integer
- :effective_type :type/Integer})
- (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "
- "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")
- qualified qualified)])
+ (is (=? {:base_type :type/BigInteger
+ :effective_type :type/BigInteger
+ :coercion_strategy nil
+ :semantic_type nil}
+ after-field)))))
+ ;; ----- repro v4: set coercion the way the PUT /api/field/:id endpoint does:
+ ;; via upsert-user-settings, which writes to metabase_field_user_settings. That
+ ;; row is overlaid back onto the field on every t2/update! via the
+ ;; sync-user-settings before-update hook (merge non-nil user-settings over
+ ;; field changes). v3 wrote only to metabase_field and so bypassed this
+ ;; overlay; v4 exercises the real UI cast-toggle path.
+ (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])
+ (when-let [stale-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)]
+ (t2/delete! :model/Field :table_id (:id stale-table))
+ (t2/delete! :model/Table :id (:id stale-table)))
+ (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)
+ (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])
(sync/sync-database! (mt/db))
- (let [after-field (t2/select-one :model/Field :id (:id v4-field))
- after-user-settings (t2/select-one :model/FieldUserSettings :field_id (:id v4-field))]
- (testing "after CREATE OR REPLACE to numeric, the user-set coercion stored in
+ (let [v4-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)
+ v4-field (t2/select-one :model/Field :table_id (:id v4-table) :name "text_column")]
+ (field-user-settings/upsert-user-settings v4-field
+ {:coercion_strategy :Coercion/String->Integer
+ :effective_type :type/Integer})
+ (t2/update! :model/Field (:id v4-field)
+ {:coercion_strategy :Coercion/String->Integer
+ :effective_type :type/Integer})
+ (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "
+ "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")
+ qualified qualified)])
+ (sync/sync-database! (mt/db))
+ (let [after-field (t2/select-one :model/Field :id (:id v4-field))
+ after-user-settings (t2/select-one :model/FieldUserSettings :field_id (:id v4-field))]
+ (testing "after CREATE OR REPLACE to numeric, the user-set coercion stored in
metabase_field_user_settings (via upsert-user-settings) should be
cleared so it doesn't overlay back on top of the synced field"
- (is (=? {:base_type :type/BigInteger
- :effective_type :type/BigInteger
- :coercion_strategy nil
- :semantic_type nil}
- after-field))
- (is (=? {:effective_type :type/BigInteger
- :coercion_strategy nil
- :semantic_type nil}
- after-user-settings)))))
- ;; ----- repro v5: same as v1 (in-place CREATE OR REPLACE TEXT -> NUMBER) but trigger
- ;; sync via sync/sync-table! — the function the per-table "Sync table now" UI button
- ;; calls. Goes through sync-table-metadata! -> sync-fields-for-table! instead of
- ;; sync-db-metadata! -> sync-fields!. Both reach the same sync-and-update! core, but
- ;; the OP's repro instructions said "Now sync the table" (singular), so this rules
- ;; out a code-path-specific bug at the entry point.
- (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])
- (when-let [stale-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)]
- (t2/delete! :model/Field :table_id (:id stale-table))
- (t2/delete! :model/Table :id (:id stale-table)))
- (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)
- (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])
- (sync/sync-database! (mt/db))
- (let [v5-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)
- v5-field (t2/select-one :model/Field :table_id (:id v5-table) :name "text_column")]
- (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "
- "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")
- qualified qualified)])
- (sync/sync-table! v5-table)
- (let [after-field (t2/select-one :model/Field :id (:id v5-field))]
- (testing "sync-table! (the per-table UI button) should also reset effective_type
+ (is (=? {:base_type :type/BigInteger
+ :effective_type :type/BigInteger
+ :coercion_strategy nil
+ :semantic_type nil}
+ after-field))
+ (is (=? {:effective_type :type/BigInteger
+ :coercion_strategy nil
+ :semantic_type nil}
+ after-user-settings)))))
+ ;; ----- repro v5: same as v1 (in-place CREATE OR REPLACE TEXT -> NUMBER) but trigger
+ ;; sync via sync/sync-table! — the function the per-table "Sync table now" UI button
+ ;; calls. Goes through sync-table-metadata! -> sync-fields-for-table! instead of
+ ;; sync-db-metadata! -> sync-fields!. Both reach the same sync-and-update! core, but
+ ;; the OP's repro instructions said "Now sync the table" (singular), so this rules
+ ;; out a code-path-specific bug at the entry point.
+ (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])
+ (when-let [stale-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)]
+ (t2/delete! :model/Field :table_id (:id stale-table))
+ (t2/delete! :model/Table :id (:id stale-table)))
+ (run-sql! [(format "CREATE OR REPLACE TRANSIENT TABLE %s (\"id\" INTEGER, \"text_column\" TEXT);" qualified)
+ (format "INSERT INTO %s (\"id\", \"text_column\") VALUES (1, '100'), (2, '200'), (3, '300');" qualified)])
+ (sync/sync-database! (mt/db))
+ (let [v5-table (t2/select-one :model/Table :db_id (mt/id) :name table-name)
+ v5-field (t2/select-one :model/Field :table_id (:id v5-table) :name "text_column")]
+ (run-sql! [(format (str "CREATE OR REPLACE TABLE %s AS "
+ "SELECT \"id\", TRY_TO_NUMBER(\"text_column\") AS \"text_column\" FROM %s;")
+ qualified qualified)])
+ (sync/sync-table! v5-table)
+ (let [after-field (t2/select-one :model/Field :id (:id v5-field))]
+ (testing "sync-table! (the per-table UI button) should also reset effective_type
when the underlying column type changes"
- (is (=? {:base_type :type/BigInteger
- :effective_type :type/BigInteger
- :coercion_strategy nil
- :semantic_type nil}
- after-field)))))
- (finally
- (let [t (t2/select-one :model/Table :db_id (mt/id) :name table-name)]
- (when t
- (u/ignore-exceptions (t2/delete! :model/Field :table_id (:id t)))
- (u/ignore-exceptions (t2/delete! :model/Table :id (:id t)))))
- (u/ignore-exceptions
- (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)]))))))))
+ (is (=? {:base_type :type/BigInteger
+ :effective_type :type/BigInteger
+ :coercion_strategy nil
+ :semantic_type nil}
+ after-field)))))
+ (finally
+ (let [t (t2/select-one :model/Table :db_id (mt/id) :name table-name)]
+ (when t
+ (u/ignore-exceptions (t2/delete! :model/Field :table_id (:id t)))
+ (u/ignore-exceptions (t2/delete! :model/Table :id (:id t)))))
+ (u/ignore-exceptions
+ (run-sql! [(format "DROP TABLE IF EXISTS %s;" qualified)])))))))))
(deftest ^:sequential describe-table-test
(mt/test-driver :snowflake
@@ -1059,7 +1063,7 @@
(is (= [expected]
(mt/first-row (qp/process-query query))))))))))
-(deftest ^:parallel local-date-time-parameter-test
+(deftest local-date-time-parameter-test
(test-temporal-instance
#t "2024-04-25T14:44:00"
"2024-04-25T14:44:00Z"))
@@ -1317,125 +1321,130 @@
;;;; (In Belize they know no DST anymore.)
;;;;
-(def ^:private belize-offset (t/zone-offset "-06:00"))
-
-(defn- rows-for-good-datetimes-in-belize
- []
- (let [number-of-points (* 4 3)
- today-dt (t/truncate-to (t/offset-date-time belize-offset) :days)
- first-dt-point (t/- today-dt (t/days 2))
- dt-points (for [i (range number-of-points)]
- (t/+ first-dt-point (t/hours (* 6 i))))
- various-offset-strs ["-10:00" "-04:00" "+02:00" "+09:00"]]
- (mapv (fn [today-dt offset-str]
- (vector (t/with-offset-same-instant today-dt (t/zone-offset "Z"))
- (t/with-offset-same-instant today-dt (t/zone-offset offset-str))
- (t/local-date-time today-dt)
+;; marking this as quarantined is not enough; when run in CI it frequently causes
+;; other unrelated tests to fail for reasons that are unclear. Disabling this
+;; and the create-or-replace-table-updates-effective-type-test test makes the
+;; unreliability go away.
+(comment
+ (def ^:private belize-offset (t/zone-offset "-06:00"))
+
+ (defn- rows-for-good-datetimes-in-belize
+ []
+ (let [number-of-points (* 4 3)
+ today-dt (t/truncate-to (t/offset-date-time belize-offset) :days)
+ first-dt-point (t/- today-dt (t/days 2))
+ dt-points (for [i (range number-of-points)]
+ (t/+ first-dt-point (t/hours (* 6 i))))
+ various-offset-strs ["-10:00" "-04:00" "+02:00" "+09:00"]]
+ (mapv (fn [today-dt offset-str]
+ (vector (t/with-offset-same-instant today-dt (t/zone-offset "Z"))
+ (t/with-offset-same-instant today-dt (t/zone-offset offset-str))
+ (t/local-date-time today-dt)
;;;; Shift local date time for belize offset so timestamp_ltz has same instant as rest!
- ;; Even though later in the test the user with America/Belize timezone does the data loading,
- ;; the timestamps have to be adjusted.
- ;; I believe that it is because of either (1) us hardcoding the :timestamp connection property
- ;; to UTC (see the `connection-details->spec :snowflake`) or (2) the fact that the JVM timezone
- ;; is set to UTC.
- (t/+ (t/local-date-time today-dt) (t/hours 6))))
- dt-points
- (cycle various-offset-strs))))
-
-;; BEWARE: No cleanup is done atm. It is expected that every CI run creates its own instance of this database.
-(mt/defdataset good-datetimes-in-belize
- [["GOOD_DATETIMES" [{:field-name "IN_Z_OFFSET"
- :base-type {:native "timestamptz"}}
- {:field-name "IN_VARIOUS_OFFSETS"
- :base-type {:native "timestamptz"}}
- {:field-name "JUST_NTZ"
- :base-type {:native "timestampntz"}}
- {:field-name "JUST_LTZ"
- :base-type {:native "timestampltz"}}]
- (rows-for-good-datetimes-in-belize)]])
-
-(deftest ^:parallel sync-datetime-types
- (mt/test-driver
- :snowflake
- (mt/dataset
- good-datetimes-in-belize
- (is (= [["id" "NUMBER" :type/BigInteger 0]
- ["IN_Z_OFFSET" "TIMESTAMPTZ" :type/DateTimeWithLocalTZ 1]
- ["IN_VARIOUS_OFFSETS" "TIMESTAMPTZ" :type/DateTimeWithLocalTZ 2]
- ["JUST_NTZ" "TIMESTAMPNTZ" :type/DateTime 3]
- ["JUST_LTZ" "TIMESTAMPLTZ" :type/DateTimeWithTZ 4]]
- (sort-by last
- (into []
- (map (juxt :name :database-type :base-type :database-position))
- (fetch-metadata/fields-metadata (mt/db)))))))))
-
-;; The test needs user with no report timezone set and database timezone other than UTC. That's the reason for redefs
-;; prior to dataset generation.
-(deftest ^:synchronized correct-timestamp-type-querying-test
- (mt/test-driver
- :snowflake
- (let [original-set-current-user-timezone! @#'test.data.snowflake/set-current-user-timezone!
- original-dbdef->connection-details (get-method tx/dbdef->connection-details :snowflake)]
- ;; load dataset as the default user before we switch to the belize-based
- ;; user who may not have permission to create the dataset
- (mt/dataset good-datetimes-in-belize
- (mt/id :GOOD_DATETIMES))
- (with-redefs [test.data.snowflake/set-current-user-timezone!
- (fn [_timezone]
- (original-set-current-user-timezone! "America/Belize"))
- tx/dbdef->connection-details
- (fn [driver connection-type database-definition]
- (-> (original-dbdef->connection-details driver connection-type database-definition)
- (assoc :user "BELIZE_PERSON")))]
- (mt/with-temporary-setting-values [report-timezone "America/Belize"]
- (mt/dataset
- good-datetimes-in-belize
- (testing "Expected data is returned using yesterday filter"
- (let [yesterday-first (t/- (t/truncate-to (t/offset-date-time belize-offset) :days) (t/days 1))
- yesterday-last (t/+ yesterday-first (t/hours 18))
- yesterday-first-str (t/format :iso-offset-date-time yesterday-first)
- yesterday-last-str (t/format :iso-offset-date-time yesterday-last)]
- (doseq [[tested-field-kw base-type database-type]
- [[:IN_Z_OFFSET :type/DateTimeWithLocalTZ "timestamptz"]
- [:IN_VARIOUS_OFFSETS :type/DateTimeWithLocalTZ "timestamptz"]
- [:JUST_NTZ :type/DateTime "timestampntz"]
- [:JUST_LTZ :type/DateTimeWithTZ "timestampltz"]]
- :let [tested-field [:field (mt/id :GOOD_DATETIMES tested-field-kw) {:base-type base-type}]]]
- (testing (str "on column type " database-type)
- (let [rows (mt/rows (qp/process-query
- {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :GOOD_DATETIMES)
- :fields [tested-field]
- :filter [:time-interval tested-field -1 :day]
- :order-by [[tested-field :asc]]}}))]
- (testing "Correct rows count returned"
- (is (= 4 (count rows))))
- (testing "First row has expected values"
- (is (= yesterday-first-str
- (ffirst rows))))
- (testing "Last row has expected values"
- (is (= yesterday-last-str
- (ffirst (reverse rows)))))))
- (testing "Rows should be properly allocated to days"
- (let [tested-day (assoc-in tested-field [2 :temporal-unit] :day)
- tested-minute (assoc-in tested-field [2 :temporal-unit] :minute)]
- (is (=? [[string? 4] [string? 4] [string? 4]]
- (mt/rows
- (qp/process-query
- (mt/mbql-query nil
- {:source-table (mt/id :GOOD_DATETIMES)
- :aggregation [[:count]]
- :breakout [tested-day]})))))
- (is (= [[yesterday-first-str 4 yesterday-first-str yesterday-last-str]]
- (mt/rows
- (qp/process-query
- (mt/mbql-query nil
- {:source-table (mt/id :GOOD_DATETIMES)
- :aggregation [[:count]
- [:min tested-minute]
- [:max tested-minute]]
- :breakout [tested-day]
- :filter [:= tested-field (t/format :iso-local-date yesterday-last)]}))))))))))))))))
+ ;; Even though later in the test the user with America/Belize timezone does the data loading,
+ ;; the timestamps have to be adjusted.
+ ;; I believe that it is because of either (1) us hardcoding the :timestamp connection property
+ ;; to UTC (see the `connection-details->spec :snowflake`) or (2) the fact that the JVM timezone
+ ;; is set to UTC.
+ (t/+ (t/local-date-time today-dt) (t/hours 6))))
+ dt-points
+ (cycle various-offset-strs))))
+
+ ;; BEWARE: No cleanup is done atm. It is expected that every CI run creates its own instance of this database.
+ (mt/defdataset good-datetimes-in-belize
+ [["GOOD_DATETIMES" [{:field-name "IN_Z_OFFSET"
+ :base-type {:native "timestamptz"}}
+ {:field-name "IN_VARIOUS_OFFSETS"
+ :base-type {:native "timestamptz"}}
+ {:field-name "JUST_NTZ"
+ :base-type {:native "timestampntz"}}
+ {:field-name "JUST_LTZ"
+ :base-type {:native "timestampltz"}}]
+ (rows-for-good-datetimes-in-belize)]])
+
+ (deftest ^:parallel sync-datetime-types
+ (mt/test-driver
+ :snowflake
+ (mt/dataset
+ good-datetimes-in-belize
+ (is (= [["id" "NUMBER" :type/BigInteger 0]
+ ["IN_Z_OFFSET" "TIMESTAMPTZ" :type/DateTimeWithLocalTZ 1]
+ ["IN_VARIOUS_OFFSETS" "TIMESTAMPTZ" :type/DateTimeWithLocalTZ 2]
+ ["JUST_NTZ" "TIMESTAMPNTZ" :type/DateTime 3]
+ ["JUST_LTZ" "TIMESTAMPLTZ" :type/DateTimeWithTZ 4]]
+ (sort-by last
+ (into []
+ (map (juxt :name :database-type :base-type :database-position))
+ (fetch-metadata/fields-metadata (mt/db)))))))))
+
+ ;; The test needs user with no report timezone set and database timezone other than UTC. That's the reason for redefs
+ ;; prior to dataset generation.
+ (deftest ^:synchronized correct-timestamp-type-querying-test
+ (mt/test-driver
+ :snowflake
+ (let [original-set-current-user-timezone! @#'test.data.snowflake/set-current-user-timezone!
+ original-dbdef->connection-details (get-method tx/dbdef->connection-details :snowflake)]
+ ;; load dataset as the default user before we switch to the belize-based
+ ;; user who may not have permission to create the dataset
+ (mt/dataset good-datetimes-in-belize
+ (mt/id :GOOD_DATETIMES))
+ (with-redefs [test.data.snowflake/set-current-user-timezone!
+ (fn [_timezone]
+ (original-set-current-user-timezone! "America/Belize"))
+ tx/dbdef->connection-details
+ (fn [driver connection-type database-definition]
+ (-> (original-dbdef->connection-details driver connection-type database-definition)
+ (assoc :user "BELIZE_PERSON")))]
+ (mt/with-temporary-setting-values [report-timezone "America/Belize"]
+ (mt/dataset
+ good-datetimes-in-belize
+ (testing "Expected data is returned using yesterday filter"
+ (let [yesterday-first (t/- (t/truncate-to (t/offset-date-time belize-offset) :days) (t/days 1))
+ yesterday-last (t/+ yesterday-first (t/hours 18))
+ yesterday-first-str (t/format :iso-offset-date-time yesterday-first)
+ yesterday-last-str (t/format :iso-offset-date-time yesterday-last)]
+ (doseq [[tested-field-kw base-type database-type]
+ [[:IN_Z_OFFSET :type/DateTimeWithLocalTZ "timestamptz"]
+ [:IN_VARIOUS_OFFSETS :type/DateTimeWithLocalTZ "timestamptz"]
+ [:JUST_NTZ :type/DateTime "timestampntz"]
+ [:JUST_LTZ :type/DateTimeWithTZ "timestampltz"]]
+ :let [tested-field [:field (mt/id :GOOD_DATETIMES tested-field-kw) {:base-type base-type}]]]
+ (testing (str "on column type " database-type)
+ (let [rows (mt/rows (qp/process-query
+ {:database (mt/id)
+ :type :query
+ :query {:source-table (mt/id :GOOD_DATETIMES)
+ :fields [tested-field]
+ :filter [:time-interval tested-field -1 :day]
+ :order-by [[tested-field :asc]]}}))]
+ (testing "Correct rows count returned"
+ (is (= 4 (count rows))))
+ (testing "First row has expected values"
+ (is (= yesterday-first-str
+ (ffirst rows))))
+ (testing "Last row has expected values"
+ (is (= yesterday-last-str
+ (ffirst (reverse rows)))))))
+ (testing "Rows should be properly allocated to days"
+ (let [tested-day (assoc-in tested-field [2 :temporal-unit] :day)
+ tested-minute (assoc-in tested-field [2 :temporal-unit] :minute)]
+ (is (=? [[string? 4] [string? 4] [string? 4]]
+ (mt/rows
+ (qp/process-query
+ (mt/mbql-query nil
+ {:source-table (mt/id :GOOD_DATETIMES)
+ :aggregation [[:count]]
+ :breakout [tested-day]})))))
+ (is (= [[yesterday-first-str 4 yesterday-first-str yesterday-last-str]]
+ (mt/rows
+ (qp/process-query
+ (mt/mbql-query nil
+ {:source-table (mt/id :GOOD_DATETIMES)
+ :aggregation [[:count]
+ [:min tested-minute]
+ [:max tested-minute]]
+ :breakout [tested-day]
+ :filter [:= tested-field (t/format :iso-local-date yesterday-last)]})))))))))))))))))
(deftest snowflake-all-auth-combos-test
(mt/test-driver