Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add hour to redshift test schema prefix for more aggressive cleanup (#67215)

bryancb7c3a09acff
diff --git a/test/metabase/driver/sql/test_util/unique_prefix.clj b/test/metabase/driver/sql/test_util/unique_prefix.clj
index 34f6a7cd9ba..bfebb8138a0 100644
--- a/test/metabase/driver/sql/test_util/unique_prefix.clj
+++ b/test/metabase/driver/sql/test_util/unique_prefix.clj
@@ -7,17 +7,18 @@
recreate it, and then another CI job would do the same thing at the same time, and eventually we'd end up with a
half-created dataset that was missing a bunch of rows. So here is the new strategy going forward:
- 1. Every instance of Metabase gets their own prefix like `<current-date-utc>_<site-uuid>_` e.g. `test-data` becomes
- something like
+ 1. Every instance of Metabase gets their own prefix like `<current-date-utc>_<hour>_<site-uuid>_` e.g. `test-data`
+ becomes something like
- 2023_02_17_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data
+ 2025_12_18_20_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data
This will prevent jobs from running at the same time from stomping on each other's work.
2. To avoid filling our Snowflake/Redshift/etc. accounts up with ephemeral data that never gets deleted, we will
- delete datasets following this pattern when they are two days old or older. E.g. if it is currently `2023-02-17` in
- UTC then we can delete anything dataset starts with `2023_02_15` or older. This is done once the first time we
- create a test dataset in this process, i.e. done in [[metabase.test.data.interface/before-run]].
+ delete datasets following this pattern when they are more than 3 hours old. This allows aggressive cleanup while
+ still being safe for long-running test suites. Old-format datasets (without hour) are deleted after more than 1
+ day for backwards compatibility. Cleanup is done once the first time we create a test dataset in this process,
+ i.e. done in [[metabase.test.data.interface/before-run]].
See [[metabase.test.data.snowflake/delete-old-datasets-if-needed!]] for example.
See this Slack thread for more info
@@ -34,51 +35,105 @@
(defn- utc-date
"`LocalDate` in UTC time."
[]
- (t/local-date (u.date/with-time-zone-same-instant (t/zoned-date-time) "UTC")))
+ (t/local-date (t/instant) (t/zone-id "UTC")))
+
+(defn- utc-date-time
+ "`LocalDateTime` in UTC time."
+ []
+ (t/local-date-time (t/instant) (t/zone-id "UTC")))
(defn- unique-prefix*
([]
- (unique-prefix* (utc-date)))
- ([local-date]
- {:pre [(instance? java.time.LocalDate local-date)]}
+ (unique-prefix* (utc-date-time)))
+ ([local-date-time]
+ {:pre [(instance? java.time.LocalDateTime local-date-time)]}
;; app DB has to be initialized to get settings
(initialize/initialize-if-needed! :db)
- (-> (format "%s_%s_" local-date (system/site-uuid))
- (str/replace #"-" "_"))))
+ ;; Format: YYYY_MM_DD_HH_<site-uuid>_ (hour precision allows cleanup after ~3 hours)
+ (-> (format "%s_%02d_%s_"
+ (t/local-date local-date-time)
+ (t/as local-date-time :hour-of-day)
+ (system/site-uuid))
+ (str/replace #"-" "_"))))
(def ^{:arglists '([])} unique-prefix
- "Unique prefix for test datasets for this instance. Format is `<current-date-utc>_<site-uuid>_`. See comments above.
+ "Unique prefix for test datasets for this instance. Format is `<current-date-utc>_<hour-utc>_<site-uuid>_`. See comments above.
Example:
- 2023_02_17_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_"
+ 2025_12_18_20_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_"
(memoize unique-prefix*))
+(def ^:private old-dataset-hours-threshold
+ "Number of hours after which a dataset is considered old and can be deleted. Because schema names only include the
+ hour (not minutes/seconds), actual deletion can occur anywhere from N to N+1 hours after creation. Set this high
+ enough that any running test won't have its schema deleted mid-run."
+ 3)
+
(defn old-dataset-name?
- "Is this dataset name prefixed by a date two days ago or older?
+ "Is this dataset name old enough to be deleted?
- If the date is invalid e.g. `2023-02-31` then we'll count it as old so it will get deleted anyway."
+ For new-format names (with hour): checks if more than [[old-dataset-hours-threshold]] hours old.
+ For old-format names (date only): checks if more than 1 day old for backwards compatibility.
+ If the date/time is invalid, we return false (not old) to be safe - we only want to delete
+ datasets that match our known format."
[dataset-name]
- (when-let [[_ year month day] (re-matches #"^(\d{4})_(\d{2})_(\d{2}).*$" dataset-name)]
- (let [dataset-date (try
- (t/local-date (parse-long year) (parse-long month) (parse-long day))
- (catch Throwable _
- nil))]
- (if-not dataset-date
- true
- (t/before? dataset-date (u.date/add (utc-date) :day -1))))))
+ ;; Try new format first: YYYY_MM_DD_HH_<uuid>_...
+ (if-let [[_ year month day hour] (re-matches #"^(\d{4})_(\d{2})_(\d{2})_(\d{2})_.*$" dataset-name)]
+ (let [dataset-date-time (try
+ (t/local-date-time (parse-long year) (parse-long month) (parse-long day) (parse-long hour) 0)
+ (catch Throwable _ nil))]
+ (if-not dataset-date-time
+ false
+ (t/before? dataset-date-time (u.date/add (utc-date-time) :hour (- old-dataset-hours-threshold)))))
+ ;; TODO (bryan 12-23-25) Remove old-format checks when this has been running for a few days.
+ ;; Fall back to old format: YYYY_MM_DD_<uuid>_...
+ (when-let [[_ year month day] (re-matches #"^(\d{4})_(\d{2})_(\d{2})_.*$" dataset-name)]
+ (let [dataset-date (try
+ (t/local-date (parse-long year) (parse-long month) (parse-long day))
+ (catch Throwable _ nil))]
+ (if-not dataset-date
+ false
+ (t/before? dataset-date (u.date/add (utc-date) :day -1)))))))
(deftest ^:parallel old-dataset-name?-test
- (are [s] (old-dataset-name? s)
- "2023_02_01_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- "2023_01_17_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- "2022_02_17_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- (str (unique-prefix* (u.date/add (utc-date) :day -2)) "test-data")
- ;; if the date is invalid we should just treat it as old and delete it.
- "2022_00_00_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- "2022_13_01_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- "2022_02_31_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data")
- (are [s] (not (old-dataset-name? s))
- "2050_02_17_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
- "v4_test-data"
- (str (unique-prefix*) "test-data")
- (str (unique-prefix* (u.date/add (utc-date) :day -1)) "test-data")))
+ (testing "old-format names (date only) - more than 1 day old"
+ (let [two-days-ago (str/replace (str (u.date/add (utc-date) :day -2)) "-" "_")
+ yesterday (str/replace (str (u.date/add (utc-date) :day -1)) "-" "_")
+ today (str/replace (str (utc-date)) "-" "_")]
+ (are [s] (old-dataset-name? s)
+ "2023_02_01_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
+ "2023_01_17_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
+ "2022_02_17_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
+ ;; 2 days ago is old
+ (str two-days-ago "_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"))
+ (are [s] (not (old-dataset-name? s))
+ "2050_02_17_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
+ "v4_test-data"
+ ;; yesterday is not old (must be MORE than 1 day)
+ (str yesterday "_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data")
+ ;; today is not old
+ (str today "_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data")
+ ;; invalid dates are not old - only delete datasets matching our known format
+ "2022_00_00_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
+ "2022_13_01_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
+ "2022_02_31_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data")))
+ (testing "new-format names (with hour) - more than 3 hours old"
+ (are [s] (old-dataset-name? s)
+ ;; Ancient dates are old
+ "2023_02_01_00_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
+ "2023_02_01_14_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
+ ;; 4 hours ago is old
+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -4)) "test-data")
+ ;; 3 hours ago by hour is old (due to hour truncation, could be up to 3:59 hours old)
+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -3)) "test-data"))
+ (are [s] (not (old-dataset-name? s))
+ ;; Current time is not old
+ (str (unique-prefix*) "test-data")
+ ;; 1 hour ago is not old
+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -1)) "test-data")
+ ;; 2 hours ago is not old
+ (str (unique-prefix* (u.date/add (utc-date-time) :hour -2)) "test-data")
+ ;; Future dates are not old
+ "2050_02_17_14_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data"
+ ;; invalid hour is not old - only delete datasets matching our known format
+ "2023_02_01_25_82e897cb_ad31_4c82_a4b6_3e9e2e1dc1cb_test-data")))