Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Improve log output in DWH GC job (#81447)

Ben Grabow1e78c186c0c2
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index 65073e53d19..2e1a302e983 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -15,7 +15,7 @@
:deprecated-var 86
:discouraged-java-method 4
:discouraged-namespace 81
- :discouraged-var 131
+ :discouraged-var 130
:duplicate-require 1
:equals-true 1
:main-without-gen-class 1
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
index eb4f6263389..3871113a706 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
@@ -153,8 +153,7 @@
;; the printlns below are on purpose because we want them to show up when running tests, even on CI, to make sure this
;; stuff is working correctly. We can change it to `log` in the future when we're satisfied everything is working as
;; intended -- Case
- #_{:clj-kondo/ignore [:discouraged-var]}
- (println "Deleting dataset: " dataset-id)
+ (tx/print-progress! :bigquery-cloud-sdk "deleting %s" dataset-id)
(when (= dataset-id (test-dataset-id (tx/get-dataset-definition (data.impl/resolve-dataset-definition *ns* 'test-data))))
(throw (Exception. "tried to delete test-data")))
(.delete (bigquery) dataset-id (u/varargs
diff --git a/modules/drivers/redshift/test/metabase/test/data/redshift.clj b/modules/drivers/redshift/test/metabase/test/data/redshift.clj
index 6b48cd94c54..269d1e74ba3 100644
--- a/modules/drivers/redshift/test/metabase/test/data/redshift.clj
+++ b/modules/drivers/redshift/test/metabase/test/data/redshift.clj
@@ -239,7 +239,7 @@
re-querying. Caller owns the Statement."
[^java.sql.Statement stmt orphans]
(mapv (fn [[fmt-str schema]]
- (log/infof fmt-str schema)
+ (tx/print-progress! :redshift fmt-str schema)
(try
(.execute stmt (format "DROP SCHEMA IF EXISTS \"%s\" CASCADE;" schema))
{:name schema, :status :deleted}
diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
index 7cee6a22931..2be4d2e17c3 100644
--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
@@ -171,8 +171,7 @@
(defn- drop-one!
[^java.sql.Statement stmt dataset-name]
- #_{:clj-kondo/ignore [:discouraged-var]}
- (println "[Snowflake] Deleting old dataset:" dataset-name)
+ (tx/print-progress! :snowflake "deleting %s" dataset-name)
(try
(.execute stmt (format "DROP DATABASE IF EXISTS \"%s\";" dataset-name))
{:name dataset-name, :status :deleted}
diff --git a/test/metabase/test/data/gc.clj b/test/metabase/test/data/gc.clj
index d02abd0c5cc..7b44d000364 100644
--- a/test/metabase/test/data/gc.clj
+++ b/test/metabase/test/data/gc.clj
@@ -5,8 +5,7 @@
[clojure.string :as str]
[com.climate.claypoole :as cp]
[metabase.test.data.interface :as tx]
- [metabase.util.json :as json]
- [metabase.util.log :as log]))
+ [metabase.util.json :as json]))
(set! *warn-on-reflection* true)
@@ -36,7 +35,7 @@
(str/split (or drivers "") #",")))
default-drivers))
-(defn- census
+(defn- census!
"Datasets on each of a driver's servers, best effort: a count we cannot take must not cost us the sweep.
Runs on its own thread, which is why the skip is bound here rather than around the whole job: a `binding` does not
@@ -47,23 +46,24 @@
(try
(tx/count-datasets driver)
(catch Exception e
- (log/errorf "[%s] could not count datasets: %s" (name driver) (ex-message e))
+ (tx/print-progress! driver "could not count datasets: %s" (ex-message e))
{}))))
(defn- log-census!
"One line per driver before anything is deleted, so a run that is killed partway still says what it was up against."
[driver->counts]
(doseq [[driver counts] driver->counts]
- (log/infof "[%s] %s dataset(s) before sweep%s"
- (name driver)
- (if (empty? counts) "unknown" (reduce + 0 (remove nil? (vals counts))))
- (if (empty? counts)
- ""
- (str " — " (str/join ", " (for [[server n] (sort-by key counts)]
- (format "%s: %s" server (if (nil? n) "unknown" n))))))))
- (log/infof "%s dataset(s) before sweep, across %d driver(s)"
- (reduce + 0 (for [counts (vals driver->counts), n (vals counts) :when (some? n)] n))
- (count driver->counts)))
+ (tx/print-progress! driver "%s dataset(s) before sweep%s"
+ (if (empty? counts) "unknown" (reduce + 0 (remove nil? (vals counts))))
+ (if (empty? counts)
+ ""
+ (str " — " (str/join ", " (for [[server n] (sort-by key counts)]
+ (format "%s: %s" server (if (nil? n) "unknown" n))))))))
+ (let [known (for [counts (vals driver->counts), n (vals counts) :when (some? n)] n)]
+ (tx/print-progress! "gc" "%s dataset(s) before sweep, across %d driver(s)"
+ ;; "0" would read as an empty account rather than as a census nothing could answer
+ (if (seq known) (reduce + 0 known) "unknown")
+ (count driver->counts))))
(defn- sweep-driver!
"Sweep one driver and take a census of what is left, each best effort. A driver that cannot connect at all, or a
@@ -77,11 +77,11 @@
(vec (tx/gc-orphans! driver options))
(catch Exception e
[{:name nil, :status :failed, :error (ex-message e)}]))
- remaining (census driver)
+ remaining (census! driver)
{deleted :deleted, failed :failed} (group-by :status results)]
- (log/infof "[%s] %d deleted, %d failed" (name driver) (count deleted) (count failed))
+ (tx/print-progress! driver "%d deleted, %d failed" (count deleted) (count failed))
(doseq [{dataset-name :name, :keys [error]} failed]
- (log/errorf "[%s] %s: %s" (name driver) (or dataset-name "<server unreachable>") error))
+ (tx/print-progress! driver "FAILED %s: %s" (or dataset-name "<server unreachable>") error))
{:driver (name driver)
:deleted (vec deleted)
:failed (vec failed)
@@ -149,7 +149,7 @@
(spit (io/file dir "report.md") markdown)
(when-let [summary-file (System/getenv "GITHUB_STEP_SUMMARY")]
(spit summary-file markdown :append true))
- (log/infof "Wrote sweep report to %s" dir)))
+ (tx/print-progress! "gc" "wrote sweep report to %s" dir)))
(defn gc-orphans!
"Sweep orphaned test data from each driver's shared cloud account.
@@ -176,7 +176,7 @@
(let [swept (let [ds (parse-drivers drivers)]
;; drivers share nothing, so the job costs the slowest one rather than the sum of all three
(cp/with-shutdown! [pool (cp/threadpool (count ds))]
- (let [before (zipmap ds (doall (cp/pmap pool census ds)))]
+ (let [before (zipmap ds (doall (cp/pmap pool census! ds)))]
(log-census! before)
(doall (cp/pmap pool #(sweep-driver! % (get before %) options) ds)))))
report (build-report options swept)]
diff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.clj
index 0ecde4dd4bc..7d4158d287a 100644
--- a/test/metabase/test/data/interface.clj
+++ b/test/metabase/test/data/interface.clj
@@ -421,6 +421,23 @@
(log/infof "%s has no orphan GC; skipping." driver)
[])
+;; `println` rather than [[metabase.util.log]] on purpose: the nightly sweep runs as `clojure -X`, and log4j2 emits
+;; nothing to stdout in that context -- `info` and `warn` alike are swallowed, so a run's log showed only the few
+;; lines that already happened to use `println`.
+#_{:clj-kondo/ignore [:discouraged-var]}
+(defn print-progress!
+ "Print one operator-facing progress line, prefixed with `label` so the interleaved output of concurrently sweeping
+ drivers stays attributable.
+
+ Progress is printed as it happens rather than assembled at the end, because the job is expected to be killed at
+ its timeout partway through a backlog.
+
+ The newline is part of the string and written once. `println` writes the text and the newline separately, which
+ let concurrently sweeping drivers splice their output into the middle of each other's lines."
+ [label fmt-str & args]
+ (print (str "[" (name label) "] " (apply format fmt-str args) \newline))
+ (flush))
+
(defmulti count-datasets
"Census of every test dataset still on each server this driver's tests write to, taken after [[gc-orphans!]] has
run. Returns `{server-label count}`, keyed the same way as `:server` in [[gc-orphans!]]'s results.