The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve log output in DWH GC job (#81447)
Ben Grabow1e78c186c0c2
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex 65073e53d19..2e1a302e983 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -15,7 +15,7 @@:deprecated-var 86:discouraged-java-method 4:discouraged-namespace 81- :discouraged-var 131+ :discouraged-var 130:duplicate-require 1:equals-true 1:main-without-gen-class 1diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.cljindex eb4f6263389..3871113a706 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj@@ -153,8 +153,7 @@;; the printlns below are on purpose because we want them to show up when running tests, even on CI, to make sure this;; stuff is working correctly. We can change it to `log` in the future when we're satisfied everything is working as;; intended -- Case- #_{:clj-kondo/ignore [:discouraged-var]}- (println "Deleting dataset: " dataset-id)+ (tx/print-progress! :bigquery-cloud-sdk "deleting %s" dataset-id)(when (= dataset-id (test-dataset-id (tx/get-dataset-definition (data.impl/resolve-dataset-definition *ns* 'test-data))))(throw (Exception. "tried to delete test-data")))(.delete (bigquery) dataset-id (u/varargsdiff --git a/modules/drivers/redshift/test/metabase/test/data/redshift.clj b/modules/drivers/redshift/test/metabase/test/data/redshift.cljindex 6b48cd94c54..269d1e74ba3 100644--- a/modules/drivers/redshift/test/metabase/test/data/redshift.clj+++ b/modules/drivers/redshift/test/metabase/test/data/redshift.clj@@ -239,7 +239,7 @@re-querying. Caller owns the Statement."[^java.sql.Statement stmt orphans](mapv (fn [[fmt-str schema]]- (log/infof fmt-str schema)+ (tx/print-progress! :redshift fmt-str schema)(try(.execute stmt (format "DROP SCHEMA IF EXISTS \"%s\" CASCADE;" schema)){:name schema, :status :deleted}diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.cljindex 7cee6a22931..2be4d2e17c3 100644--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj@@ -171,8 +171,7 @@(defn- drop-one![^java.sql.Statement stmt dataset-name]- #_{:clj-kondo/ignore [:discouraged-var]}- (println "[Snowflake] Deleting old dataset:" dataset-name)+ (tx/print-progress! :snowflake "deleting %s" dataset-name)(try(.execute stmt (format "DROP DATABASE IF EXISTS \"%s\";" dataset-name)){:name dataset-name, :status :deleted}diff --git a/test/metabase/test/data/gc.clj b/test/metabase/test/data/gc.cljindex d02abd0c5cc..7b44d000364 100644--- a/test/metabase/test/data/gc.clj+++ b/test/metabase/test/data/gc.clj@@ -5,8 +5,7 @@[clojure.string :as str][com.climate.claypoole :as cp][metabase.test.data.interface :as tx]- [metabase.util.json :as json]- [metabase.util.log :as log]))+ [metabase.util.json :as json]))(set! *warn-on-reflection* true)@@ -36,7 +35,7 @@(str/split (or drivers "") #",")))default-drivers))-(defn- census+(defn- census!"Datasets on each of a driver's servers, best effort: a count we cannot take must not cost us the sweep.Runs on its own thread, which is why the skip is bound here rather than around the whole job: a `binding` does not@@ -47,23 +46,24 @@(try(tx/count-datasets driver)(catch Exception e- (log/errorf "[%s] could not count datasets: %s" (name driver) (ex-message e))+ (tx/print-progress! driver "could not count datasets: %s" (ex-message e)){}))))(defn- log-census!"One line per driver before anything is deleted, so a run that is killed partway still says what it was up against."[driver->counts](doseq [[driver counts] driver->counts]- (log/infof "[%s] %s dataset(s) before sweep%s"- (name driver)- (if (empty? counts) "unknown" (reduce + 0 (remove nil? (vals counts))))- (if (empty? counts)- ""- (str " — " (str/join ", " (for [[server n] (sort-by key counts)]- (format "%s: %s" server (if (nil? n) "unknown" n))))))))- (log/infof "%s dataset(s) before sweep, across %d driver(s)"- (reduce + 0 (for [counts (vals driver->counts), n (vals counts) :when (some? n)] n))- (count driver->counts)))+ (tx/print-progress! driver "%s dataset(s) before sweep%s"+ (if (empty? counts) "unknown" (reduce + 0 (remove nil? (vals counts))))+ (if (empty? counts)+ ""+ (str " — " (str/join ", " (for [[server n] (sort-by key counts)]+ (format "%s: %s" server (if (nil? n) "unknown" n))))))))+ (let [known (for [counts (vals driver->counts), n (vals counts) :when (some? n)] n)]+ (tx/print-progress! "gc" "%s dataset(s) before sweep, across %d driver(s)"+ ;; "0" would read as an empty account rather than as a census nothing could answer+ (if (seq known) (reduce + 0 known) "unknown")+ (count driver->counts))))(defn- sweep-driver!"Sweep one driver and take a census of what is left, each best effort. A driver that cannot connect at all, or a@@ -77,11 +77,11 @@(vec (tx/gc-orphans! driver options))(catch Exception e[{:name nil, :status :failed, :error (ex-message e)}]))- remaining (census driver)+ remaining (census! driver){deleted :deleted, failed :failed} (group-by :status results)]- (log/infof "[%s] %d deleted, %d failed" (name driver) (count deleted) (count failed))+ (tx/print-progress! driver "%d deleted, %d failed" (count deleted) (count failed))(doseq [{dataset-name :name, :keys [error]} failed]- (log/errorf "[%s] %s: %s" (name driver) (or dataset-name "<server unreachable>") error))+ (tx/print-progress! driver "FAILED %s: %s" (or dataset-name "<server unreachable>") error)){:driver (name driver):deleted (vec deleted):failed (vec failed)@@ -149,7 +149,7 @@(spit (io/file dir "report.md") markdown)(when-let [summary-file (System/getenv "GITHUB_STEP_SUMMARY")](spit summary-file markdown :append true))- (log/infof "Wrote sweep report to %s" dir)))+ (tx/print-progress! "gc" "wrote sweep report to %s" dir)))(defn gc-orphans!"Sweep orphaned test data from each driver's shared cloud account.@@ -176,7 +176,7 @@(let [swept (let [ds (parse-drivers drivers)];; drivers share nothing, so the job costs the slowest one rather than the sum of all three(cp/with-shutdown! [pool (cp/threadpool (count ds))]- (let [before (zipmap ds (doall (cp/pmap pool census ds)))]+ (let [before (zipmap ds (doall (cp/pmap pool census! ds)))](log-census! before)(doall (cp/pmap pool #(sweep-driver! % (get before %) options) ds)))))report (build-report options swept)]diff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.cljindex 0ecde4dd4bc..7d4158d287a 100644--- a/test/metabase/test/data/interface.clj+++ b/test/metabase/test/data/interface.clj@@ -421,6 +421,23 @@(log/infof "%s has no orphan GC; skipping." driver)[])+;; `println` rather than [[metabase.util.log]] on purpose: the nightly sweep runs as `clojure -X`, and log4j2 emits+;; nothing to stdout in that context -- `info` and `warn` alike are swallowed, so a run's log showed only the few+;; lines that already happened to use `println`.+#_{:clj-kondo/ignore [:discouraged-var]}+(defn print-progress!+ "Print one operator-facing progress line, prefixed with `label` so the interleaved output of concurrently sweeping+ drivers stays attributable.++ Progress is printed as it happens rather than assembled at the end, because the job is expected to be killed at+ its timeout partway through a backlog.++ The newline is part of the string and written once. `println` writes the text and the newline separately, which+ let concurrently sweeping drivers splice their output into the middle of each other's lines."+ [label fmt-str & args]+ (print (str "[" (name label) "] " (apply format fmt-str args) \newline))+ (flush))+(defmulti count-datasets"Census of every test dataset still on each server this driver's tests write to, taken after [[gc-orphans!]] hasrun. Returns `{server-label count}`, keyed the same way as `:server` in [[gc-orphans!]]'s results.