The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Transform cancellation test & fixes (#63833)
Dan Stone11856c2aeac0
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljindex 06427e90138..5b6828f3b5d 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.transforms-python.python-runner(:require[clj-http.client :as http]+ [clojure.core.async :as a][clojure.java.io :as io][medley.core :as m][metabase-enterprise.transforms-python.s3 :as s3]@@ -38,14 +39,14 @@(defn- python-runner-request"Helper function for making HTTP requests to the python runner service."- [server-url method endpoint & [request-options]]+ [server-url method endpoint request-options & extra-args](let [url (str server-url "/v1" endpoint)base-options {:content-type :json:accept :json:throw-exceptions false:as :json:headers (authorization-headers)}]- (http/request (merge base-options request-options {:method method, :url url}))))+ (apply http/request (merge base-options request-options {:method method, :url url}) extra-args)))(defn- write-jsonl-to-stream! [^OutputStream os col-names reducible-rows](let [none? (volatile! true)@@ -146,24 +147,29 @@:elsev))+(defn- throw-if-cancelled [cancel-chan]+ (when (a/poll! cancel-chan)+ (throw (ex-info "Run cancelled" {:error-type :cancelled}))))+(defn- write-table-data-to-file! [{:keys [db-id driver table-id fields-meta temp-file cancel-chan limit]}]{:pre [(or (nil? limit) (pos-int? limit))]}- (let [query (cond-> {:source-table table-id} limit (assoc :limit limit))]- (execute-mbql-query driver db-id query- (fn [{cols-meta :cols} reducible-rows]- (with-open [os (io/output-stream temp-file)]- (let [filtered-col-meta (m/index-by :name fields-meta)- col-names (map :name cols-meta)- filtered-rows (eduction (map (fn [row]- (->>- (map vector col-names row)- (filter (fn [[n _]]- (contains? filtered-col-meta n)))- (map (fn [[n v]]- (maybe-fixup-value (filtered-col-meta n) v))))))- reducible-rows)]- (write-jsonl-to-stream! os (filter filtered-col-meta col-names) filtered-rows))))- cancel-chan)))+ (let [query (cond-> {:source-table table-id} limit (assoc :limit limit))+ respond (fn [{cols-meta :cols} reducible-rows]+ (with-open [os (io/output-stream temp-file)]+ (let [filtered-col-meta (m/index-by :name fields-meta)+ col-names (map :name cols-meta)+ filtered-rows (eduction (map (fn [row]+ (->>+ (map vector col-names row)+ (filter (fn [[n _]]+ (contains? filtered-col-meta n)))+ (map (fn [[n v]]+ (maybe-fixup-value (filtered-col-meta n) v))))))+ reducible-rows)]+ (write-jsonl-to-stream! os (filter filtered-col-meta col-names) filtered-rows))))]+ (execute-mbql-query driver db-id query respond cancel-chan)+ (some-> cancel-chan throw-if-cancelled)+ nil))(defn get-logs"Return the logs of the current running python process"diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.cljindex 77e8db1bf51..3051ac4ca48 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj@@ -3,15 +3,17 @@[clojure.string :as str][clojure.test :refer :all][metabase-enterprise.transforms-python.execute :as transforms-python.execute]+ [metabase-enterprise.transforms-python.python-runner :as transforms-python.python-runner][metabase-enterprise.transforms.test-dataset :as transforms-dataset][metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup! get-test-schema]][metabase.driver :as driver][metabase.test :as mt]+ [metabase.util :as u][toucan2.core :as t2])(:import- (clojure.lang IDeref)+ (clojure.lang IDeref IReduceInit)(java.io Closeable)- (java.time Duration)))+ (java.time Duration Instant)))(set! *warn-on-reflection* true)@@ -156,6 +158,37 @@(= (first xs) (first ys)) (recur (rest xs) (rest ys)):else (recur (rest xs) ys)))+(defn- get-last-run [transform-id]+ (:last_run (mt/user-http-request :crowberto :get 200 (format "ee/transform/%d" transform-id))))++(defn- open-message-value-observer+ "Polls the `:message` state of the last run and stores the value every time it is different to the last observation.+ Close with .close, grab the vector of values with deref."+ ^Closeable [transform-id]+ (let [states (atom [])+ fut (future+ (try+ (loop []+ (let [{:keys [message]} (get-last-run transform-id)]+ (cond+ (.isInterrupted (Thread/currentThread))+ nil++ ;; same message as last time+ (= message (peek @states))+ (recur)++ ;; new message value+ :else (do (swap! states conj message)+ (recur)))))+ (catch InterruptedException _ nil)))]+ (reify IDeref+ (deref [_] @states)+ Closeable+ (close [_]+ (future-cancel fut)+ (assert (not= :timeout (try (deref fut 1000 :timeout) (catch Throwable _))) "Observation thread did not exit!")))))+(deftest python-transform-logging-test(letfn [(program->source [program](->> (concat ["import pandas as pd"@@ -183,34 +216,6 @@(when (= :succeeded expect-status)(transforms.tu/wait-for-table (:name target) 5000)))- (get-last-run [transform-id]- (:last_run (mt/user-http-request :crowberto :get 200 (format "ee/transform/%d" transform-id))))-- (open-message-value-observer [transform-id]- (let [states (atom [])- fut (future- (try- (loop []- (let [{:keys [message]} (get-last-run transform-id)]- (cond- (.isInterrupted (Thread/currentThread))- nil-- ;; same message as last time- (= message (peek @states))- (recur)-- ;; new message value- :else (do (swap! states conj message)- (recur)))))- (catch InterruptedException _ nil)))]- (reify IDeref- (deref [_] @states)- Closeable- (close [_]- (future-cancel fut)- (assert (not= :timeout (try (deref fut 1000 :timeout) (catch Throwable _))) "Observation thread did not exit!")))))-(run-scenario [scenario schema](with-redefs [transforms-python.execute/python-message-loop-sleep-duration Duration/ZEROtransforms-python.execute/transfer-file-to-db (if-some [e (:writeback-ex scenario)]@@ -220,7 +225,7 @@:schema schema:name "result"}](let [transform-id (create-transform scenario target)- observed-messages (with-open [observer ^Closeable (open-message-value-observer transform-id)]+ observed-messages (with-open [observer (open-message-value-observer transform-id)](block-on-run scenario target transform-id)@observer)last-run (get-last-run transform-id)]@@ -315,6 +320,176 @@:db_id (:id target-db)}(:table response))))))))+;; NOTE: The test does not simulate a multi-node setup.+;; In the general case we should expect the signal some time after the /cancel API call, so smaller runs may never see the signal.+;; If you extend the size of the tables / run time then any behaviour this test exposes should also be possible for delayed signals.+#_{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]} ; todo factor to maybe some private helpers or share test impl for multiple top-level tests+(deftest python-cancellation-test+ (letfn [(create-transform [{:keys [desc program]} target]+ (mt/user-http-request :crowberto :post 200 "ee/transform"+ {:name (format "Python cancellation test: %s" desc)+ :source {:type "python"+ :body (str/join "\n" program)+ :source-database (mt/id)+ :source-tables {"transforms_customers" (mt/id :transforms_customers)}}+ :target (assoc target :database (mt/id))}))++ ;; using clojure-ey coordination with promises (I know j.u.c could be better here)+ ;; goal is blocking at the right point during the run so that we can test the cancellation behaviour+ ;; for a particular stage during the run or area of interest (e.g. during a table copy out to shared storage)++ (await-signal [wait-signal]+ (when-not (deref wait-signal 5000 nil)+ (throw (ex-info "Expected delivery of wait signal within a reasonable amount of time" {}))))++ (reducible-proxy [ready-signal+ wait-signal+ reducible-rows]+ (reify IReduceInit+ (reduce [_ f init]+ (deliver ready-signal true)+ (await-signal wait-signal)+ (reduce f init reducible-rows))))++ (blocking-redefs [{:keys [block]} ready-signal wait-signal]+ (case block+ :read+ (let [f-ref #'transforms-python.python-runner/execute-mbql-query+ f @f-ref]+ {f-ref+ (fn [driver db-id query respond cancel-chan]+ (f driver db-id query+ (fn [metadata reducible-rows]+ (respond metadata (reducible-proxy ready-signal wait-signal reducible-rows)))+ cancel-chan))})+ :write+ (let [f-ref #'transforms-python.execute/transfer-file-to-db+ f @f-ref]+ {f-ref+ (fn [& args]+ (deliver ready-signal true)+ (await-signal wait-signal)+ (apply f args))})+ nil))++ (run-scenario [{:keys [expect-script] :as scenario} target]+ (let [ready-signal (promise) ; test blocks: until the run is ready to be cancelled+ wait-signal (promise) ; run blocks: until the test has cancelled+ finished-signal (promise) ; test blocks: until the run has finished (exceptionally or not)+ exec-fn @#'transforms-python.execute/execute-python-transform!+ redefs (blocking-redefs scenario ready-signal wait-signal)]+ (with-redefs-fn+ (merge+ {#'transforms-python.execute/execute-python-transform!+ (fn [& args]+ (try+ (apply exec-fn args)+ (finally+ (deliver finished-signal true))))}+ redefs)+ (fn []+ (let [{transform-id :id} (create-transform scenario target)]+ (with-open [message-observer (open-message-value-observer transform-id)]+ (let [_ (mt/user-http-request :crowberto :post 202 (format "ee/transform/%d/run" transform-id))+ ;; Cancellation currently overwrites the message, so we should wait for some log output if we expect it+ ;; Ideally we would not use the message field for log output, or otherwise avoid having logs being lost on cancellation.+ _ (when expect-script+ (u/poll {:thunk #(deref message-observer)+ :done? #(some #{"script started"} %)+ :interval-ms 1+ :timeout-ms 5000}))+ _ (when redefs (await-signal ready-signal))+ _ (get-last-run transform-id)+ _ (mt/user-http-request :crowberto :post 204 (format "ee/transform/%d/cancel" transform-id))+ _ (deliver wait-signal true)+ _ (await-signal finished-signal)+ last-run (get-last-run transform-id)]+ {:messages @message-observer+ :last-run last-run})))))))]++ (let [blocking-script+ ["import time"+ "import pandas as pd"+ "def transform(transforms_customers):"+ " print(\"script started\")"+ ;; longer sleep gives more room for slow CI when checking for 'how quickly should a trivial run complete'+ ;; this is important for figuring out whether new runs can happen after a cancelled job that might have blocked the runner+ ;; N.B. if cancellation works, we do not pay the timeout during the test, the test does not always wait 30 seconds!+ " time.sleep(30)"+ " return pd.DataFrame({'x': [42]})"]++ non-blocking-script+ ["import pandas as pd"+ "def transform(transforms_customers):"+ " print(\"script started\")"+ " return pd.DataFrame({'x': [42]})"]++ scenarios+ [{:desc "during python eval"+ :program blocking-script+ :expect-script true+ :expect-write false+ :expect-status :canceled}+ {:desc "during table read"+ :program non-blocking-script+ :block :read+ :expect-script false+ :expect-write false+ :expect-status :canceled}+ {:desc "during table write"+ :program non-blocking-script+ :block :write,+ :expect-script true+ :expect-write true ; note: the cancellation signal is currently ignored during the write phase+ :expect-status :canceled}]]++ (doseq [{:keys [desc expect-status expect-script expect-write] :as scenario} scenarios]+ (mt/test-drivers (-> (mt/normal-drivers-with-feature :transforms/python)+ ; these drivers cause timing issues, could be fixed if we change timeout / time variables in test+ (disj :snowflake :bigquery-cloud-sdk :redshift :mongo))+ (mt/with-premium-features #{:transforms :transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]+ (with-redefs [transforms-python.execute/python-message-loop-sleep-duration Duration/ZERO]+ (with-transform-cleanup! [target {:type "table"+ :schema schema+ :name "result"}]+ (testing desc+ (testing "initial cancellation"+ (let [scenario-result (run-scenario scenario target)+ {:keys [messages last-run]} scenario-result]+ (is (= (name expect-status) (:status last-run)))++ (when (some? expect-script)+ (if expect-script+ (testing "script should have started"+ (is (some #(str/includes? % "script started") messages)))+ (testing "script should not have started"+ (is (not-any? #(str/includes? % "script started") messages))+ (is (not (str/includes? (str (:message last-run)) "script started"))))))++ (when (some? expect-write)+ (testing "table existence"+ (is (= expect-write (driver/table-exists? driver/*driver* (mt/db) target)))))))))++ ; todo We have not yet covered the case where we rerun the same transform while there might be some hangover.+ ; Cancellation addresses the transform and not the run, there is shared mutable state and races on it are possible+ (testing "the runner is not blocked for a new run"+ (with-transform-cleanup! [target {:type "table"+ :schema schema+ :name "result"}]+ (let [{transform-id :id} (create-transform {:desc "normal run", :program non-blocking-script} target)+ _ (transforms.tu/test-run transform-id)+ {:keys [status start_time end_time]} (get-last-run transform-id)+ start-inst (some-> start_time Instant/parse)+ end-inst (some-> end_time Instant/parse)+ duration (when (and start-inst end-inst) (Duration/between start-inst end-inst))]+ (is (= "succeeded" status))+ (when duration+ ;; 10 seconds is an estimate of the maximum time we should expect the normal run to take.+ ;; if the runner was blocked for 30 seconds, and only then did the new run get scheduled - we would exceed this time.+ (is (< (.toSeconds duration) 10)))))))))))))))+(deftest python-transform-schema-change-integration-test(testing "Python transform handles schema changes using appropriate rename strategy"(mt/test-drivers (disj (mt/normal-drivers-with-feature :transforms/python)