Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

13.8 KB
Raw Blame
(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.execute-test
  (:require
   [clojure.test :refer :all]
   [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
   [metabase-enterprise.transforms-python.python-runner :as python-runner]
   [metabase.test :as mt]
   [metabase.test.util :as test.util]
   [metabase.transforms-base.util :as transforms-base.u]
   [metabase.transforms.test-dataset :as transforms-dataset]
   [metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
   [metabase.transforms.util :as transforms.u]
   [toucan2.core :as t2])
  (:import
   (java.net SocketTimeoutException)
   (java.time Duration)
   (java.util.concurrent CountDownLatch)))

(set! *warn-on-reflection* true)

(deftest atomic-python-transform-swap-test
  (testing "Python transform execution with atomic table swap"
    (mt/test-drivers #{:mysql :postgres}
      (mt/with-premium-features #{:transforms-python :transforms-basic}
        (mt/dataset transforms-dataset/transforms-test
          (let [schema (t2/select-one-fn :schema :model/Table :id (mt/id :transforms_products))]
            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
                                                                     :schema schema
                                                                     :name   "swap_tbl"}]
              (let [initial-transform {:name   "Python Transform Initial"
                                       :source {:type            :python
                                                :source-tables   []
                                                :source-database (mt/id)
                                                :body            (str "import pandas as pd\n"
                                                                      "\n"
                                                                      "def transform():\n"
                                                                      "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
                                       :target (assoc target :database (mt/id))}]
                (mt/with-temp [:model/Transform transform initial-transform]
                  (transforms-python.execute/execute-python-transform! transform {:run-method :manual})
                  (transforms.tu/wait-for-table table-name 10000)
                  (let [initial-rows (transforms.tu/table-rows table-name)]
                    (is (= [["Alice" 25] ["Bob" 30]] initial-rows) "Initial data should be Alice and Bob")
                    (t2/update! :model/Transform :id (:id transform)
                                {:source {:type            :python
                                          :source-tables   []
                                          :source-database (mt/id)
                                          :body            (str "import pandas as pd\n"
                                                                "\n"
                                                                "def transform():\n"
                                                                "    return pd.DataFrame({'name': ['Charlie', 'Diana', 'Eve'], 'age': [35, 40, 45]})")}}))
                  (let [swap-latch (CountDownLatch. 1)
                        original-rename-tables-atomic! transforms-base.u/rename-tables!]
                    (mt/with-dynamic-fn-redefs [transforms-base.u/rename-tables! (fn [driver db-id rename-pairs]
                                                                                   (.await swap-latch)
                                                                                   (original-rename-tables-atomic! driver db-id rename-pairs))]
                      (let [transform-future (future
                                               (transforms-python.execute/execute-python-transform!
                                                (t2/select-one :model/Transform :id (:id transform))
                                                {:run-method :manual}))]
                        (is (= [["Alice" 25] ["Bob" 30]]
                               (transforms.tu/table-rows table-name))
                            "Original data should still be accessible during transform")
                        (.countDown swap-latch)
                        @transform-future
                        (is (= [["Charlie" 35] ["Diana" 40] ["Eve" 45]]
                               (transforms.tu/table-rows table-name))
                            "Table should now contain Charlie, Diana, and Eve")))))))))))))

(deftest python-transform-temp-table-cleanup-test
  (testing "Python transform cleans up temp tables on success"
    (mt/test-drivers #{:mysql :postgres}
      (mt/with-premium-features #{:transforms-python :transforms-basic}
        (mt/dataset transforms-dataset/transforms-test
          (let [schema (t2/select-one-fn :schema :model/Table :id (mt/id :transforms_products))]
            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
                                                                     :schema schema
                                                                     :name   "cleanup_"}]
              (let [transform-def {:name   "Python Transform Cleanup"
                                   :source {:type            :python
                                            :source-tables   []
                                            :source-database (mt/id)
                                            :body            (str "import pandas as pd\n"
                                                                  "\n"
                                                                  "def transform():\n"
                                                                  "    return pd.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']})")}
                                   :target (assoc target :database (mt/id))}]
                (mt/with-temp [:model/Transform transform transform-def]
                  (transforms-python.execute/execute-python-transform! transform {:run-method :manual})
                  (transforms.tu/wait-for-table table-name 10000)
                  (transforms-python.execute/execute-python-transform! transform {:run-method :manual})
                  (let [db-id (mt/id)
                        tables (t2/select :model/Table :db_id db-id :active true)]
                    (is (not-any? transforms.u/is-temp-transform-table? tables)
                        "No temp tables should remain after successful Python transform")
                    (is (= [[1 "a"] [2 "b"] [3 "c"]] (transforms.tu/table-rows table-name))
                        "Table should contain the expected data after swap")))))))))))

(deftest python-transform-timeout-status-test
  (testing "Python transform execution sets correct timeout status when script times out"
    (mt/test-drivers #{:postgres}
      (mt/with-premium-features #{:transforms-python :transforms-basic}
        (mt/dataset transforms-dataset/transforms-test
          (let [schema (t2/select-one-fn :schema :model/Table :id (mt/id :transforms_products))]
            (with-transform-cleanup! [target {:type   "table"
                                              :schema schema
                                              :name   "timeout_test"}]
              (test.util/with-temporary-setting-values [python-runner-timeout-seconds 5]
                (let [long-running-code (str "import time\n"
                                             "import pandas as pd\n"
                                             "\n"
                                             "def transform():\n"
                                             "    time.sleep(10)  # Sleep longer than timeout\n"
                                             "    return pd.DataFrame({'result': ['should_not_reach_here']})")
                      transform-def {:name   "Python Transform Timeout Test"
                                     :source {:type            :python
                                              :source-tables   []
                                              :source-database (mt/id)
                                              :body            long-running-code}
                                     :target (assoc target :database (mt/id))}]
                  (mt/with-temp [:model/Transform transform transform-def]
                    (let [{:keys [run_id]} (try
                                             (transforms-python.execute/execute-python-transform! transform {:run-method :manual})
                                             (catch Exception _
                                               ;; We expect this to fail due to timeout
                                               {:run_id (t2/select-one-fn :id :model/TransformRun
                                                                          :transform_id (:id transform)
                                                                          {:order-by [[:start_time :desc]]})}))
                          run-status (t2/select-one-fn :status :model/TransformRun :id run_id)]
                      (testing "Transform run should have timeout status"
                        (is (= :timeout run-status)
                            "Transform run status should be :timeout when Python script times out")))))))))))))

(deftest python-transform-unresolved-source-table-test
  (testing "Python transform execution throws when source table cannot be resolved"
    (mt/test-drivers #{:postgres}
      (mt/with-premium-features #{:transforms-python :transforms-basic}
        (mt/dataset transforms-dataset/transforms-test
          (let [schema (t2/select-one-fn :schema :model/Table :id (mt/id :transforms_products))]
            (with-transform-cleanup! [target {:type   "table"
                                              :schema schema
                                              :name   "unresolved_test"}]
              (mt/with-temp [:model/Transform transform {:name   "Transform with unresolved ref"
                                                         :source {:type            :python
                                                                  :body            "def transform(input): return input"
                                                                  :source-database (mt/id)
                                                                  :source-tables   [{:alias       "input"
                                                                                     :database_id (mt/id)
                                                                                     :schema      schema
                                                                                     :table       "nonexistent_table"}]}
                                                         :target (assoc target :database (mt/id))}]
                (testing "Execution throws with informative error message"
                  (is (thrown-with-msg?
                       clojure.lang.ExceptionInfo
                       #"Tables not found:.*nonexistent_table"
                       (transforms-python.execute/execute-python-transform! transform {:run-method :manual}))))))))))))

(deftest python-transform-unresolved-source-table-no-schema-test
  (testing "Python transform error message omits schema when nil"
    (mt/test-drivers #{:postgres}
      (mt/with-premium-features #{:transforms-python :transforms-basic}
        (mt/dataset transforms-dataset/transforms-test
          (with-transform-cleanup! [target {:type   "table"
                                            :schema nil
                                            :name   "unresolved_no_schema_test"}]
            (mt/with-temp [:model/Transform transform {:name   "Transform with unresolved ref no schema"
                                                       :source {:type            :python
                                                                :body            "def transform(input): return input"
                                                                :source-database (mt/id)
                                                                :source-tables   [{:alias       "input"
                                                                                   :database_id (mt/id)
                                                                                   :schema      nil
                                                                                   :table       "missing_table"}]}
                                                       :target (assoc target :database (mt/id))}]
              (is (thrown-with-msg?
                   clojure.lang.ExceptionInfo
                   #"Tables not found: missing_table"
                   (transforms-python.execute/execute-python-transform! transform {:run-method :manual}))))))))))

(deftest log-poll-survives-transient-timeout-test
  (testing "a read timeout while polling for logs is retried instead of ending the loop"
    (let [calls (atom 0)
          saved (atom nil)]
      (with-redefs [transforms-python.execute/python-message-loop-sleep-duration (Duration/ofMillis 0)
                    python-runner/get-logs
                    (fn [run-id]
                      (condp = (swap! calls inc)
                        1 (throw (SocketTimeoutException. "read timed out"))
                        2 {:status 200 :body {:execution_id run-id :events [{:message "hi"}]}}
                        {:status 500 :body {}}))
                    transforms-python.execute/save-log-to-transform-run-message!
                    (fn [_run-id message-log] (reset! saved @message-log))]
        (#'transforms-python.execute/python-message-update-loop! 42 (atom {}))
        (is (= 3 @calls) "polling carried on past the timeout")
        (is (= [{:message "hi"}] (:python @saved)) "and the logs that arrived after it were still captured")))))