The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Python runner timeouts (#64095)
Nicola Mometto0ac7bce92b43
diff --git a/.github/actions/cleanup-python-runner/action.yml b/.github/actions/cleanup-python-runner/action.ymlindex cec3496b177..2e116e93c34 100644--- a/.github/actions/cleanup-python-runner/action.yml+++ b/.github/actions/cleanup-python-runner/action.yml@@ -25,7 +25,7 @@ runs:shell: bash- name: Upload python-runner logs- if: inputs.upload-logs == 'true' && inputs.logs-artifact-name != ''+ if: ${{inputs.upload-logs == 'true' && inputs.logs-artifact-suffix != ''}}uses: actions/upload-artifact@v4with:name: python-runner-logs-${{inputs.logs-artifact-suffix}}diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex 33783ceba72..df4110792ad 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -160,7 +160,7 @@ jobs:uses: ./.github/actions/cleanup-python-runnerwith:upload-logs: ${{ steps.test-driver.outcome == 'failure' }}- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}be-tests-clickhouse-ee:if: ${{ !inputs.skip }}@@ -230,7 +230,7 @@ jobs:uses: ./.github/actions/cleanup-python-runnerwith:upload-logs: ${{ steps.test-driver.outcome == 'failure' }}- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}be-tests-druid-ee:needs: determine-driver-skips@@ -418,7 +418,7 @@ jobs:uses: ./.github/actions/cleanup-python-runnerwith:upload-logs: ${{ steps.test-driver.outcome == 'failure' }}- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}be-tests-mysql-mariadb:if: ${{ !inputs.skip }}@@ -540,7 +540,7 @@ jobs:uses: ./.github/actions/cleanup-python-runnerwith:upload-logs: ${{ steps.test-driver.outcome == 'failure' }}- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}be-tests-mongo:needs: determine-driver-skips@@ -609,7 +609,7 @@ jobs:uses: ./.github/actions/cleanup-python-runnerwith:upload-logs: ${{ steps.test-driver.outcome == 'failure' }}- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}be-tests-mongo-ssl:needs: determine-driver-skips@@ -880,7 +880,7 @@ jobs:uses: ./.github/actions/cleanup-python-runnerwith:upload-logs: ${{ steps.test-driver.outcome == 'failure' }}- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}be-tests-presto-jdbc-ee:needs: determine-driver-skips@@ -1018,7 +1018,7 @@ jobs:uses: ./.github/actions/cleanup-python-runnerwith:upload-logs: ${{ steps.test-driver.outcome == 'failure' }}- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}be-tests-snowflake-ee:if: ${{ !inputs.skip }}@@ -1085,7 +1085,7 @@ jobs:uses: ./.github/actions/cleanup-python-runnerwith:upload-logs: ${{ steps.test-driver.outcome == 'failure' }}- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}be-tests-sparksql-ee:needs: determine-driver-skips@@ -1221,7 +1221,7 @@ jobs:uses: ./.github/actions/cleanup-python-runnerwith:upload-logs: ${{ steps.test-driver.outcome == 'failure' }}- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}# TODO(dpsutton, 2025-08-20): Re-enable starburst when we figure out why it randomly dies in CI# be-tests-starburst-ee:diff --git a/enterprise/backend/src/metabase_enterprise/transforms/core.clj b/enterprise/backend/src/metabase_enterprise/transforms/core.cljindex 313b6008d6d..f76b1d7bee9 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/core.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/core.clj@@ -1,9 +1,12 @@(ns metabase-enterprise.transforms.core"API namespace for the `metabase-enterprise.transform` module."(:require+ [metabase-enterprise.transforms.models.transform-run][metabase-enterprise.transforms.settings][potemkin :as p]))(p/import-vars[metabase-enterprise.transforms.settings- transform-timeout])+ transform-timeout]+ [metabase-enterprise.transforms.models.transform-run+ timeout-run!])diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljindex 43a88f38df3..810ea7b4dea 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -6,6 +6,7 @@[metabase-enterprise.transforms-python.python-runner :as python-runner][metabase-enterprise.transforms-python.s3 :as s3][metabase-enterprise.transforms-python.settings :as transforms-python.settings]+ [metabase-enterprise.transforms.core :as transforms][metabase-enterprise.transforms.instrumentation :as transforms.instrumentation][metabase-enterprise.transforms.util :as transforms.util][metabase.driver :as driver]@@ -264,12 +265,15 @@(when (seq events)(replace-python-logs! message-log events))(if (not= 200 status)- (throw (ex-info "Python runner call failed"- {:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))- :status-code 400- :api-status-code status- :body body- :events events}))+ (do+ (when (:timeout body)+ (transforms/timeout-run! run-id))+ (throw (ex-info "Python runner call failed"+ {:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))+ :status-code 400+ :api-status-code status+ :body body+ :events events})))(try(let [temp-path (Files/createTempFile "transform-output-" ".jsonl" (u/varargs FileAttribute))temp-file (.toFile temp-path)]diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljindex da7230eb72e..f420f1f2734 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -183,7 +183,7 @@payload {:code code:library (t2/select-fn->fn :path :source :model/PythonLibrary)- :timeout (* 30 60) ;; 30 mins harcoded for now+ :timeout (transforms-python.settings/python-runner-timeout-seconds):request_id run-id:output_url (:url output):output_manifest_url (:url output-manifest)diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/settings.cljindex f451e84890e..850eb8e4de0 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj@@ -113,3 +113,14 @@:export? false:encryption :no:audit :getter)++(setting/defsetting python-runner-timeout-seconds+ (deferred-tru "Timeout in seconds for Python script execution. Defaults to 30 minutes (1800 seconds).")+ :type :integer+ :visibility :admin+ :default 1800 ;; 30 minutes+ :feature :transforms-python+ :doc false+ :export? false+ :encryption :no+ :audit :getter)diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.cljindex 520759892f7..b79f175b2d6 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj@@ -6,6 +6,7 @@[metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]][metabase-enterprise.transforms.util :as transforms.util][metabase.test :as mt]+ [metabase.test.util :as test.util][toucan2.core :as t2])(:import(java.util.concurrent CountDownLatch)))@@ -92,3 +93,37 @@(is (= [[1 "a"] [2 "b"] [3 "c"]] (transforms.tu/table-rows table-name))"Table should contain the expected data after swap")))))))))))++(deftest python-transform-timeout-status-test+ (testing "Python transform execution sets correct timeout status when script times out"+ (mt/test-drivers #{:postgres}+ (mt/with-premium-features #{:transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]+ (with-transform-cleanup! [target {:type "table"+ :schema schema+ :name "timeout_test"}]+ (test.util/with-temporary-setting-values [python-runner-timeout-seconds 5]+ (let [long-running-code (str "import time\n"+ "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " time.sleep(10) # Sleep longer than timeout\n"+ " return pd.DataFrame({'result': ['should_not_reach_here']})")+ transform-def {:name "Python Transform Timeout Test"+ :source {:type "python"+ :source-tables {}+ :body long-running-code}+ :target (assoc target :database (mt/id))}]+ (mt/with-temp [:model/Transform transform transform-def]+ (let [{:keys [run_id]} (try+ (transforms-python.execute/execute-python-transform! transform {:run-method :manual})+ (catch Exception _+ ;; We expect this to fail due to timeout+ {:run_id (t2/select-one-fn :id :model/TransformRun+ :transform_id (:id transform)+ {:order-by [[:start_time :desc]]})}))+ run-status (t2/select-one-fn :status :model/TransformRun :id run_id)]+ (testing "Transform run should have timeout status"+ (is (= :timeout run-status)+ "Transform run status should be :timeout when Python script times out")))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.cljindex 705f758812f..c542983351c 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj@@ -15,6 +15,7 @@[metabase.test :as mt][metabase.test.data.interface :as tx][metabase.test.data.sql :as sql.tx]+ [metabase.test.util :as tu][metabase.util :as u][metabase.util.json :as json][toucan2.core :as t2]))@@ -494,3 +495,18 @@;; cleanup(driver/drop-table! driver db-id qualified-table-name)))))++(deftest python-runner-timeout-test+ (testing "Python script execution respects timeout setting"+ (mt/with-premium-features #{:transforms-python}+ (tu/with-temporary-setting-values [python-runner-timeout-seconds 5]+ (let [long-running-code (str "import time\n"+ "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " time.sleep(10) # Sleep longer than timeout\n"+ " return pd.DataFrame({'result': ['should_not_reach_here']})")+ result (execute! {:code long-running-code})]+ (testing "Script should timeout after 5 seconds"+ (is (contains? result :error))+ (is (str/includes? (:error result) "timeout"))))))))diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsxindex aa144670cf4..0d616696c4c 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsx@@ -17,6 +17,13 @@ export function PythonRunnerSettingsPage() {placeholder="http://localhost:5001"inputType="text"/>+ <AdminSettingInput+ name="python-runner-timeout-seconds"+ title={t`Python Script Execution Timeout`}+ description={t`Timeout in seconds for Python script execution. Defaults to 30 minutes (1800 seconds).`}+ placeholder="1800"+ inputType="number"+ /></SettingsSection><SettingsSection title={t`S3 Storage Configuration`}><AdminSettingInputdiff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.tsindex 1e87e5ac8e3..035b0290445 100644--- a/frontend/src/metabase-types/api/settings.ts+++ b/frontend/src/metabase-types/api/settings.ts@@ -687,6 +687,7 @@ export interface EnterpriseSettings extends Settings {"python-storage-s-3-secret-key"?: string | null;"python-storage-s-3-container-endpoint"?: string | null;"python-storage-s-3-path-style-access"?: boolean | null;+ "python-runner-timeout-seconds"?: number | null;/*** @deprecated*/