Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Python runner timeouts (#64095)

Nicola Mometto0ac7bce92b43
diff --git a/.github/actions/cleanup-python-runner/action.yml b/.github/actions/cleanup-python-runner/action.yml
index cec3496b177..2e116e93c34 100644
--- a/.github/actions/cleanup-python-runner/action.yml
+++ b/.github/actions/cleanup-python-runner/action.yml
@@ -25,7 +25,7 @@ runs:
shell: bash
- name: Upload python-runner logs
- if: inputs.upload-logs == 'true' && inputs.logs-artifact-name != ''
+ if: ${{inputs.upload-logs == 'true' && inputs.logs-artifact-suffix != ''}}
uses: actions/upload-artifact@v4
with:
name: python-runner-logs-${{inputs.logs-artifact-suffix}}
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index 33783ceba72..df4110792ad 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -160,7 +160,7 @@ jobs:
uses: ./.github/actions/cleanup-python-runner
with:
upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}
+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
be-tests-clickhouse-ee:
if: ${{ !inputs.skip }}
@@ -230,7 +230,7 @@ jobs:
uses: ./.github/actions/cleanup-python-runner
with:
upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}
+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
be-tests-druid-ee:
needs: determine-driver-skips
@@ -418,7 +418,7 @@ jobs:
uses: ./.github/actions/cleanup-python-runner
with:
upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}
+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
be-tests-mysql-mariadb:
if: ${{ !inputs.skip }}
@@ -540,7 +540,7 @@ jobs:
uses: ./.github/actions/cleanup-python-runner
with:
upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}
+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
be-tests-mongo:
needs: determine-driver-skips
@@ -609,7 +609,7 @@ jobs:
uses: ./.github/actions/cleanup-python-runner
with:
upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}
+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
be-tests-mongo-ssl:
needs: determine-driver-skips
@@ -880,7 +880,7 @@ jobs:
uses: ./.github/actions/cleanup-python-runner
with:
upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}
+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
be-tests-presto-jdbc-ee:
needs: determine-driver-skips
@@ -1018,7 +1018,7 @@ jobs:
uses: ./.github/actions/cleanup-python-runner
with:
upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}
+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
be-tests-snowflake-ee:
if: ${{ !inputs.skip }}
@@ -1085,7 +1085,7 @@ jobs:
uses: ./.github/actions/cleanup-python-runner
with:
upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}
+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
be-tests-sparksql-ee:
needs: determine-driver-skips
@@ -1221,7 +1221,7 @@ jobs:
uses: ./.github/actions/cleanup-python-runner
with:
upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
- logs-artifact-suffix: ${{ github.job }} ${{ matrix.job.name }}
+ logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
# TODO(dpsutton, 2025-08-20): Re-enable starburst when we figure out why it randomly dies in CI
# be-tests-starburst-ee:
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/core.clj b/enterprise/backend/src/metabase_enterprise/transforms/core.clj
index 313b6008d6d..f76b1d7bee9 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/core.clj
@@ -1,9 +1,12 @@
(ns metabase-enterprise.transforms.core
"API namespace for the `metabase-enterprise.transform` module."
(:require
+ [metabase-enterprise.transforms.models.transform-run]
[metabase-enterprise.transforms.settings]
[potemkin :as p]))
(p/import-vars
[metabase-enterprise.transforms.settings
- transform-timeout])
+ transform-timeout]
+ [metabase-enterprise.transforms.models.transform-run
+ timeout-run!])
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index 43a88f38df3..810ea7b4dea 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -6,6 +6,7 @@
[metabase-enterprise.transforms-python.python-runner :as python-runner]
[metabase-enterprise.transforms-python.s3 :as s3]
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+ [metabase-enterprise.transforms.core :as transforms]
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
@@ -264,12 +265,15 @@
(when (seq events)
(replace-python-logs! message-log events))
(if (not= 200 status)
- (throw (ex-info "Python runner call failed"
- {:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))
- :status-code 400
- :api-status-code status
- :body body
- :events events}))
+ (do
+ (when (:timeout body)
+ (transforms/timeout-run! run-id))
+ (throw (ex-info "Python runner call failed"
+ {:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))
+ :status-code 400
+ :api-status-code status
+ :body body
+ :events events})))
(try
(let [temp-path (Files/createTempFile "transform-output-" ".jsonl" (u/varargs FileAttribute))
temp-file (.toFile temp-path)]
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
index da7230eb72e..f420f1f2734 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
@@ -183,7 +183,7 @@
payload {:code code
:library (t2/select-fn->fn :path :source :model/PythonLibrary)
- :timeout (* 30 60) ;; 30 mins harcoded for now
+ :timeout (transforms-python.settings/python-runner-timeout-seconds)
:request_id run-id
:output_url (:url output)
:output_manifest_url (:url output-manifest)
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj
index f451e84890e..850eb8e4de0 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj
@@ -113,3 +113,14 @@
:export? false
:encryption :no
:audit :getter)
+
+(setting/defsetting python-runner-timeout-seconds
+ (deferred-tru "Timeout in seconds for Python script execution. Defaults to 30 minutes (1800 seconds).")
+ :type :integer
+ :visibility :admin
+ :default 1800 ;; 30 minutes
+ :feature :transforms-python
+ :doc false
+ :export? false
+ :encryption :no
+ :audit :getter)
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
index 520759892f7..b79f175b2d6 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
@@ -6,6 +6,7 @@
[metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.test :as mt]
+ [metabase.test.util :as test.util]
[toucan2.core :as t2])
(:import
(java.util.concurrent CountDownLatch)))
@@ -92,3 +93,37 @@
(is (= [[1 "a"] [2 "b"] [3 "c"]] (transforms.tu/table-rows table-name))
"Table should contain the expected data after swap")))))))))))
+
+(deftest python-transform-timeout-status-test
+ (testing "Python transform execution sets correct timeout status when script times out"
+ (mt/test-drivers #{:postgres}
+ (mt/with-premium-features #{:transforms-python}
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+ (with-transform-cleanup! [target {:type "table"
+ :schema schema
+ :name "timeout_test"}]
+ (test.util/with-temporary-setting-values [python-runner-timeout-seconds 5]
+ (let [long-running-code (str "import time\n"
+ "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " time.sleep(10) # Sleep longer than timeout\n"
+ " return pd.DataFrame({'result': ['should_not_reach_here']})")
+ transform-def {:name "Python Transform Timeout Test"
+ :source {:type "python"
+ :source-tables {}
+ :body long-running-code}
+ :target (assoc target :database (mt/id))}]
+ (mt/with-temp [:model/Transform transform transform-def]
+ (let [{:keys [run_id]} (try
+ (transforms-python.execute/execute-python-transform! transform {:run-method :manual})
+ (catch Exception _
+ ;; We expect this to fail due to timeout
+ {:run_id (t2/select-one-fn :id :model/TransformRun
+ :transform_id (:id transform)
+ {:order-by [[:start_time :desc]]})}))
+ run-status (t2/select-one-fn :status :model/TransformRun :id run_id)]
+ (testing "Transform run should have timeout status"
+ (is (= :timeout run-status)
+ "Transform run status should be :timeout when Python script times out")))))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
index 705f758812f..c542983351c 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
@@ -15,6 +15,7 @@
[metabase.test :as mt]
[metabase.test.data.interface :as tx]
[metabase.test.data.sql :as sql.tx]
+ [metabase.test.util :as tu]
[metabase.util :as u]
[metabase.util.json :as json]
[toucan2.core :as t2]))
@@ -494,3 +495,18 @@
;; cleanup
(driver/drop-table! driver db-id qualified-table-name)))))
+
+(deftest python-runner-timeout-test
+ (testing "Python script execution respects timeout setting"
+ (mt/with-premium-features #{:transforms-python}
+ (tu/with-temporary-setting-values [python-runner-timeout-seconds 5]
+ (let [long-running-code (str "import time\n"
+ "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " time.sleep(10) # Sleep longer than timeout\n"
+ " return pd.DataFrame({'result': ['should_not_reach_here']})")
+ result (execute! {:code long-running-code})]
+ (testing "Script should timeout after 5 seconds"
+ (is (contains? result :error))
+ (is (str/includes? (:error result) "timeout"))))))))
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsx
index aa144670cf4..0d616696c4c 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsx
@@ -17,6 +17,13 @@ export function PythonRunnerSettingsPage() {
placeholder="http://localhost:5001"
inputType="text"
/>
+ <AdminSettingInput
+ name="python-runner-timeout-seconds"
+ title={t`Python Script Execution Timeout`}
+ description={t`Timeout in seconds for Python script execution. Defaults to 30 minutes (1800 seconds).`}
+ placeholder="1800"
+ inputType="number"
+ />
</SettingsSection>
<SettingsSection title={t`S3 Storage Configuration`}>
<AdminSettingInput
diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.ts
index 1e87e5ac8e3..035b0290445 100644
--- a/frontend/src/metabase-types/api/settings.ts
+++ b/frontend/src/metabase-types/api/settings.ts
@@ -687,6 +687,7 @@ export interface EnterpriseSettings extends Settings {
"python-storage-s-3-secret-key"?: string | null;
"python-storage-s-3-container-endpoint"?: string | null;
"python-storage-s-3-path-style-access"?: boolean | null;
+ "python-runner-timeout-seconds"?: number | null;
/**
* @deprecated
*/