Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add column names / ordering to test-run output (#64448)

Dan Stone3f8edd52ce37
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj
index d59ec61bb70..75427a69ea4 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj
@@ -36,7 +36,9 @@
:- [:map
[:logs :string]
[:error {:optional true} [:map [:message i18n/LocalizedString]]]
- [:output {:optional true} [:map [:rows :any]]]]
+ [:output {:optional true} [:map
+ [:cols [:sequential [:map [:name :string]]]]
+ [:rows [:sequential :any]]]]]
"Evaluate an ad-hoc python transform on a sample of input data.
Intended for short runs for early feedback. Input/output/timeout limits apply."
[_
@@ -80,11 +82,14 @@
:error {:message (i18n/deferred-tru "Python execution failure (exit code {0})" (:exit_code runner-body "?"))}}
:else
- {:logs run-logs
- :output (with-open [in (python-runner/open-output @shared-storage-ref)
- rdr (io/reader in)]
- (let [sample-rows (take output_row_limit (line-seq rdr))]
- {:rows (mapv json/decode (remove str/blank? sample-rows))}))}))))
+ (let [output-manifest (python-runner/read-output-manifest @shared-storage-ref)]
+ {:logs run-logs
+ :output (with-open [in (python-runner/open-output @shared-storage-ref)
+ rdr (io/reader in)]
+ (let [sample-rows (take output_row_limit (line-seq rdr))
+ {:keys [fields]} output-manifest]
+ {:cols (mapv #(select-keys % [:name]) fields)
+ :rows (mapv json/decode (remove str/blank? sample-rows))}))})))))
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/transforms-python` routes."
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj
index a770f70b88e..a27368b879a 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj
@@ -104,7 +104,7 @@
{:keys [error logs output]} (mt/user-http-request :crowberto :post 200 "ee/transforms-python/test-run" body)]
(is (nil? error))
(is (str/includes? logs "out1\nerr1\nout2\nerr2"))
- (is (= {:rows [{:x 42} {:x 43}]} output)))))
+ (is (=? {:cols [{:name "x"}] :rows [{:x 42} {:x 43}]} output)))))
(defn- test-run [& {:keys [program user features source-tables extra-opts]
:or {program ["import pandas as pd" "def transform():" " return pd.DataFrame()"]
@@ -183,3 +183,10 @@
(mt/with-temporary-setting-values [python-runner-test-run-timeout-seconds 1]
(let [response (test-run :program program)]
(is (=? {:body {:error {:message "Python execution timed out"}}} response))))))
+
+(deftest test-run-cols-meta-empty-return-test
+ (let [program ["import pandas as pd"
+ "def transform():"
+ " return pd.DataFrame(columns=['y', 'z', 'x'])"]
+ response (test-run :program program)]
+ (is (=? {:output {:cols [{:name "y"} {:name "z"} {:name "x"}]}} (:body response)))))