The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
use qp directly rather than reaching for internals for python transforms (#64452)
Nicola Mometto6da9f0a91194
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 2f3d34a2bbb..be1b4fc8b02 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2070,6 +2070,7 @@app-dbconfigdriver+ libmodelsquery-processorsettingsdiff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljindex 5b6828f3b5d..7de16b82ca3 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -8,12 +8,8 @@[metabase-enterprise.transforms-python.settings :as transforms-python.settings][metabase-enterprise.transforms.instrumentation :as transforms.instrumentation][metabase.config.core :as config]- [metabase.driver :as driver]- [metabase.query-processor.compile :as qp.compile]+ [metabase.query-processor :as qp][metabase.query-processor.pipeline :as qp.pipeline]- ;; TODO check that querying team are ok with us accessing this directly, otherwise make another plan- ^{:clj-kondo/ignore [:discouraged-namespace]}- [metabase.query-processor.store :as qp.store][metabase.util.i18n :as i18n][metabase.util.json :as json][metabase.util.log :as log]@@ -48,38 +44,6 @@:headers (authorization-headers)}](apply http/request (merge base-options request-options {:method method, :url url}) extra-args)))-(defn- write-jsonl-to-stream! [^OutputStream os col-names reducible-rows]- (let [none? (volatile! true)- writer (-> os- (OutputStreamWriter. StandardCharsets/UTF_8)- (BufferedWriter.))]-- (run! (fn [row]- (when @none? (vreset! none? false))- (let [row-map (zipmap col-names row)]- (json/encode-to row-map writer {})- (.newLine writer)))- reducible-rows)-- ;; Workaround for LocalStack, which doesn't support zero byte files.- (when @none?- (.write writer " "))-- (doto writer- (.flush)- (.close))))--(defn- execute-mbql-query- [driver db-id query respond cancel-chan]- (driver/with-driver driver- (let [native (qp.compile/compile {:type :query, :database db-id :query query})- query {:database db-id- :type :native- :native native}]- (qp.store/with-metadata-provider db-id- (binding [qp.pipeline/*canceled-chan* cancel-chan]- (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond))))))-(defn root-type"Supported type for roundtrip/insertion"[base-type]@@ -93,6 +57,77 @@:type/Text:type/Boolean])))+(defn- maybe-fixup-value [col v]+ (cond+ (nil? (root-type (:base_type col)))+ ;; we're not a supported base type, so we just stringify it+ (when v (json/encode v))++ ;; the clickhouse driver returns bigdecimals for int64 values+ (and (isa? (:base_type col) :type/Integer)+ (or (instance? BigDecimal v)+ (float? v)))+ (bigint v)++ :else+ v))++(defn- write-jsonl-row-to-os-rff+ "Returns a rff that writes query results as JSONL to an OutputStream.++ Only fields present in `fields-meta` are included in the output. Values are processed via+ `maybe-fixup-value` before JSON encoding."+ [^OutputStream os fields-meta {cols-meta :cols}]+ (let [filtered-col-meta (m/index-by :name fields-meta)+ col-names (map :name cols-meta)+ none? (volatile! true)]+ (fn+ ([]+ (-> os+ (OutputStreamWriter. StandardCharsets/UTF_8)+ (BufferedWriter.)))++ ([^BufferedWriter writer]+ ;; Workaround for LocalStack, which doesn't support zero byte files.+ (when @none?+ (.write writer " "))++ (doto writer+ (.flush)+ (.close)))++ ([^BufferedWriter writer row]+ (let [row-map (->>+ (map vector col-names row)+ (filter (fn [[n _]]+ (contains? filtered-col-meta n)))+ (map (fn [[n v]]+ (maybe-fixup-value (filtered-col-meta n) v)))+ (zipmap (filter filtered-col-meta col-names)))]++ (when @none? (vreset! none? false))+ (json/encode-to row-map writer {})++ (doto writer+ (.newLine)))))))++(defn- execute-mbql-query+ [db-id query rff cancel-chan]+ (binding [qp.pipeline/*canceled-chan* (a/go (a/<! cancel-chan))]+ (qp/process-query {:type :query :database db-id :query query} rff)))++(defn- throw-if-cancelled [cancel-chan]+ (when (a/poll! cancel-chan)+ (throw (ex-info "Run cancelled" {:error-type :cancelled}))))++(defn- write-table-data-to-file! [{:keys [db-id table-id fields-meta temp-file cancel-chan limit]}]+ (with-open [os (io/output-stream temp-file)]+ (let [query (cond-> {:source-table table-id} limit (assoc :limit limit))+ rff (fn [cols-meta] (write-jsonl-row-to-os-rff os fields-meta cols-meta))]+ (execute-mbql-query db-id query rff cancel-chan)+ (some-> cancel-chan throw-if-cancelled)+ nil)))+(defn restricted-insert-type"Type for insertion restricted to supported"[base_type]@@ -132,45 +167,6 @@:field_id (:id col-meta)})cols-meta)})-(defn- maybe-fixup-value [col v]- (cond- (nil? (root-type (:base_type col)))- ;; we're not a supported base type, so we just stringify it- (when v (json/encode v))-- ;; the clickhouse driver returns bigdecimals for int64 values- (and (isa? (:base_type col) :type/Integer)- (or (instance? BigDecimal v)- (float? v)))- (bigint v)-- :else- v))--(defn- throw-if-cancelled [cancel-chan]- (when (a/poll! cancel-chan)- (throw (ex-info "Run cancelled" {:error-type :cancelled}))))--(defn- write-table-data-to-file! [{:keys [db-id driver table-id fields-meta temp-file cancel-chan limit]}]- {:pre [(or (nil? limit) (pos-int? limit))]}- (let [query (cond-> {:source-table table-id} limit (assoc :limit limit))- respond (fn [{cols-meta :cols} reducible-rows]- (with-open [os (io/output-stream temp-file)]- (let [filtered-col-meta (m/index-by :name fields-meta)- col-names (map :name cols-meta)- filtered-rows (eduction (map (fn [row]- (->>- (map vector col-names row)- (filter (fn [[n _]]- (contains? filtered-col-meta n)))- (map (fn [[n v]]- (maybe-fixup-value (filtered-col-meta n) v))))))- reducible-rows)]- (write-jsonl-to-stream! os (filter filtered-col-meta col-names) filtered-rows))))]- (execute-mbql-query driver db-id query respond cancel-chan)- (some-> cancel-chan throw-if-cancelled)- nil))-(defn get-logs"Return the logs of the current running python process"[run-id]diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.cljindex 3051ac4ca48..0dbbb9c9c3d 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj@@ -11,7 +11,7 @@[metabase.util :as u][toucan2.core :as t2])(:import- (clojure.lang IDeref IReduceInit)+ (clojure.lang IDeref)(java.io Closeable)(java.time Duration Instant)))@@ -342,26 +342,25 @@(when-not (deref wait-signal 5000 nil)(throw (ex-info "Expected delivery of wait signal within a reasonable amount of time" {}))))- (reducible-proxy [ready-signal- wait-signal- reducible-rows]- (reify IReduceInit- (reduce [_ f init]- (deliver ready-signal true)- (await-signal wait-signal)- (reduce f init reducible-rows))))+ (rf-proxy [ready-signal+ wait-signal+ rf]+ (fn+ ([] (rf))+ ([w] (rf w))+ ([w e]+ (deliver ready-signal true)+ (await-signal wait-signal)+ (rf w e))))(blocking-redefs [{:keys [block]} ready-signal wait-signal](case block:read- (let [f-ref #'transforms-python.python-runner/execute-mbql-query+ (let [f-ref #'transforms-python.python-runner/write-jsonl-row-to-os-rfff @f-ref]{f-ref- (fn [driver db-id query respond cancel-chan]- (f driver db-id query- (fn [metadata reducible-rows]- (respond metadata (reducible-proxy ready-signal wait-signal reducible-rows)))- cancel-chan))})+ (fn [os fields-meta col-meta]+ (rf-proxy ready-signal wait-signal (f os fields-meta col-meta)))}):write(let [f-ref #'transforms-python.execute/transfer-file-to-dbf @f-ref]