Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

use qp directly rather than reaching for internals for python transforms (#64452)

Nicola Mometto6da9f0a91194
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 2f3d34a2bbb..be1b4fc8b02 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2070,6 +2070,7 @@
app-db
config
driver
+ lib
models
query-processor
settings
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
index 5b6828f3b5d..7de16b82ca3 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
@@ -8,12 +8,8 @@
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
[metabase.config.core :as config]
- [metabase.driver :as driver]
- [metabase.query-processor.compile :as qp.compile]
+ [metabase.query-processor :as qp]
[metabase.query-processor.pipeline :as qp.pipeline]
- ;; TODO check that querying team are ok with us accessing this directly, otherwise make another plan
- ^{:clj-kondo/ignore [:discouraged-namespace]}
- [metabase.query-processor.store :as qp.store]
[metabase.util.i18n :as i18n]
[metabase.util.json :as json]
[metabase.util.log :as log]
@@ -48,38 +44,6 @@
:headers (authorization-headers)}]
(apply http/request (merge base-options request-options {:method method, :url url}) extra-args)))
-(defn- write-jsonl-to-stream! [^OutputStream os col-names reducible-rows]
- (let [none? (volatile! true)
- writer (-> os
- (OutputStreamWriter. StandardCharsets/UTF_8)
- (BufferedWriter.))]
-
- (run! (fn [row]
- (when @none? (vreset! none? false))
- (let [row-map (zipmap col-names row)]
- (json/encode-to row-map writer {})
- (.newLine writer)))
- reducible-rows)
-
- ;; Workaround for LocalStack, which doesn't support zero byte files.
- (when @none?
- (.write writer " "))
-
- (doto writer
- (.flush)
- (.close))))
-
-(defn- execute-mbql-query
- [driver db-id query respond cancel-chan]
- (driver/with-driver driver
- (let [native (qp.compile/compile {:type :query, :database db-id :query query})
- query {:database db-id
- :type :native
- :native native}]
- (qp.store/with-metadata-provider db-id
- (binding [qp.pipeline/*canceled-chan* cancel-chan]
- (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond))))))
-
(defn root-type
"Supported type for roundtrip/insertion"
[base-type]
@@ -93,6 +57,77 @@
:type/Text
:type/Boolean])))
+(defn- maybe-fixup-value [col v]
+ (cond
+ (nil? (root-type (:base_type col)))
+ ;; we're not a supported base type, so we just stringify it
+ (when v (json/encode v))
+
+ ;; the clickhouse driver returns bigdecimals for int64 values
+ (and (isa? (:base_type col) :type/Integer)
+ (or (instance? BigDecimal v)
+ (float? v)))
+ (bigint v)
+
+ :else
+ v))
+
+(defn- write-jsonl-row-to-os-rff
+ "Returns a rff that writes query results as JSONL to an OutputStream.
+
+ Only fields present in `fields-meta` are included in the output. Values are processed via
+ `maybe-fixup-value` before JSON encoding."
+ [^OutputStream os fields-meta {cols-meta :cols}]
+ (let [filtered-col-meta (m/index-by :name fields-meta)
+ col-names (map :name cols-meta)
+ none? (volatile! true)]
+ (fn
+ ([]
+ (-> os
+ (OutputStreamWriter. StandardCharsets/UTF_8)
+ (BufferedWriter.)))
+
+ ([^BufferedWriter writer]
+ ;; Workaround for LocalStack, which doesn't support zero byte files.
+ (when @none?
+ (.write writer " "))
+
+ (doto writer
+ (.flush)
+ (.close)))
+
+ ([^BufferedWriter writer row]
+ (let [row-map (->>
+ (map vector col-names row)
+ (filter (fn [[n _]]
+ (contains? filtered-col-meta n)))
+ (map (fn [[n v]]
+ (maybe-fixup-value (filtered-col-meta n) v)))
+ (zipmap (filter filtered-col-meta col-names)))]
+
+ (when @none? (vreset! none? false))
+ (json/encode-to row-map writer {})
+
+ (doto writer
+ (.newLine)))))))
+
+(defn- execute-mbql-query
+ [db-id query rff cancel-chan]
+ (binding [qp.pipeline/*canceled-chan* (a/go (a/<! cancel-chan))]
+ (qp/process-query {:type :query :database db-id :query query} rff)))
+
+(defn- throw-if-cancelled [cancel-chan]
+ (when (a/poll! cancel-chan)
+ (throw (ex-info "Run cancelled" {:error-type :cancelled}))))
+
+(defn- write-table-data-to-file! [{:keys [db-id table-id fields-meta temp-file cancel-chan limit]}]
+ (with-open [os (io/output-stream temp-file)]
+ (let [query (cond-> {:source-table table-id} limit (assoc :limit limit))
+ rff (fn [cols-meta] (write-jsonl-row-to-os-rff os fields-meta cols-meta))]
+ (execute-mbql-query db-id query rff cancel-chan)
+ (some-> cancel-chan throw-if-cancelled)
+ nil)))
+
(defn restricted-insert-type
"Type for insertion restricted to supported"
[base_type]
@@ -132,45 +167,6 @@
:field_id (:id col-meta)})
cols-meta)})
-(defn- maybe-fixup-value [col v]
- (cond
- (nil? (root-type (:base_type col)))
- ;; we're not a supported base type, so we just stringify it
- (when v (json/encode v))
-
- ;; the clickhouse driver returns bigdecimals for int64 values
- (and (isa? (:base_type col) :type/Integer)
- (or (instance? BigDecimal v)
- (float? v)))
- (bigint v)
-
- :else
- v))
-
-(defn- throw-if-cancelled [cancel-chan]
- (when (a/poll! cancel-chan)
- (throw (ex-info "Run cancelled" {:error-type :cancelled}))))
-
-(defn- write-table-data-to-file! [{:keys [db-id driver table-id fields-meta temp-file cancel-chan limit]}]
- {:pre [(or (nil? limit) (pos-int? limit))]}
- (let [query (cond-> {:source-table table-id} limit (assoc :limit limit))
- respond (fn [{cols-meta :cols} reducible-rows]
- (with-open [os (io/output-stream temp-file)]
- (let [filtered-col-meta (m/index-by :name fields-meta)
- col-names (map :name cols-meta)
- filtered-rows (eduction (map (fn [row]
- (->>
- (map vector col-names row)
- (filter (fn [[n _]]
- (contains? filtered-col-meta n)))
- (map (fn [[n v]]
- (maybe-fixup-value (filtered-col-meta n) v))))))
- reducible-rows)]
- (write-jsonl-to-stream! os (filter filtered-col-meta col-names) filtered-rows))))]
- (execute-mbql-query driver db-id query respond cancel-chan)
- (some-> cancel-chan throw-if-cancelled)
- nil))
-
(defn get-logs
"Return the logs of the current running python process"
[run-id]
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
index 3051ac4ca48..0dbbb9c9c3d 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
@@ -11,7 +11,7 @@
[metabase.util :as u]
[toucan2.core :as t2])
(:import
- (clojure.lang IDeref IReduceInit)
+ (clojure.lang IDeref)
(java.io Closeable)
(java.time Duration Instant)))
@@ -342,26 +342,25 @@
(when-not (deref wait-signal 5000 nil)
(throw (ex-info "Expected delivery of wait signal within a reasonable amount of time" {}))))
- (reducible-proxy [ready-signal
- wait-signal
- reducible-rows]
- (reify IReduceInit
- (reduce [_ f init]
- (deliver ready-signal true)
- (await-signal wait-signal)
- (reduce f init reducible-rows))))
+ (rf-proxy [ready-signal
+ wait-signal
+ rf]
+ (fn
+ ([] (rf))
+ ([w] (rf w))
+ ([w e]
+ (deliver ready-signal true)
+ (await-signal wait-signal)
+ (rf w e))))
(blocking-redefs [{:keys [block]} ready-signal wait-signal]
(case block
:read
- (let [f-ref #'transforms-python.python-runner/execute-mbql-query
+ (let [f-ref #'transforms-python.python-runner/write-jsonl-row-to-os-rff
f @f-ref]
{f-ref
- (fn [driver db-id query respond cancel-chan]
- (f driver db-id query
- (fn [metadata reducible-rows]
- (respond metadata (reducible-proxy ready-signal wait-signal reducible-rows)))
- cancel-chan))})
+ (fn [os fields-meta col-meta]
+ (rf-proxy ready-signal wait-signal (f os fields-meta col-meta)))})
:write
(let [f-ref #'transforms-python.execute/transfer-file-to-db
f @f-ref]