Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Do not realize objects in memory early for python transform results (#64034)

Nicola Momettoe1dcfdab2e74
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index 9d4c53ff12e..43a88f38df3 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -17,8 +17,10 @@
[metabase.util.log :as log]
[toucan2.core :as t2])
(:import
- (java.io Closeable File)
+ (java.io Closeable)
(java.net SocketException)
+ (java.nio.file Files)
+ (java.nio.file.attribute FileAttribute)
(java.time Duration)))
(set! *warn-on-reflection* true)
@@ -240,13 +242,14 @@
;; TODO restructure things such that s3 can we swapped out for other transfer mechanisms
(with-open [log-future-ref (open-python-message-update-future! run-id message-log)
shared-storage-ref (s3/open-shared-storage! (:source-tables source))]
- (let [driver (:engine db)
- server-url (transforms-python.settings/python-runner-url)
- _ (python-runner/copy-tables-to-s3! {:run-id run-id
- :shared-storage @shared-storage-ref
- :table-name->id (:source-tables source)
- :cancel-chan cancel-chan})
- _ (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+ (let [driver (:engine db)
+ server-url (transforms-python.settings/python-runner-url)
+ _ (python-runner/copy-tables-to-s3! {:run-id run-id
+ :shared-storage @shared-storage-ref
+ :table-name->id (:source-tables source)
+ :cancel-chan cancel-chan})
+ _ (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+
{:keys [status body] :as response}
(python-runner/execute-python-code-http-call!
{:server-url server-url
@@ -254,9 +257,9 @@
:run-id run-id
:table-name->id (:source-tables source)
:shared-storage @shared-storage-ref})
- ;; TODO temporary to keep more code stable while refactoring
- ;; no need to materialize these early (i.e output we can stream directly into a tmp file or db if small)
- {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+
+ output-manifest (python-runner/read-output-manifest @shared-storage-ref)
+ events (python-runner/read-events @shared-storage-ref)]
(.close log-future-ref) ; early close to force any writes to flush
(when (seq events)
(replace-python-logs! message-log events))
@@ -268,15 +271,16 @@
:body body
:events events}))
(try
- (let [temp-file (File/createTempFile "transform-output-" ".jsonl")]
+ (let [temp-path (Files/createTempFile "transform-output-" ".jsonl" (u/varargs FileAttribute))
+ temp-file (.toFile temp-path)]
(when-not (seq (:fields output-manifest))
(throw (ex-info "No fields in metadata"
{:metadata output-manifest
:raw-body body
:events events})))
(try
- (with-open [writer (io/writer temp-file)]
- (.write writer ^String output))
+ (with-open [in (python-runner/open-output @shared-storage-ref)]
+ (io/copy in temp-file))
(let [file-size (.length temp-file)]
(transforms.instrumentation/with-stage-timing [run-id :file-to-dwh]
(transfer-file-to-db driver db transform output-manifest temp-file))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
index 1adaf5599bd..da7230eb72e 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
@@ -2,7 +2,6 @@
(:require
[clj-http.client :as http]
[clojure.java.io :as io]
- [clojure.string :as str]
[medley.core :as m]
[metabase-enterprise.transforms-python.s3 :as s3]
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
@@ -20,7 +19,7 @@
[toucan2.core :as t2])
(:import
(clojure.lang PersistentQueue)
- (java.io BufferedWriter File OutputStream OutputStreamWriter)
+ (java.io BufferedWriter File InputStream OutputStream OutputStreamWriter)
(java.nio.charset StandardCharsets)))
(set! *warn-on-reflection* true)
@@ -204,17 +203,24 @@
{:error string-if-error}))
string-if-error)))))
-;; temporary, we should not need to realize data/events files into memory longer term
-(defn read-output-objects
- "Temporary function that strings/jsons stuff in S3 and returns it for compatibility."
+(defn read-events
+ "Returns a vector of event contents (or nil if the event file does not exist)"
[{:keys [s3-client bucket-name objects]}]
- (let [{:keys [output output-manifest events]} objects
- output-content (s3/read-to-string s3-client bucket-name (:path output) nil)
- output-manifest-content (s3/read-to-string s3-client bucket-name (:path output-manifest) "{}")
- events-content (s3/read-to-string s3-client bucket-name (:path events))]
- {:output output-content
- :output-manifest (json/decode+kw output-manifest-content)
- :events (mapv json/decode+kw (str/split-lines events-content))}))
+ ;; note we expect :events to be a small file, limits ought to be enforced by the runner
+ (when-some [in (s3/open-object s3-client bucket-name (:path (:events objects)))]
+ (with-open [in in
+ rdr (io/reader in)]
+ (mapv json/decode+kw (line-seq rdr)))))
+
+(defn read-output-manifest
+ "Return the output manifest map. Returns nil if it does not exist."
+ [{:keys [s3-client bucket-name objects]}]
+ (json/decode+kw (s3/read-to-string s3-client bucket-name (:path (:output-manifest objects)) "{}")))
+
+(defn open-output
+ "Return an InputStream with the output jsonl contents. Close with .close. Returns nil if the object does not exist."
+ ^InputStream [{:keys [s3-client bucket-name objects]}]
+ (s3/open-object s3-client bucket-name (:path (:output objects))))
(defn cancel-python-code-http-call!
"Calls the /cancel endpoint of the python runner. Returns immediately."
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj
index 75448593736..22cc6c01523 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj
@@ -4,7 +4,7 @@
[metabase.util.log :as log])
(:import
(clojure.lang IDeref)
- (java.io Closeable File)
+ (java.io Closeable File InputStream)
(java.net URI)
(java.time Duration)
(software.amazon.awssdk.auth.credentials AwsBasicCredentials StaticCredentialsProvider)
@@ -183,19 +183,22 @@
(let [^PutObjectRequest request (put-object-request bucket-name key)]
(.putObject s3-client request (RequestBody/fromFile file))))
-;; TODO optimize our ingestion to stream the data. reading a string will not work in any case for binary files.
+(defn open-object
+ "Get back the contents of the given key as a InputStream."
+ ^InputStream [^S3Client s3-client ^String bucket-name ^String key]
+ (try
+ (let [^GetObjectRequest request (get-object-request bucket-name key)]
+ (.getObject s3-client request))
+ (catch NoSuchKeyException _ nil)))
+
(defn read-to-string
"Get back the contents of the given key as a string."
- ([s3-client bucket-name key] (read-to-string s3-client bucket-name key ::throw))
+ ([s3-client bucket-name key] (read-to-string s3-client bucket-name key nil))
([^S3Client s3-client ^String bucket-name ^String key not-found]
- (try
- (let [^GetObjectRequest request (get-object-request bucket-name key)
- response (.getObject s3-client request)]
- (slurp response))
- (catch NoSuchKeyException e
- (if (identical? ::throw not-found)
- (throw e)
- not-found)))))
+ (if-some [in (open-object s3-client bucket-name key)]
+ (with-open [ret in]
+ (slurp ret))
+ not-found)))
(defn delete
;; TODO better error handling
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
index 0a155546169..6e7b50ae061 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
@@ -29,21 +29,22 @@
"Execute a Python transform with the given code and tables"
[{:keys [code tables]}]
(with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]
- (let [server-url (transforms-python.settings/python-runner-url)
- cancel-chan (a/promise-chan)
- table-name->id (or tables {})
- _ (python-runner/copy-tables-to-s3! {:run-id test-id
- :shared-storage @shared-storage-ref
- :table-name->id table-name->id
- :cancel-chan cancel-chan})
- response (python-runner/execute-python-code-http-call! {:server-url server-url
- :code code
- :run-id test-id
- :table-name->id table-name->id
- :shared-storage @shared-storage-ref})
- {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+ (let [server-url (transforms-python.settings/python-runner-url)
+ cancel-chan (a/promise-chan)
+ table-name->id (or tables {})
+ _ (python-runner/copy-tables-to-s3! {:run-id test-id
+ :shared-storage @shared-storage-ref
+ :table-name->id table-name->id
+ :cancel-chan cancel-chan})
+ response (python-runner/execute-python-code-http-call! {:server-url server-url
+ :code code
+ :run-id test-id
+ :table-name->id table-name->id
+ :shared-storage @shared-storage-ref})
+ events (python-runner/read-events @shared-storage-ref)
+ output-manifest (python-runner/read-output-manifest @shared-storage-ref)]
(merge (:body response)
- {:output output
+ {:output (when-some [in (python-runner/open-output @shared-storage-ref)] (with-open [in in] (slurp in)))
:output-manifest output-manifest
:stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
:stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
index 52c8e04a810..705f758812f 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
@@ -69,10 +69,11 @@
:run-id test-id
:table-name->id table-name->id
:shared-storage @shared-storage-ref})
- {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+ events (python-runner/read-events @shared-storage-ref)
+ output-manifest (python-runner/read-output-manifest @shared-storage-ref)]
;; not sure about munging this all together but its what tests expect for now
(merge (:body response)
- {:output output
+ {:output (when-some [in (python-runner/open-output @shared-storage-ref)] (with-open [in in] (slurp in)))
:output-manifest output-manifest
:stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
:stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj
index eb3f7539fe6..fc8ca56591f 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj
@@ -15,8 +15,7 @@
(java.net ConnectException)
(java.nio.file Files)
(java.nio.file.attribute FileAttribute)
- (software.amazon.awssdk.services.s3 S3Client)
- (software.amazon.awssdk.services.s3.model NoSuchKeyException)))
+ (software.amazon.awssdk.services.s3 S3Client)))
(set! *warn-on-reflection* true)
@@ -43,7 +42,7 @@
key (str "test-object-" (random-uuid) ".txt")
body (str "Hello, S3! My secret is:" (random-uuid))]
- (is (thrown? NoSuchKeyException (s3/read-to-string s3-client bucket key)))
+ (is (nil? (s3/read-to-string s3-client bucket key)))
(is (= :default (s3/read-to-string s3-client bucket key :default)))
(let [tmp-file (Files/createTempFile "s3-test" ".txt" (into-array FileAttribute []))]
@@ -57,7 +56,7 @@
(s3/delete s3-client bucket key)
- (is (thrown? NoSuchKeyException (s3/read-to-string s3-client bucket key)))
+ (is (nil? (s3/read-to-string s3-client bucket key)))
(is (= :default (s3/read-to-string s3-client bucket key :default)))
(finally
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.clj
index 89b1373c33f..263572020b4 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj
@@ -695,8 +695,8 @@
documents (map #(into {} (map vector col-names %))
data)]
(if (> (bounded-count 2 documents) 1)
- ;; TODO: batch to avoid realizing everything in memory
- (mongo.util/insert-many collection documents)
+ (doseq [chunk (partition-all (or driver/*insert-chunk-rows* 1000) documents)]
+ (mongo.util/insert-many collection chunk))
(mongo.util/insert-one collection (first documents)))))))
;; Following code is using monger. Leaving it here for a reference as it could be transformed when there is need