Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Enforce the 50 MB CSV upload limit on the backend (#77363)

Chris Truterc6a0c72b016c
diff --git a/src/metabase/api/macros.clj b/src/metabase/api/macros.clj
index 7fbd2188051..6048a67e64e 100644
--- a/src/metabase/api/macros.clj
+++ b/src/metabase/api/macros.clj
@@ -15,6 +15,7 @@
On that note please don't add any unrelated macros to this namespace! Do cleanup first."
(:require
[clojure.core.specs.alpha]
+ [clojure.java.io :as io]
[clojure.spec.alpha :as s]
[clojure.string :as str]
[clout.core :as clout]
@@ -347,8 +348,19 @@
params-type :- ::param-type]
(get-in args [:params params-type :binding] '_))
+(defn- redact-files
+ "Replace [[java.io.File]] values (multipart tempfiles) so validation errors don't leak server temp paths."
+ [x]
+ (cond
+ (instance? java.io.File x) "<redacted File>"
+ (map? x) (update-vals x redact-files)
+ (sequential? x) (mapv redact-files x)
+ :else x))
+
(defn- invalid-params-specific-errors [explanation]
(-> explanation
+ (update :value redact-files)
+ (update :errors (partial mapv #(update % :value redact-files)))
me/with-spell-checking
(me/humanize {:wrap mu/humanize-include-value})))
@@ -382,9 +394,10 @@
decoded ((decoder schema) params)]
(when-not (mr/validate schema decoded)
(throw (ex-info (format "Invalid %s" (case params-type
- :route "route parameters"
- :query "query parameters"
- :body "body"))
+ :route "route parameters"
+ :query "query parameters"
+ :body "body"
+ :request "request"))
(let [explanation (mr/explain schema decoded)
specific-errors (invalid-params-specific-errors explanation)
errors (invalid-params-errors explanation)]
@@ -583,6 +596,38 @@
(when-not (instance? org.eclipse.jetty.ee9.nested.HttpInput body)
body))))
+(defn- delete-multipart-tempfiles!
+ [request]
+ (doseq [v (vals (:multipart-params request))
+ v (if (sequential? v) v [v])
+ :when (map? v)
+ :let [f (:tempfile v)]
+ :when f]
+ (io/delete-file f :silently)))
+
+(defn wrap-multipart-tempfile-cleanup
+ "Ring middleware that deletes the parsed multipart tempfiles when the wrapped handler throws,
+ e.g. when param validation rejects the request.
+ Applied inside [[ring.middleware.multipart-params/wrap-multipart-params]].
+ Ring's temp-file store only sweeps orphaned tempfiles after an hour.
+ Handlers that complete normally own the tempfiles they consume and must delete them themselves."
+ [handler]
+ (fn
+ ([request]
+ (try
+ (handler request)
+ (catch Throwable e
+ (delete-multipart-tempfiles! request)
+ (throw e))))
+ ([request respond raise]
+ (try
+ (handler request respond (fn [e]
+ (delete-multipart-tempfiles! request)
+ (raise e)))
+ (catch Throwable e
+ (delete-multipart-tempfiles! request)
+ (throw e))))))
+
(mu/defn- middleware-forms
"Middleware to apply to base handler. Supports:
@@ -594,17 +639,20 @@
Endpoints without `:scope` get [[metabase.api.macros.scope/ensure-scopes-checked]] to prevent scoped
tokens from reaching endpoints that haven't opted in."
[{:keys [metadata], :as _args} :- ::parsed-args]
- (let [scope (:scope metadata)
- scope-middleware (cond
- (= scope :unchecked) []
- (some? scope) [(list 'metabase.api.macros.scope/enforce-scope scope)]
- :else ['metabase.api.macros.scope/ensure-scopes-checked])
- multipart (:multipart metadata)]
- (cond-> (vec scope-middleware)
- multipart
- (conj (if (map? multipart)
- `(fn [handler#] (ring.middleware.multipart-params/wrap-multipart-params handler# ~multipart))
- 'ring.middleware.multipart-params/wrap-multipart-params)))))
+ (let [scope (:scope metadata)
+ scope-middleware (cond
+ (= scope :unchecked) []
+ (some? scope) [(list 'metabase.api.macros.scope/enforce-scope scope)]
+ :else ['metabase.api.macros.scope/ensure-scopes-checked])
+ multipart (:multipart metadata)
+ multipart-middleware (when multipart
+ [`(fn [handler#]
+ (ring.middleware.multipart-params/wrap-multipart-params
+ (wrap-multipart-tempfile-cleanup handler#)
+ ~(if (map? multipart) multipart {})))])]
+ ;; [[apply-middleware]] wraps left-to-right, so the last entry is outermost and sees the request first.
+ ;; Scope goes last: a scope rejection must respond 403 before multipart parses the body to tempfiles.
+ (vec (concat multipart-middleware scope-middleware))))
(mu/defn- apply-middleware :- ::handler
[handler :- ::handler
diff --git a/src/metabase/upload/api.clj b/src/metabase/upload/api.clj
index a6fe7403e47..bf088ee34b1 100644
--- a/src/metabase/upload/api.clj
+++ b/src/metabase/upload/api.clj
@@ -35,8 +35,12 @@
;;
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :post "/csv"
- "Create a table and model populated with the values from the attached CSV. Returns the model ID if successful."
- {:multipart true}
+ "Create a table and model populated with the values from the attached CSV.
+ Returns the model ID if successful.
+
+ The file may be at most 50 MB; larger uploads are rejected with a 413 response."
+ {:multipart {:max-file-size upload/max-upload-size-bytes
+ :max-file-count upload/max-upload-part-count}}
;; TODO -- not clear collection_id and file are supposed to come from `:multipart-params`
[_route-params
_query-params
@@ -44,7 +48,7 @@
{{collection-id "collection_id", file "file"} :multipart-params, :as _request}
:- [:map
[:multipart-params
- [:map
+ [:map {:closed true}
["collection_id" [:maybe
{:decode/api (fn [collection-id]
(when-not (= collection-id "root")
diff --git a/src/metabase/upload/core.clj b/src/metabase/upload/core.clj
index 586c441ba40..dd1565adc33 100644
--- a/src/metabase/upload/core.clj
+++ b/src/metabase/upload/core.clj
@@ -15,6 +15,8 @@
can-create-upload?
create-csv-upload!
delete-upload!
+ max-upload-part-count
+ max-upload-size-bytes
model-hydrate-based-on-upload
update-action-schema
update-csv!])
diff --git a/src/metabase/upload/impl.clj b/src/metabase/upload/impl.clj
index 7b0a7b2a100..e0208be8b1c 100644
--- a/src/metabase/upload/impl.clj
+++ b/src/metabase/upload/impl.clj
@@ -43,6 +43,18 @@
(set! *warn-on-reflection* true)
+(def max-upload-size-bytes
+ "Maximum size in bytes of a file that can be uploaded to create or update an upload table.
+ Keep in sync with `MAX_UPLOAD_SIZE` in `frontend/src/metabase/redux/uploads.ts`.
+ The limit documented in `docs/exploration-and-organization/uploads.md` must match as well."
+ (* 50 1024 1024))
+
+(def max-upload-part-count
+ "Maximum number of multipart parts accepted by the CSV upload endpoints.
+ Ring's :max-file-count option counts every part, form fields included, so this must allow for the
+ collection_id field the frontend sends alongside the file part."
+ 2)
+
;; TODO: move these to a more appropriate namespace if they need to be reused
(defmulti max-bytes
"This tracks the size of various text fields in bytes."
diff --git a/src/metabase/warehouse_schema_rest/api/table.clj b/src/metabase/warehouse_schema_rest/api/table.clj
index 597d5346a2d..f95a5e7db9b 100644
--- a/src/metabase/warehouse_schema_rest/api/table.clj
+++ b/src/metabase/warehouse_schema_rest/api/table.clj
@@ -462,19 +462,25 @@
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :post "/:id/append-csv"
"Inserts the rows of an uploaded CSV file into the table identified by `:id`. The table must have been created by
- uploading a CSV file."
- {:multipart true}
+ uploading a CSV file.
+
+ The file may be at most 50 MB; larger uploads are rejected with a 413 response."
+ {:multipart {:max-file-size upload/max-upload-size-bytes
+ :max-file-count upload/max-upload-part-count}}
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]
_query-params
_body
+ ;; Closed so a file part smuggled under another part name is rejected; collection_id is the text field the
+ ;; frontend sends alongside the file.
{:keys [multipart-params], :as _request} :- [:map
[:multipart-params
- [:map
+ [:map {:closed true}
["file"
[:map
[:filename :string]
- [:tempfile (ms/InstanceOfClass java.io.File)]]]]]]]
+ [:tempfile (ms/InstanceOfClass java.io.File)]]]
+ ["collection_id" {:optional true} :string]]]]]
(update-csv! {:table-id id
:filename (get-in multipart-params ["file" :filename])
:file (get-in multipart-params ["file" :tempfile])
@@ -486,19 +492,25 @@
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :post "/:id/replace-csv"
"Replaces the contents of the table identified by `:id` with the rows of an uploaded CSV file. The table must have
- been created by uploading a CSV file."
- {:multipart true}
+ been created by uploading a CSV file.
+
+ The file may be at most 50 MB; larger uploads are rejected with a 413 response."
+ {:multipart {:max-file-size upload/max-upload-size-bytes
+ :max-file-count upload/max-upload-part-count}}
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]
_query-params
_body
+ ;; Closed so a file part smuggled under another part name is rejected; collection_id is the text field the
+ ;; frontend sends alongside the file.
{:keys [multipart-params], :as _request} :- [:map
[:multipart-params
- [:map
+ [:map {:closed true}
["file"
[:map
[:filename :string]
- [:tempfile (ms/InstanceOfClass java.io.File)]]]]]]]
+ [:tempfile (ms/InstanceOfClass java.io.File)]]]
+ ["collection_id" {:optional true} :string]]]]]
(update-csv! {:table-id id
:filename (get-in multipart-params ["file" :filename])
:file (get-in multipart-params ["file" :tempfile])
diff --git a/test/metabase/api/macros_test.clj b/test/metabase/api/macros_test.clj
index 08788871e92..bb293b81a10 100644
--- a/test/metabase/api/macros_test.clj
+++ b/test/metabase/api/macros_test.clj
@@ -4,6 +4,8 @@
[metabase.api.macros :as api.macros]
[metabase.util.malli.registry :as mr]))
+(set! *warn-on-reflection* true)
+
(deftest ^:parallel parse-args-test
(are [args expected] (= expected
(#'api.macros/parse-args args))
@@ -96,6 +98,45 @@
(neat))
{:route {:path "/move/:id", :regexes {:id #"[abc]{4}"}}}))
+(deftest ^:parallel multipart-tempfile-cleanup-on-throw-test
+ (testing "tempfiles are deleted when the handler throws, e.g. on a param validation 400"
+ (let [file-1 (java.io.File/createTempFile "cleanup-test" nil)
+ file-2 (java.io.File/createTempFile "cleanup-test" nil)
+ request {:multipart-params {"file" {:filename "a.csv", :tempfile file-1}
+ ;; duplicate part names arrive as a vector of values
+ "sneaky" [{:filename "b.csv", :tempfile file-2}]
+ "some_id" "123"}}
+ handler (api.macros/wrap-multipart-tempfile-cleanup
+ (fn [_request]
+ (throw (ex-info "Invalid request" {:status-code 400}))))]
+ (is (thrown-with-msg? Exception #"Invalid request" (handler request)))
+ (is (not (.exists file-1)))
+ (is (not (.exists file-2))))))
+
+(deftest ^:parallel multipart-tempfile-cleanup-on-raise-test
+ (testing "async arity: tempfiles are deleted when the handler raises"
+ (let [file (java.io.File/createTempFile "cleanup-test" nil)
+ request {:multipart-params {"file" {:filename "a.csv", :tempfile file}}}
+ raised (atom nil)
+ handler (api.macros/wrap-multipart-tempfile-cleanup
+ (fn [_request _respond raise]
+ (raise (ex-info "Invalid request" {:status-code 400}))))]
+ (handler request
+ (fn [_response] (is false "respond should not be called"))
+ (fn [e] (reset! raised e)))
+ (is (some? @raised))
+ (is (not (.exists file))))))
+
+(deftest ^:parallel multipart-tempfile-preserved-on-success-test
+ (testing "tempfiles are left alone when the handler completes (it owns and deletes what it consumes)"
+ (let [file (java.io.File/createTempFile "cleanup-test" nil)
+ request {:multipart-params {"file" {:filename "a.csv", :tempfile file}}}
+ handler (api.macros/wrap-multipart-tempfile-cleanup (fn [_request] :ok))]
+ (try
+ (is (= :ok (handler request)))
+ (is (.exists file))
+ (finally (.delete file))))))
+
(deftest ^:parallel parse-args-metadata-test
(testing "metadata map is parsed correctly"
(are [args expected] (= expected
diff --git a/test/metabase/upload/api_test.clj b/test/metabase/upload/api_test.clj
index 8380e71ccbd..f82984cf163 100644
--- a/test/metabase/upload/api_test.clj
+++ b/test/metabase/upload/api_test.clj
@@ -1,8 +1,11 @@
(ns metabase.upload.api-test
(:require
+ [clojure.java.io :as io]
+ [clojure.string :as str]
[clojure.test :refer :all]
[metabase.test :as mt]
[metabase.upload.api :as upload.api]
+ [metabase.upload.core :as upload]
[metabase.upload.impl-test :as upload-test]
[toucan2.core :as t2]))
@@ -33,3 +36,67 @@
status))
(is (= body
(t2/select-one-pk :model/Card :database_id (mt/id)))))))))
+
+(def ^:private multipart-request-options
+ {:request-options {:headers {"content-type" "multipart/form-data"}}})
+
+(deftest csv-upload-too-large-test
+ (testing "POST /api/upload/csv rejects a file over the size cap with a 413"
+ ;; One byte over the cap is enough: the multipart middleware aborts streaming the file part as soon as it
+ ;; crosses the limit, so the oversized body is never fully buffered.
+ (let [oversized (byte-array (inc upload/max-upload-size-bytes))
+ calls (atom 0)]
+ (mt/with-dynamic-fn-redefs [upload.api/from-csv! (fn [& _] (swap! calls inc) {:status 200})]
+ (is (= "Uploaded content exceeded limits."
+ (mt/user-http-request :crowberto :post 413 "upload/csv" multipart-request-options
+ {:file oversized})))
+ (is (zero? @calls) "the endpoint body should never run")))))
+
+(deftest csv-upload-at-size-cap-test
+ (testing "POST /api/upload/csv accepts a file of exactly the size cap"
+ (let [at-cap (byte-array upload/max-upload-size-bytes)
+ calls (atom 0)]
+ ;; like the real handler, the stub owns the uploaded tempfile and must delete it
+ (mt/with-dynamic-fn-redefs [upload.api/from-csv! (fn [{:keys [file]}]
+ (swap! calls inc)
+ (io/delete-file file :silently)
+ {:status 200, :body 1})]
+ (mt/user-http-request :crowberto :post 200 "upload/csv" multipart-request-options
+ [[:collection_id "root"]
+ [:file at-cap]])
+ (is (= 1 @calls) "the endpoint body should run")))))
+
+(deftest csv-upload-too-many-parts-test
+ (testing "POST /api/upload/csv rejects too many multipart parts with a 413"
+ (let [calls (atom 0)]
+ (mt/with-dynamic-fn-redefs [upload.api/from-csv! (fn [& _] (swap! calls inc) {:status 200})]
+ (is (= "Uploaded content exceeded limits."
+ (mt/user-http-request :crowberto :post 413 "upload/csv" multipart-request-options
+ [[:collection_id "root"]
+ [:file (byte-array [97 44 98 10 49 44 50])]
+ [:file (byte-array [99 44 100 10 51 44 52])]])))
+ (is (zero? @calls) "the endpoint body should never run")))))
+
+(deftest csv-upload-with-collection-id-test
+ (testing "POST /api/upload/csv accepts the collection_id field the frontend sends alongside the file"
+ (let [calls (atom 0)]
+ ;; like the real handler, the stub owns the uploaded tempfile and must delete it
+ (mt/with-dynamic-fn-redefs [upload.api/from-csv! (fn [{:keys [file]}]
+ (swap! calls inc)
+ (io/delete-file file :silently)
+ {:status 200, :body 1})]
+ (mt/user-http-request :crowberto :post 200 "upload/csv" multipart-request-options
+ [[:collection_id "root"]
+ [:file (byte-array [97 44 98 10 49 44 50])]])
+ (is (= 1 @calls) "the endpoint body should run")))))
+
+(deftest csv-upload-smuggled-file-part-test
+ (testing "POST /api/upload/csv rejects a second file part smuggled as collection_id"
+ (let [calls (atom 0)]
+ (mt/with-dynamic-fn-redefs [upload.api/from-csv! (fn [& _] (swap! calls inc) {:status 200})]
+ (let [response (mt/user-http-request :crowberto :post 400 "upload/csv" multipart-request-options
+ [[:file (byte-array [97 44 98 10 49 44 50])]
+ [:collection_id (byte-array [99 44 100 10 51 44 52])]])]
+ (testing "the validation error does not leak the tempfile path"
+ (is (not (str/includes? (pr-str response) "ring-multipart-")))))
+ (is (zero? @calls) "the endpoint body should never run")))))
diff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.clj
index 209ac8fdc14..fa727118b2e 100644
--- a/test/metabase/warehouse_schema_rest/api/table_test.clj
+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj
@@ -2,6 +2,7 @@
"Tests for /api/table endpoints."
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.warehouse-schema-rest.api.table-test]}}}}}}
(:require
+ [clojure.java.io :as io]
[clojure.java.jdbc :as jdbc]
[clojure.set :as set]
[clojure.test :refer :all]
@@ -18,6 +19,7 @@
[metabase.sync.core :as sync]
[metabase.test :as mt]
[metabase.test.http-client :as client]
+ [metabase.upload.core :as upload]
[metabase.upload.impl-test :as upload-test]
[metabase.util :as u]
[metabase.warehouse-schema-rest.api.table :as api.table]
@@ -1195,6 +1197,65 @@
:file file}))
(is (not (.exists file)) "File should be deleted after replace-csv!")))))))
+(def ^:private multipart-request-options
+ {:request-options {:headers {"content-type" "multipart/form-data"}}})
+
+(deftest update-csv-too-large-test
+ ;; One byte over the cap is enough: the multipart middleware aborts streaming the file part as soon as it
+ ;; crosses the limit, so the oversized body is never fully buffered.
+ (let [oversized (byte-array (inc upload/max-upload-size-bytes))
+ calls (atom 0)]
+ (mt/with-dynamic-fn-redefs [api.table/update-csv! (fn [& _] (swap! calls inc) nil)]
+ (doseq [endpoint ["append-csv" "replace-csv"]]
+ (testing (format "POST /api/table/:id/%s rejects a file over the size cap with a 413" endpoint)
+ (is (= "Uploaded content exceeded limits."
+ (mt/user-http-request :crowberto :post 413 (format "table/%d/%s" (mt/id :venues) endpoint)
+ multipart-request-options
+ {:file oversized})))))
+ (is (zero? @calls) "the endpoint bodies should never run"))))
+
+(deftest update-csv-too-many-parts-test
+ (let [calls (atom 0)]
+ (mt/with-dynamic-fn-redefs [api.table/update-csv! (fn [& _] (swap! calls inc) nil)]
+ (doseq [endpoint ["append-csv" "replace-csv"]]
+ (testing (format "POST /api/table/:id/%s rejects too many multipart parts with a 413" endpoint)
+ (is (= "Uploaded content exceeded limits."
+ (mt/user-http-request :crowberto :post 413 (format "table/%d/%s" (mt/id :venues) endpoint)
+ multipart-request-options
+ [[:collection_id "root"]
+ [:file (byte-array [97 44 98 10 49 44 50])]
+ [:file (byte-array [99 44 100 10 51 44 52])]])))))
+ (is (zero? @calls) "the endpoint bodies should never run"))))
+
+(deftest update-csv-with-extra-field-test
+ (let [calls (atom 0)]
+ ;; like the real handler, the stub owns the uploaded tempfile and must delete it
+ (mt/with-dynamic-fn-redefs [api.table/update-csv! (fn [{:keys [file]}]
+ (swap! calls inc)
+ (io/delete-file file :silently)
+ {:status 200, :body ""})]
+ (doseq [endpoint ["append-csv" "replace-csv"]]
+ (testing (format "POST /api/table/:id/%s accepts the collection_id field the frontend sends" endpoint)
+ (mt/user-http-request :crowberto :post 200 (format "table/%d/%s" (mt/id :venues) endpoint)
+ multipart-request-options
+ [[:collection_id "root"]
+ [:file (byte-array [97 44 98 10 49 44 50])]])))
+ (is (= 2 @calls) "the endpoint bodies should run"))))
+
+(deftest update-csv-smuggled-file-part-test
+ (let [calls (atom 0)]
+ (mt/with-dynamic-fn-redefs [api.table/update-csv! (fn [& _] (swap! calls inc) nil)]
+ (doseq [endpoint ["append-csv" "replace-csv"]
+ ;; collection_id is caught by its :string constraint, unknown names by the closed map
+ part-name [:collection_id :unexpected_part]]
+ (testing (format "POST /api/table/:id/%s rejects a second file part smuggled as %s"
+ endpoint (name part-name))
+ (mt/user-http-request :crowberto :post 400 (format "table/%d/%s" (mt/id :venues) endpoint)
+ multipart-request-options
+ [[:file (byte-array [97 44 98 10 49 44 50])]
+ [part-name (byte-array [99 44 100 10 51 44 52])]])))
+ (is (zero? @calls) "the endpoint bodies should never run"))))
+
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | POST /api/table/:id/sync_schema |
;;; +----------------------------------------------------------------------------------------------------------------+