The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Enforce the 50 MB CSV upload limit on the backend (#77363)
Chris Truterc6a0c72b016c
diff --git a/src/metabase/api/macros.clj b/src/metabase/api/macros.cljindex 7fbd2188051..6048a67e64e 100644--- a/src/metabase/api/macros.clj+++ b/src/metabase/api/macros.clj@@ -15,6 +15,7 @@On that note please don't add any unrelated macros to this namespace! Do cleanup first."(:require[clojure.core.specs.alpha]+ [clojure.java.io :as io][clojure.spec.alpha :as s][clojure.string :as str][clout.core :as clout]@@ -347,8 +348,19 @@params-type :- ::param-type](get-in args [:params params-type :binding] '_))+(defn- redact-files+ "Replace [[java.io.File]] values (multipart tempfiles) so validation errors don't leak server temp paths."+ [x]+ (cond+ (instance? java.io.File x) "<redacted File>"+ (map? x) (update-vals x redact-files)+ (sequential? x) (mapv redact-files x)+ :else x))+(defn- invalid-params-specific-errors [explanation](-> explanation+ (update :value redact-files)+ (update :errors (partial mapv #(update % :value redact-files)))me/with-spell-checking(me/humanize {:wrap mu/humanize-include-value})))@@ -382,9 +394,10 @@decoded ((decoder schema) params)](when-not (mr/validate schema decoded)(throw (ex-info (format "Invalid %s" (case params-type- :route "route parameters"- :query "query parameters"- :body "body"))+ :route "route parameters"+ :query "query parameters"+ :body "body"+ :request "request"))(let [explanation (mr/explain schema decoded)specific-errors (invalid-params-specific-errors explanation)errors (invalid-params-errors explanation)]@@ -583,6 +596,38 @@(when-not (instance? org.eclipse.jetty.ee9.nested.HttpInput body)body))))+(defn- delete-multipart-tempfiles!+ [request]+ (doseq [v (vals (:multipart-params request))+ v (if (sequential? v) v [v])+ :when (map? v)+ :let [f (:tempfile v)]+ :when f]+ (io/delete-file f :silently)))++(defn wrap-multipart-tempfile-cleanup+ "Ring middleware that deletes the parsed multipart tempfiles when the wrapped handler throws,+ e.g. when param validation rejects the request.+ Applied inside [[ring.middleware.multipart-params/wrap-multipart-params]].+ Ring's temp-file store only sweeps orphaned tempfiles after an hour.+ Handlers that complete normally own the tempfiles they consume and must delete them themselves."+ [handler]+ (fn+ ([request]+ (try+ (handler request)+ (catch Throwable e+ (delete-multipart-tempfiles! request)+ (throw e))))+ ([request respond raise]+ (try+ (handler request respond (fn [e]+ (delete-multipart-tempfiles! request)+ (raise e)))+ (catch Throwable e+ (delete-multipart-tempfiles! request)+ (throw e))))))+(mu/defn- middleware-forms"Middleware to apply to base handler. Supports:@@ -594,17 +639,20 @@Endpoints without `:scope` get [[metabase.api.macros.scope/ensure-scopes-checked]] to prevent scopedtokens from reaching endpoints that haven't opted in."[{:keys [metadata], :as _args} :- ::parsed-args]- (let [scope (:scope metadata)- scope-middleware (cond- (= scope :unchecked) []- (some? scope) [(list 'metabase.api.macros.scope/enforce-scope scope)]- :else ['metabase.api.macros.scope/ensure-scopes-checked])- multipart (:multipart metadata)]- (cond-> (vec scope-middleware)- multipart- (conj (if (map? multipart)- `(fn [handler#] (ring.middleware.multipart-params/wrap-multipart-params handler# ~multipart))- 'ring.middleware.multipart-params/wrap-multipart-params)))))+ (let [scope (:scope metadata)+ scope-middleware (cond+ (= scope :unchecked) []+ (some? scope) [(list 'metabase.api.macros.scope/enforce-scope scope)]+ :else ['metabase.api.macros.scope/ensure-scopes-checked])+ multipart (:multipart metadata)+ multipart-middleware (when multipart+ [`(fn [handler#]+ (ring.middleware.multipart-params/wrap-multipart-params+ (wrap-multipart-tempfile-cleanup handler#)+ ~(if (map? multipart) multipart {})))])]+ ;; [[apply-middleware]] wraps left-to-right, so the last entry is outermost and sees the request first.+ ;; Scope goes last: a scope rejection must respond 403 before multipart parses the body to tempfiles.+ (vec (concat multipart-middleware scope-middleware))))(mu/defn- apply-middleware :- ::handler[handler :- ::handlerdiff --git a/src/metabase/upload/api.clj b/src/metabase/upload/api.cljindex a6fe7403e47..bf088ee34b1 100644--- a/src/metabase/upload/api.clj+++ b/src/metabase/upload/api.clj@@ -35,8 +35,12 @@;;#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}(api.macros/defendpoint :post "/csv"- "Create a table and model populated with the values from the attached CSV. Returns the model ID if successful."- {:multipart true}+ "Create a table and model populated with the values from the attached CSV.+ Returns the model ID if successful.++ The file may be at most 50 MB; larger uploads are rejected with a 413 response."+ {:multipart {:max-file-size upload/max-upload-size-bytes+ :max-file-count upload/max-upload-part-count}};; TODO -- not clear collection_id and file are supposed to come from `:multipart-params`[_route-params_query-params@@ -44,7 +48,7 @@{{collection-id "collection_id", file "file"} :multipart-params, :as _request}:- [:map[:multipart-params- [:map+ [:map {:closed true}["collection_id" [:maybe{:decode/api (fn [collection-id](when-not (= collection-id "root")diff --git a/src/metabase/upload/core.clj b/src/metabase/upload/core.cljindex 586c441ba40..dd1565adc33 100644--- a/src/metabase/upload/core.clj+++ b/src/metabase/upload/core.clj@@ -15,6 +15,8 @@can-create-upload?create-csv-upload!delete-upload!+ max-upload-part-count+ max-upload-size-bytesmodel-hydrate-based-on-uploadupdate-action-schemaupdate-csv!])diff --git a/src/metabase/upload/impl.clj b/src/metabase/upload/impl.cljindex 7b0a7b2a100..e0208be8b1c 100644--- a/src/metabase/upload/impl.clj+++ b/src/metabase/upload/impl.clj@@ -43,6 +43,18 @@(set! *warn-on-reflection* true)+(def max-upload-size-bytes+ "Maximum size in bytes of a file that can be uploaded to create or update an upload table.+ Keep in sync with `MAX_UPLOAD_SIZE` in `frontend/src/metabase/redux/uploads.ts`.+ The limit documented in `docs/exploration-and-organization/uploads.md` must match as well."+ (* 50 1024 1024))++(def max-upload-part-count+ "Maximum number of multipart parts accepted by the CSV upload endpoints.+ Ring's :max-file-count option counts every part, form fields included, so this must allow for the+ collection_id field the frontend sends alongside the file part."+ 2)+;; TODO: move these to a more appropriate namespace if they need to be reused(defmulti max-bytes"This tracks the size of various text fields in bytes."diff --git a/src/metabase/warehouse_schema_rest/api/table.clj b/src/metabase/warehouse_schema_rest/api/table.cljindex 597d5346a2d..f95a5e7db9b 100644--- a/src/metabase/warehouse_schema_rest/api/table.clj+++ b/src/metabase/warehouse_schema_rest/api/table.clj@@ -462,19 +462,25 @@#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}(api.macros/defendpoint :post "/:id/append-csv""Inserts the rows of an uploaded CSV file into the table identified by `:id`. The table must have been created by- uploading a CSV file."- {:multipart true}+ uploading a CSV file.++ The file may be at most 50 MB; larger uploads are rejected with a 413 response."+ {:multipart {:max-file-size upload/max-upload-size-bytes+ :max-file-count upload/max-upload-part-count}}[{:keys [id]} :- [:map[:id ms/PositiveInt]]_query-params_body+ ;; Closed so a file part smuggled under another part name is rejected; collection_id is the text field the+ ;; frontend sends alongside the file.{:keys [multipart-params], :as _request} :- [:map[:multipart-params- [:map+ [:map {:closed true}["file"[:map[:filename :string]- [:tempfile (ms/InstanceOfClass java.io.File)]]]]]]]+ [:tempfile (ms/InstanceOfClass java.io.File)]]]+ ["collection_id" {:optional true} :string]]]]](update-csv! {:table-id id:filename (get-in multipart-params ["file" :filename]):file (get-in multipart-params ["file" :tempfile])@@ -486,19 +492,25 @@#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}(api.macros/defendpoint :post "/:id/replace-csv""Replaces the contents of the table identified by `:id` with the rows of an uploaded CSV file. The table must have- been created by uploading a CSV file."- {:multipart true}+ been created by uploading a CSV file.++ The file may be at most 50 MB; larger uploads are rejected with a 413 response."+ {:multipart {:max-file-size upload/max-upload-size-bytes+ :max-file-count upload/max-upload-part-count}}[{:keys [id]} :- [:map[:id ms/PositiveInt]]_query-params_body+ ;; Closed so a file part smuggled under another part name is rejected; collection_id is the text field the+ ;; frontend sends alongside the file.{:keys [multipart-params], :as _request} :- [:map[:multipart-params- [:map+ [:map {:closed true}["file"[:map[:filename :string]- [:tempfile (ms/InstanceOfClass java.io.File)]]]]]]]+ [:tempfile (ms/InstanceOfClass java.io.File)]]]+ ["collection_id" {:optional true} :string]]]]](update-csv! {:table-id id:filename (get-in multipart-params ["file" :filename]):file (get-in multipart-params ["file" :tempfile])diff --git a/test/metabase/api/macros_test.clj b/test/metabase/api/macros_test.cljindex 08788871e92..bb293b81a10 100644--- a/test/metabase/api/macros_test.clj+++ b/test/metabase/api/macros_test.clj@@ -4,6 +4,8 @@[metabase.api.macros :as api.macros][metabase.util.malli.registry :as mr]))+(set! *warn-on-reflection* true)+(deftest ^:parallel parse-args-test(are [args expected] (= expected(#'api.macros/parse-args args))@@ -96,6 +98,45 @@(neat)){:route {:path "/move/:id", :regexes {:id #"[abc]{4}"}}}))+(deftest ^:parallel multipart-tempfile-cleanup-on-throw-test+ (testing "tempfiles are deleted when the handler throws, e.g. on a param validation 400"+ (let [file-1 (java.io.File/createTempFile "cleanup-test" nil)+ file-2 (java.io.File/createTempFile "cleanup-test" nil)+ request {:multipart-params {"file" {:filename "a.csv", :tempfile file-1}+ ;; duplicate part names arrive as a vector of values+ "sneaky" [{:filename "b.csv", :tempfile file-2}]+ "some_id" "123"}}+ handler (api.macros/wrap-multipart-tempfile-cleanup+ (fn [_request]+ (throw (ex-info "Invalid request" {:status-code 400}))))]+ (is (thrown-with-msg? Exception #"Invalid request" (handler request)))+ (is (not (.exists file-1)))+ (is (not (.exists file-2))))))++(deftest ^:parallel multipart-tempfile-cleanup-on-raise-test+ (testing "async arity: tempfiles are deleted when the handler raises"+ (let [file (java.io.File/createTempFile "cleanup-test" nil)+ request {:multipart-params {"file" {:filename "a.csv", :tempfile file}}}+ raised (atom nil)+ handler (api.macros/wrap-multipart-tempfile-cleanup+ (fn [_request _respond raise]+ (raise (ex-info "Invalid request" {:status-code 400}))))]+ (handler request+ (fn [_response] (is false "respond should not be called"))+ (fn [e] (reset! raised e)))+ (is (some? @raised))+ (is (not (.exists file))))))++(deftest ^:parallel multipart-tempfile-preserved-on-success-test+ (testing "tempfiles are left alone when the handler completes (it owns and deletes what it consumes)"+ (let [file (java.io.File/createTempFile "cleanup-test" nil)+ request {:multipart-params {"file" {:filename "a.csv", :tempfile file}}}+ handler (api.macros/wrap-multipart-tempfile-cleanup (fn [_request] :ok))]+ (try+ (is (= :ok (handler request)))+ (is (.exists file))+ (finally (.delete file))))))+(deftest ^:parallel parse-args-metadata-test(testing "metadata map is parsed correctly"(are [args expected] (= expecteddiff --git a/test/metabase/upload/api_test.clj b/test/metabase/upload/api_test.cljindex 8380e71ccbd..f82984cf163 100644--- a/test/metabase/upload/api_test.clj+++ b/test/metabase/upload/api_test.clj@@ -1,8 +1,11 @@(ns metabase.upload.api-test(:require+ [clojure.java.io :as io]+ [clojure.string :as str][clojure.test :refer :all][metabase.test :as mt][metabase.upload.api :as upload.api]+ [metabase.upload.core :as upload][metabase.upload.impl-test :as upload-test][toucan2.core :as t2]))@@ -33,3 +36,67 @@status))(is (= body(t2/select-one-pk :model/Card :database_id (mt/id)))))))))++(def ^:private multipart-request-options+ {:request-options {:headers {"content-type" "multipart/form-data"}}})++(deftest csv-upload-too-large-test+ (testing "POST /api/upload/csv rejects a file over the size cap with a 413"+ ;; One byte over the cap is enough: the multipart middleware aborts streaming the file part as soon as it+ ;; crosses the limit, so the oversized body is never fully buffered.+ (let [oversized (byte-array (inc upload/max-upload-size-bytes))+ calls (atom 0)]+ (mt/with-dynamic-fn-redefs [upload.api/from-csv! (fn [& _] (swap! calls inc) {:status 200})]+ (is (= "Uploaded content exceeded limits."+ (mt/user-http-request :crowberto :post 413 "upload/csv" multipart-request-options+ {:file oversized})))+ (is (zero? @calls) "the endpoint body should never run")))))++(deftest csv-upload-at-size-cap-test+ (testing "POST /api/upload/csv accepts a file of exactly the size cap"+ (let [at-cap (byte-array upload/max-upload-size-bytes)+ calls (atom 0)]+ ;; like the real handler, the stub owns the uploaded tempfile and must delete it+ (mt/with-dynamic-fn-redefs [upload.api/from-csv! (fn [{:keys [file]}]+ (swap! calls inc)+ (io/delete-file file :silently)+ {:status 200, :body 1})]+ (mt/user-http-request :crowberto :post 200 "upload/csv" multipart-request-options+ [[:collection_id "root"]+ [:file at-cap]])+ (is (= 1 @calls) "the endpoint body should run")))))++(deftest csv-upload-too-many-parts-test+ (testing "POST /api/upload/csv rejects too many multipart parts with a 413"+ (let [calls (atom 0)]+ (mt/with-dynamic-fn-redefs [upload.api/from-csv! (fn [& _] (swap! calls inc) {:status 200})]+ (is (= "Uploaded content exceeded limits."+ (mt/user-http-request :crowberto :post 413 "upload/csv" multipart-request-options+ [[:collection_id "root"]+ [:file (byte-array [97 44 98 10 49 44 50])]+ [:file (byte-array [99 44 100 10 51 44 52])]])))+ (is (zero? @calls) "the endpoint body should never run")))))++(deftest csv-upload-with-collection-id-test+ (testing "POST /api/upload/csv accepts the collection_id field the frontend sends alongside the file"+ (let [calls (atom 0)]+ ;; like the real handler, the stub owns the uploaded tempfile and must delete it+ (mt/with-dynamic-fn-redefs [upload.api/from-csv! (fn [{:keys [file]}]+ (swap! calls inc)+ (io/delete-file file :silently)+ {:status 200, :body 1})]+ (mt/user-http-request :crowberto :post 200 "upload/csv" multipart-request-options+ [[:collection_id "root"]+ [:file (byte-array [97 44 98 10 49 44 50])]])+ (is (= 1 @calls) "the endpoint body should run")))))++(deftest csv-upload-smuggled-file-part-test+ (testing "POST /api/upload/csv rejects a second file part smuggled as collection_id"+ (let [calls (atom 0)]+ (mt/with-dynamic-fn-redefs [upload.api/from-csv! (fn [& _] (swap! calls inc) {:status 200})]+ (let [response (mt/user-http-request :crowberto :post 400 "upload/csv" multipart-request-options+ [[:file (byte-array [97 44 98 10 49 44 50])]+ [:collection_id (byte-array [99 44 100 10 51 44 52])]])]+ (testing "the validation error does not leak the tempfile path"+ (is (not (str/includes? (pr-str response) "ring-multipart-")))))+ (is (zero? @calls) "the endpoint body should never run")))))diff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.cljindex 209ac8fdc14..fa727118b2e 100644--- a/test/metabase/warehouse_schema_rest/api/table_test.clj+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj@@ -2,6 +2,7 @@"Tests for /api/table endpoints."{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.warehouse-schema-rest.api.table-test]}}}}}}(:require+ [clojure.java.io :as io][clojure.java.jdbc :as jdbc][clojure.set :as set][clojure.test :refer :all]@@ -18,6 +19,7 @@[metabase.sync.core :as sync][metabase.test :as mt][metabase.test.http-client :as client]+ [metabase.upload.core :as upload][metabase.upload.impl-test :as upload-test][metabase.util :as u][metabase.warehouse-schema-rest.api.table :as api.table]@@ -1195,6 +1197,65 @@:file file}))(is (not (.exists file)) "File should be deleted after replace-csv!")))))))+(def ^:private multipart-request-options+ {:request-options {:headers {"content-type" "multipart/form-data"}}})++(deftest update-csv-too-large-test+ ;; One byte over the cap is enough: the multipart middleware aborts streaming the file part as soon as it+ ;; crosses the limit, so the oversized body is never fully buffered.+ (let [oversized (byte-array (inc upload/max-upload-size-bytes))+ calls (atom 0)]+ (mt/with-dynamic-fn-redefs [api.table/update-csv! (fn [& _] (swap! calls inc) nil)]+ (doseq [endpoint ["append-csv" "replace-csv"]]+ (testing (format "POST /api/table/:id/%s rejects a file over the size cap with a 413" endpoint)+ (is (= "Uploaded content exceeded limits."+ (mt/user-http-request :crowberto :post 413 (format "table/%d/%s" (mt/id :venues) endpoint)+ multipart-request-options+ {:file oversized})))))+ (is (zero? @calls) "the endpoint bodies should never run"))))++(deftest update-csv-too-many-parts-test+ (let [calls (atom 0)]+ (mt/with-dynamic-fn-redefs [api.table/update-csv! (fn [& _] (swap! calls inc) nil)]+ (doseq [endpoint ["append-csv" "replace-csv"]]+ (testing (format "POST /api/table/:id/%s rejects too many multipart parts with a 413" endpoint)+ (is (= "Uploaded content exceeded limits."+ (mt/user-http-request :crowberto :post 413 (format "table/%d/%s" (mt/id :venues) endpoint)+ multipart-request-options+ [[:collection_id "root"]+ [:file (byte-array [97 44 98 10 49 44 50])]+ [:file (byte-array [99 44 100 10 51 44 52])]])))))+ (is (zero? @calls) "the endpoint bodies should never run"))))++(deftest update-csv-with-extra-field-test+ (let [calls (atom 0)]+ ;; like the real handler, the stub owns the uploaded tempfile and must delete it+ (mt/with-dynamic-fn-redefs [api.table/update-csv! (fn [{:keys [file]}]+ (swap! calls inc)+ (io/delete-file file :silently)+ {:status 200, :body ""})]+ (doseq [endpoint ["append-csv" "replace-csv"]]+ (testing (format "POST /api/table/:id/%s accepts the collection_id field the frontend sends" endpoint)+ (mt/user-http-request :crowberto :post 200 (format "table/%d/%s" (mt/id :venues) endpoint)+ multipart-request-options+ [[:collection_id "root"]+ [:file (byte-array [97 44 98 10 49 44 50])]])))+ (is (= 2 @calls) "the endpoint bodies should run"))))++(deftest update-csv-smuggled-file-part-test+ (let [calls (atom 0)]+ (mt/with-dynamic-fn-redefs [api.table/update-csv! (fn [& _] (swap! calls inc) nil)]+ (doseq [endpoint ["append-csv" "replace-csv"]+ ;; collection_id is caught by its :string constraint, unknown names by the closed map+ part-name [:collection_id :unexpected_part]]+ (testing (format "POST /api/table/:id/%s rejects a second file part smuggled as %s"+ endpoint (name part-name))+ (mt/user-http-request :crowberto :post 400 (format "table/%d/%s" (mt/id :venues) endpoint)+ multipart-request-options+ [[:file (byte-array [97 44 98 10 49 44 50])]+ [part-name (byte-array [99 44 100 10 51 44 52])]])))+ (is (zero? @calls) "the endpoint bodies should never run"))))+;;; +----------------------------------------------------------------------------------------------------------------+;;; | POST /api/table/:id/sync_schema |;;; +----------------------------------------------------------------------------------------------------------------+