Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Improve SQLite loading (#83285)

Nathan Voxland9dfd6951f316
diff --git a/src/metabase/driver/sqlite.clj b/src/metabase/driver/sqlite.clj
index 4afd29a560a..cdc85842be8 100644
--- a/src/metabase/driver/sqlite.clj
+++ b/src/metabase/driver/sqlite.clj
@@ -25,6 +25,7 @@
[metabase.util.malli :as mu]
[metabase.util.performance :refer [mapv]])
(:import
+ (java.net MalformedURLException URL)
(java.sql Connection ResultSet Types)
(java.time LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
(java.time.temporal Temporal)))
@@ -111,6 +112,16 @@
(.setReadOnly config true)
(.getProperty (.toProperties config) "open_mode")))
+(def ^:private protected-connection-properties
+ "Connection properties the driver sets itself for safety. Compared case-insensitively and without any namespace: the
+ client reads a property by its name alone, so `:foo/enable_load_extension` and `:ENABLE_LOAD_EXTENSION` are
+ `enable_load_extension` to it, and the JSON request decoder keeps such keys."
+ #{"limit_attached" "enable_load_extension" "open_mode"})
+
+(defn- protected-property-key? [k]
+ (contains? protected-connection-properties
+ (u/lower-case-en (if (instance? clojure.lang.Named k) (name k) (str k)))))
+
(defmethod sql-jdbc.conn/connection-details->spec :sqlite
[_ {:keys [db read-only?]
:or {db "sqlite.db"}
@@ -118,12 +129,46 @@
(merge {:classname "org.sqlite.JDBC"
:subprotocol "sqlite"
:subname db}
- (dissoc details :db :read-only?)
+ ;; a detail that names a property set below, in any spelling, would otherwise sit beside the forced value and
+ ;; the client would keep whichever it read last
+ (into {} (remove (comp protected-property-key? key)) (dissoc details :db :read-only?))
;; SQLite-JDBC can't flip a live connection to read-only, so set it at open time via `open_mode`.
(when read-only?
{:open_mode (read-only-open-mode)})
;; disallow "FDW" (connecting to other SQLite databases on the local filesystem) -- see https://github.com/metabase/metaboat/issues/152
- {:limit_attached 0}))
+ {:limit_attached 0
+ ;; `load_extension()` runs native code from a file on the Metabase host. Set here, the property also wins over
+ ;; an `enable_load_extension=true` written into the database path.
+ :enable_load_extension "false"}))
+
+(defn- url-hosts
+ "The hosts `java.net.URL` would contact to read `s`, or nil when `s` is not a URL it accepts. Parsed by `URL` itself
+ rather than by a pattern: it ignores leading and trailing whitespace and a `url:` prefix, and it takes the host after
+ the last `@` of the authority, so any other reading of the string can name a different host than the one fetched."
+ [^String s]
+ (when-let [^URL url (try (URL. s) (catch MalformedURLException _ nil))]
+ (if (= "jar" (.getProtocol url))
+ ;; `jar:<url>!/<entry>` fetches the jar from the nested URL
+ (url-hosts (first (str/split (.getPath url) #"!/" 2)))
+ (let [host (str/replace (.getHost url) #"^\[(.*)\]$" "$1")]
+ ;; a `file:` URL naming no host, or this one, is a local file; one naming another machine is fetched from it
+ (when-not (or (str/blank? host)
+ (and (= "file" (.getProtocol url))
+ (#{"localhost" "~"} (u/lower-case-en host))))
+ [host])))))
+
+(defmethod driver/connection-parameter-hosts :sqlite
+ [driver {:keys [db] :as details}]
+ ;; Two things in `:db` have the client fetch the database file from a URL, and no SSH tunnel carries either, so they
+ ;; are reported here, where they are checked whether or not a tunnel is in use, rather than as a connection host:
+ ;; - `:resource:<url>`, which SQLite-JDBC reads from that URL (after trying the classpath);
+ ;; - a `:db` that is itself a URL, which the file check in [[can-connect?]] opens with `io/input-stream`.
+ (let [db (when (string? db) db)
+ resource (some-> db (str/trim) (->> (re-find #"(?i)^:resource:(.*)$")) second)]
+ (into ((get-method driver/connection-parameter-hosts :sql-jdbc) driver details)
+ cat
+ [(some-> resource url-hosts)
+ (some-> db url-hosts)])))
(defmethod driver/describe-table-indexes :sqlite
[driver database table]
diff --git a/test/metabase/driver/sqlite_test.clj b/test/metabase/driver/sqlite_test.clj
index 1403a0236e1..2583b5a31be 100644
--- a/test/metabase/driver/sqlite_test.clj
+++ b/test/metabase/driver/sqlite_test.clj
@@ -11,6 +11,7 @@
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor-test-util :as sql.qp-test-util]
+ [metabase.driver.util :as driver.u]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.query-processor.test :as qp]
@@ -327,6 +328,83 @@
#"SQL error or missing database \(no such table: fdw_test\.products\)"
(qp/process-query (mt/native-query {:query "SELECT count(*) FROM fdw_test.products;"})))))))))
+(defn- load-extension-refusal
+ "The error SQLite gives when asked to load an extension that does not exist over a connection built from `details`:
+ `not authorized` while extension loading is disabled, `no such file` once it tries to open the file."
+ [details]
+ (let [spec (sql-jdbc.conn/connection-details->spec :sqlite (merge {:db ":memory:"} details))]
+ (try
+ (jdbc/query spec ["SELECT load_extension('/nonexistent/metabase-test-extension')"])
+ ::loaded
+ (catch Exception e
+ (re-find #"not authorized|no such file" (ex-message e))))))
+
+(deftest ^:parallel extension-loading-is-always-disabled-test
+ (testing "loading an extension runs native code in the Metabase process, so it is disabled whatever the details say"
+ ;; detail keys reach the client as connection properties, and it also reads settings from the database path
+ (doseq [details [{}
+ {:enable_load_extension true}
+ {:enable_load_extension "true"}
+ {:db ":memory:?enable_load_extension=true"}
+ {:db "file::memory:?ENABLE_LOAD_EXTENSION=true"}
+ ;; a namespaced or differently-cased key names the same connection property once it reaches the
+ ;; client, and the JSON decoder keeps such keys
+ {:foo/enable_load_extension "true"}
+ {:ENABLE_LOAD_EXTENSION "true"}
+ {:foo/Enable_Load_Extension true}]]
+ (testing (pr-str details)
+ (is (= "not authorized" (load-extension-refusal details)))))))
+
+(deftest protected-connection-properties-cannot-be-overridden-test
+ (testing "properties the driver sets for safety survive a namespaced or differently-cased detail key"
+ (doseq [[k v] [[:foo/limit_attached "10"]
+ [:LIMIT_ATTACHED "10"]
+ [:foo/enable_load_extension "true"]]]
+ (let [spec (sql-jdbc.conn/connection-details->spec :sqlite {:db ":memory:" k v})]
+ (testing (pr-str k)
+ (is (= 0 (:limit_attached spec)))
+ (is (= "false" (:enable_load_extension spec)))
+ (is (= [] (->> (keys spec)
+ (filter #(re-find #"(?i)limit_attached|enable_load_extension" (name %)))
+ (remove #{:limit_attached :enable_load_extension})))))))
+ (testing "a read-only database cannot be made writable the same way"
+ (let [spec (sql-jdbc.conn/connection-details->spec :sqlite {:db ":memory:" :read-only? true :foo/open_mode "6"})]
+ (is (not-any? #(= "foo" (namespace %)) (keys spec)))
+ (is (= (:open_mode spec) (:open_mode (sql-jdbc.conn/connection-details->spec :sqlite {:db ":memory:" :read-only? true}))))))))
+
+(deftest resource-urls-are-subject-to-the-network-policy-test
+ ;; `:resource:<url>` has the client fetch the database file from that URL
+ (mt/with-temp-env-var-value! [mb-warehouse-allowed-networks "external-only"]
+ (testing "a resource URL naming an internal host is refused"
+ (doseq [details [{:db ":resource:http://localhost/x.db"}
+ {:db ":resource:HTTP://127.0.0.1:8080/x.db"}
+ {:db ":resource:jar:http://localhost/x.jar!/x.db"}
+ ;; the SSH tunnel details mean nothing to a file-backed database, so they cannot swap the
+ ;; database's own hosts for the tunnel's
+ {:db ":resource:http://localhost/x.db" :tunnel-enabled true :tunnel-host "8.8.8.8"}
+ ;; `java.net.URL` ignores leading whitespace and a `url:` prefix, so the host it contacts is
+ ;; not the one a looser reading of the string would name
+ {:db ":resource: http://8.8.8.8;@127.0.0.1:8099/x.db"}
+ {:db ":resource:url:http://8.8.8.8;@127.0.0.1:8099/x.db"}
+ {:db ":RESOURCE: url:http://127.0.0.1:8099/x.db"}
+ {:db ":resource:jar:url:http://127.0.0.1:8099/x.jar!/x.db"}
+ ;; a `file:` URL naming another machine is fetched from it
+ {:db ":resource:file://10.0.0.5/x.db"}
+ ;; the file check opens a plain URL in `:db` too, with or without tunnel details
+ {:db "http://127.0.0.1:8099/plain"}
+ {:db " http://127.0.0.1:8099/plain"}
+ {:db "http://127.0.0.1:8099/plain" :tunnel-enabled true :tunnel-host "8.8.8.8"}]]
+ (testing (pr-str details)
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"private or internal network address"
+ (driver.u/validate-connection-hosts! :sqlite details))))))
+ (testing "a local database file, or a resource that is not fetched over the network, names no host"
+ (doseq [details [{:db "/tmp/x.db"}
+ {:db ":memory:"}
+ {:db ":resource:file:/tmp/x.db"}
+ {:db "file:/tmp/x.db?mode=ro"}]]
+ (testing (pr-str details)
+ (is (nil? (driver.u/validate-connection-hosts! :sqlite details))))))))
+
(deftest ^:parallel non-date-in-date-columns-are-returned-test
(mt/test-driver :sqlite
(mt/dataset (mt/dataset-definition