The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve SQLite loading (#83285)
Nathan Voxland9dfd6951f316
diff --git a/src/metabase/driver/sqlite.clj b/src/metabase/driver/sqlite.cljindex 4afd29a560a..cdc85842be8 100644--- a/src/metabase/driver/sqlite.clj+++ b/src/metabase/driver/sqlite.clj@@ -25,6 +25,7 @@[metabase.util.malli :as mu][metabase.util.performance :refer [mapv]])(:import+ (java.net MalformedURLException URL)(java.sql Connection ResultSet Types)(java.time LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)(java.time.temporal Temporal)))@@ -111,6 +112,16 @@(.setReadOnly config true)(.getProperty (.toProperties config) "open_mode")))+(def ^:private protected-connection-properties+ "Connection properties the driver sets itself for safety. Compared case-insensitively and without any namespace: the+ client reads a property by its name alone, so `:foo/enable_load_extension` and `:ENABLE_LOAD_EXTENSION` are+ `enable_load_extension` to it, and the JSON request decoder keeps such keys."+ #{"limit_attached" "enable_load_extension" "open_mode"})++(defn- protected-property-key? [k]+ (contains? protected-connection-properties+ (u/lower-case-en (if (instance? clojure.lang.Named k) (name k) (str k)))))+(defmethod sql-jdbc.conn/connection-details->spec :sqlite[_ {:keys [db read-only?]:or {db "sqlite.db"}@@ -118,12 +129,46 @@(merge {:classname "org.sqlite.JDBC":subprotocol "sqlite":subname db}- (dissoc details :db :read-only?)+ ;; a detail that names a property set below, in any spelling, would otherwise sit beside the forced value and+ ;; the client would keep whichever it read last+ (into {} (remove (comp protected-property-key? key)) (dissoc details :db :read-only?));; SQLite-JDBC can't flip a live connection to read-only, so set it at open time via `open_mode`.(when read-only?{:open_mode (read-only-open-mode)});; disallow "FDW" (connecting to other SQLite databases on the local filesystem) -- see https://github.com/metabase/metaboat/issues/152- {:limit_attached 0}))+ {:limit_attached 0+ ;; `load_extension()` runs native code from a file on the Metabase host. Set here, the property also wins over+ ;; an `enable_load_extension=true` written into the database path.+ :enable_load_extension "false"}))++(defn- url-hosts+ "The hosts `java.net.URL` would contact to read `s`, or nil when `s` is not a URL it accepts. Parsed by `URL` itself+ rather than by a pattern: it ignores leading and trailing whitespace and a `url:` prefix, and it takes the host after+ the last `@` of the authority, so any other reading of the string can name a different host than the one fetched."+ [^String s]+ (when-let [^URL url (try (URL. s) (catch MalformedURLException _ nil))]+ (if (= "jar" (.getProtocol url))+ ;; `jar:<url>!/<entry>` fetches the jar from the nested URL+ (url-hosts (first (str/split (.getPath url) #"!/" 2)))+ (let [host (str/replace (.getHost url) #"^\[(.*)\]$" "$1")]+ ;; a `file:` URL naming no host, or this one, is a local file; one naming another machine is fetched from it+ (when-not (or (str/blank? host)+ (and (= "file" (.getProtocol url))+ (#{"localhost" "~"} (u/lower-case-en host))))+ [host])))))++(defmethod driver/connection-parameter-hosts :sqlite+ [driver {:keys [db] :as details}]+ ;; Two things in `:db` have the client fetch the database file from a URL, and no SSH tunnel carries either, so they+ ;; are reported here, where they are checked whether or not a tunnel is in use, rather than as a connection host:+ ;; - `:resource:<url>`, which SQLite-JDBC reads from that URL (after trying the classpath);+ ;; - a `:db` that is itself a URL, which the file check in [[can-connect?]] opens with `io/input-stream`.+ (let [db (when (string? db) db)+ resource (some-> db (str/trim) (->> (re-find #"(?i)^:resource:(.*)$")) second)]+ (into ((get-method driver/connection-parameter-hosts :sql-jdbc) driver details)+ cat+ [(some-> resource url-hosts)+ (some-> db url-hosts)])))(defmethod driver/describe-table-indexes :sqlite[driver database table]diff --git a/test/metabase/driver/sqlite_test.clj b/test/metabase/driver/sqlite_test.cljindex 1403a0236e1..2583b5a31be 100644--- a/test/metabase/driver/sqlite_test.clj+++ b/test/metabase/driver/sqlite_test.clj@@ -11,6 +11,7 @@[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.query-processor-test-util :as sql.qp-test-util]+ [metabase.driver.util :as driver.u][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.query-processor.test :as qp]@@ -327,6 +328,83 @@#"SQL error or missing database \(no such table: fdw_test\.products\)"(qp/process-query (mt/native-query {:query "SELECT count(*) FROM fdw_test.products;"})))))))))+(defn- load-extension-refusal+ "The error SQLite gives when asked to load an extension that does not exist over a connection built from `details`:+ `not authorized` while extension loading is disabled, `no such file` once it tries to open the file."+ [details]+ (let [spec (sql-jdbc.conn/connection-details->spec :sqlite (merge {:db ":memory:"} details))]+ (try+ (jdbc/query spec ["SELECT load_extension('/nonexistent/metabase-test-extension')"])+ ::loaded+ (catch Exception e+ (re-find #"not authorized|no such file" (ex-message e))))))++(deftest ^:parallel extension-loading-is-always-disabled-test+ (testing "loading an extension runs native code in the Metabase process, so it is disabled whatever the details say"+ ;; detail keys reach the client as connection properties, and it also reads settings from the database path+ (doseq [details [{}+ {:enable_load_extension true}+ {:enable_load_extension "true"}+ {:db ":memory:?enable_load_extension=true"}+ {:db "file::memory:?ENABLE_LOAD_EXTENSION=true"}+ ;; a namespaced or differently-cased key names the same connection property once it reaches the+ ;; client, and the JSON decoder keeps such keys+ {:foo/enable_load_extension "true"}+ {:ENABLE_LOAD_EXTENSION "true"}+ {:foo/Enable_Load_Extension true}]]+ (testing (pr-str details)+ (is (= "not authorized" (load-extension-refusal details)))))))++(deftest protected-connection-properties-cannot-be-overridden-test+ (testing "properties the driver sets for safety survive a namespaced or differently-cased detail key"+ (doseq [[k v] [[:foo/limit_attached "10"]+ [:LIMIT_ATTACHED "10"]+ [:foo/enable_load_extension "true"]]]+ (let [spec (sql-jdbc.conn/connection-details->spec :sqlite {:db ":memory:" k v})]+ (testing (pr-str k)+ (is (= 0 (:limit_attached spec)))+ (is (= "false" (:enable_load_extension spec)))+ (is (= [] (->> (keys spec)+ (filter #(re-find #"(?i)limit_attached|enable_load_extension" (name %)))+ (remove #{:limit_attached :enable_load_extension})))))))+ (testing "a read-only database cannot be made writable the same way"+ (let [spec (sql-jdbc.conn/connection-details->spec :sqlite {:db ":memory:" :read-only? true :foo/open_mode "6"})]+ (is (not-any? #(= "foo" (namespace %)) (keys spec)))+ (is (= (:open_mode spec) (:open_mode (sql-jdbc.conn/connection-details->spec :sqlite {:db ":memory:" :read-only? true}))))))))++(deftest resource-urls-are-subject-to-the-network-policy-test+ ;; `:resource:<url>` has the client fetch the database file from that URL+ (mt/with-temp-env-var-value! [mb-warehouse-allowed-networks "external-only"]+ (testing "a resource URL naming an internal host is refused"+ (doseq [details [{:db ":resource:http://localhost/x.db"}+ {:db ":resource:HTTP://127.0.0.1:8080/x.db"}+ {:db ":resource:jar:http://localhost/x.jar!/x.db"}+ ;; the SSH tunnel details mean nothing to a file-backed database, so they cannot swap the+ ;; database's own hosts for the tunnel's+ {:db ":resource:http://localhost/x.db" :tunnel-enabled true :tunnel-host "8.8.8.8"}+ ;; `java.net.URL` ignores leading whitespace and a `url:` prefix, so the host it contacts is+ ;; not the one a looser reading of the string would name+ {:db ":resource: http://8.8.8.8;@127.0.0.1:8099/x.db"}+ {:db ":resource:url:http://8.8.8.8;@127.0.0.1:8099/x.db"}+ {:db ":RESOURCE: url:http://127.0.0.1:8099/x.db"}+ {:db ":resource:jar:url:http://127.0.0.1:8099/x.jar!/x.db"}+ ;; a `file:` URL naming another machine is fetched from it+ {:db ":resource:file://10.0.0.5/x.db"}+ ;; the file check opens a plain URL in `:db` too, with or without tunnel details+ {:db "http://127.0.0.1:8099/plain"}+ {:db " http://127.0.0.1:8099/plain"}+ {:db "http://127.0.0.1:8099/plain" :tunnel-enabled true :tunnel-host "8.8.8.8"}]]+ (testing (pr-str details)+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"private or internal network address"+ (driver.u/validate-connection-hosts! :sqlite details))))))+ (testing "a local database file, or a resource that is not fetched over the network, names no host"+ (doseq [details [{:db "/tmp/x.db"}+ {:db ":memory:"}+ {:db ":resource:file:/tmp/x.db"}+ {:db "file:/tmp/x.db?mode=ro"}]]+ (testing (pr-str details)+ (is (nil? (driver.u/validate-connection-hosts! :sqlite details))))))))+(deftest ^:parallel non-date-in-date-columns-are-returned-test(mt/test-driver :sqlite(mt/dataset (mt/dataset-definition