The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Remove legacy `:druid` driver / migrate to `:druid-jdbc` (#75143)
Cam Saul4eb1408f48cd
diff --git a/.claude/agents/drivers-and-sync-backend-expert.md b/.claude/agents/drivers-and-sync-backend-expert.mdindex 80bb2ee43bc..85f00b9aa4a 100644--- a/.claude/agents/drivers-and-sync-backend-expert.md+++ b/.claude/agents/drivers-and-sync-backend-expert.md@@ -22,7 +22,6 @@ You understand Metabase's driver architecture built on Clojure multimethods with:bigquery-cloud-sdk, :databricks, :clickhouse, :athena, :sparksql,:presto-jdbc / :starburst, :vertica, :sqlite, :h2:mongo (non-SQL, custom protocol)-:druid (non-SQL, REST-based)```Key driver extension points you know intimately:diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex c5bc1c167ce..d635e274342 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -249,7 +249,6 @@{:team "Graphy":api #{metabase.analyze.core}:uses #{config- driverlibmodelsquery-processordiff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex 501b96243cd..3971e4556e3 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -296,54 +296,6 @@ jobs:upload-logs: ${{ steps.test-driver.outcome == 'failure' }}logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}- be-tests-druid-ee:- needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.druid-should-run == 'true' || needs.determine-driver-skips.outputs.druid-quarantine-conflict == 'true' }}- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}- timeout-minutes: 40- env:- CI: 'true'- DRIVERS: druid- services:- druid:- image: metabase/druid:0.20.2- ports:- - "8082:8082"- env:- CLUSTER_SIZE: nano-quickstart- name: Druid (Legacy)- steps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.druid-quarantine-conflict == 'true' }}- run: |- echo "::error::Druid driver is quarantined but you changed files in modules/drivers/druid/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run Druid tests, add the label: break-quarantine-druid"- echo ""- echo "If you merge without testing, you risk breaking the Druid driver!"- exit 1- - uses: actions/checkout@v6- - name: Test Druid driver- id: test-driver- uses: ./.github/actions/test-driver- with:- junit-name: 'be-tests-druid-ee'- test-args: >-- :only-tags [:mb/driver-tests]- :exclude-tags [:mb/transforms-python-test]- - name: Upload Test Results- uses: ./.github/actions/upload-test-results- if: always()- with:- input-path: ./target/junit/- output-name: ${{ github.job }}- bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}- role-to-assume: ${{ secrets.CI_TEST_RESULTS_IAM_ROLE }}- aws-region: ${{ vars.AWS_REGION }}- trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}- previous-step-outcome: ${{ steps.test-driver.outputs.test-outcome }}-be-tests-druid-jdbc-ee:needs: determine-driver-skipsif: ${{ needs.determine-driver-skips.outputs.druid-jdbc-should-run == 'true' || needs.determine-driver-skips.outputs.druid-jdbc-quarantine-conflict == 'true' }}@@ -1629,7 +1581,6 @@ jobs:# these are not reliable, but we want to run them anyway to gather data about failures# - be-tests-snowflake-ee# - be-tests-bigquery-cloud-sdk-ee- - be-tests-druid-ee- be-tests-databricks-ee- be-tests-druid-jdbc-ee- be-tests-mysql-mariadbdiff --git a/.github/workflows/snyk.yml b/.github/workflows/snyk.ymlindex 50f830c3c1a..9291dbcde67 100644--- a/.github/workflows/snyk.yml+++ b/.github/workflows/snyk.yml@@ -107,9 +107,6 @@ jobs:- name: Driver - Databricksfile: modules/drivers/databricks/pom.xmlcategory: driver-databricks- - name: Driver - Druid- file: modules/drivers/druid/pom.xml- category: driver-druid- name: Driver - Druid JDBCfile: modules/drivers/druid-jdbc/pom.xmlcategory: driver-druid-jdbcdiff --git a/bin/build/src/i18n/enumerate.clj b/bin/build/src/i18n/enumerate.cljindex eca36140616..0b41f70d9c3 100644--- a/bin/build/src/i18n/enumerate.clj+++ b/bin/build/src/i18n/enumerate.clj@@ -19,7 +19,6 @@["/src""/enterprise/backend/src""/modules/drivers/bigquery-cloud-sdk/src"- "/modules/drivers/druid/src""/modules/drivers/druid-jdbc/src""/modules/drivers/mongo/src""/modules/drivers/oracle/src"diff --git a/deps.edn b/deps.ednindex 8ab8d42c7a9..45927a5eb83 100644--- a/deps.edn+++ b/deps.edn@@ -497,7 +497,6 @@"modules/drivers/bigquery-cloud-sdk/test""modules/drivers/clickhouse/test""modules/drivers/databricks/test"- "modules/drivers/druid/test""modules/drivers/druid-jdbc/test""modules/drivers/hive-like/test""modules/drivers/mongo/test"@@ -561,8 +560,6 @@"modules/drivers/databricks/test""modules/drivers/druid-jdbc/src""modules/drivers/druid-jdbc/test"- "modules/drivers/druid/src"- "modules/drivers/druid/test""modules/drivers/hive-like/src""modules/drivers/hive-like/test""modules/drivers/mongo/src"@@ -600,7 +597,6 @@"modules/drivers/bigquery-cloud-sdk/src""modules/drivers/clickhouse/src""modules/drivers/databricks/src"- "modules/drivers/druid/src""modules/drivers/druid-jdbc/src""modules/drivers/hive-like/src""modules/drivers/mongo/src"@@ -626,7 +622,6 @@"modules/drivers/bigquery-cloud-sdk/src""modules/drivers/clickhouse/src""modules/drivers/databricks/src"- "modules/drivers/druid/src""modules/drivers/druid-jdbc/src""modules/drivers/hive-like/src""modules/drivers/mongo/src"diff --git a/docs/databases/connections/druid.md b/docs/databases/connections/druid.mdindex 4536edce8c6..e0894ae8cdd 100644--- a/docs/databases/connections/druid.md+++ b/docs/databases/connections/druid.md@@ -4,8 +4,6 @@ title: Druid# Druid-> Prefer the Druid JDBC connection.-To add a database connection, click the **grid** icon in the top right, and navigate to **Admin** > **Databases** > **Add a database**.## Edit connection details@@ -34,16 +32,12 @@ See our [guide to SSH tunneling](../ssh-tunnel.md).### Allow unfolding of JSON columns-> Only available for the Druid JDBC connection.-This enables unfolding of JSON columns into their component fields.See [JSON unfolding](../../data-modeling/json-unfolding.md).### Authentication-> Only available for the Druid JDBC connection.-Toggle on to enable basic authentication with a username and password.### Re-run queries for simple explorationsdiff --git a/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts b/e2e/test/scenarios/admin/database-connection-strings.cy.spec.tsindex 1f9c53fd52e..ddca0407faa 100644--- a/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts+++ b/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts@@ -51,12 +51,12 @@ const databaseTestCases = [],},{- engine: "Druid",+ engine: "Druid JDBC",connectionString:"jdbc:avatica:remote:url=http://localhost:8888/druid/v2/sql/avatica/;transparent_reconnection=true",expectedFields: [{ label: "Host", value: "localhost" },- { label: "Broker node port", value: "8888" },+ { label: "Router node port", value: "8888" },],},{diff --git a/e2e/test/scenarios/admin/databases.cy.spec.js b/e2e/test/scenarios/admin/databases.cy.spec.jsindex d0011b6efae..e0228f5a548 100644--- a/e2e/test/scenarios/admin/databases.cy.spec.js+++ b/e2e/test/scenarios/admin/databases.cy.spec.js@@ -449,7 +449,7 @@ describe("database page > side panel", () => {{ name: "Amazon Redshift", file: "redshift" },{ name: "ClickHouse", file: "clickhouse" },{ name: "Databricks", file: "databricks" },- { name: "Druid", file: "druid" },+ { name: "Druid JDBC", file: "druid" },{ name: "MongoDB", file: "mongo" },{ name: "MySQL", file: "mysql" },{ name: "PostgreSQL", file: "postgresql" },diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex 4eb0033c361..957f456fcc4 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -266,7 +266,6 @@:bigquery:clickhouse:databricks- :druid:druid-jdbc:mongo:mongo-ssl@@ -289,7 +288,6 @@"bigquery-cloud-sdk" [:bigquery]"clickhouse" [:clickhouse]"databricks" [:databricks]- "druid" [:druid]"druid-jdbc" [:druid-jdbc]"mongo" [:mongo :mongo-ssl :mongo-sharded-cluster]"oracle" [:oracle]diff --git a/modules/drivers/deps.edn b/modules/drivers/deps.ednindex 98f65a210d0..b23239e9ae5 100644--- a/modules/drivers/deps.edn+++ b/modules/drivers/deps.edn@@ -11,7 +11,6 @@metabase/bigquery-cloud-sdk {:local/root "bigquery-cloud-sdk"}metabase/clickhouse {:local/root "clickhouse"}metabase/databricks {:local/root "databricks"}- metabase/druid {:local/root "druid"}metabase/druid-jdbc {:local/root "druid-jdbc"}metabase/hive-like {:local/root "hive-like"}metabase/mongo {:local/root "mongo"}diff --git a/modules/drivers/druid/deps.edn b/modules/drivers/druid/deps.edndeleted file mode 100644index aa7725e18a5..00000000000--- a/modules/drivers/druid/deps.edn+++ /dev/null@@ -1 +0,0 @@-{:paths ["src" "resources"]}diff --git a/modules/drivers/druid/resources/metabase/druid/metabase-plugin.yaml b/modules/drivers/druid/resources/metabase/druid/metabase-plugin.yamldeleted file mode 100644index dd892dcf94a..00000000000--- a/modules/drivers/druid/resources/metabase/druid/metabase-plugin.yaml+++ /dev/null@@ -1,48 +0,0 @@-info:- name: Metabase Druid Driver- version: 1.0.0-SNAPSHOT- description: Allows Metabase to connect to Druid databases.-driver:- name: druid- display-name: Druid- lazy-load: true- connection-properties:- - group:- container-style: ["grid", "3fr 1fr"]- fields:- - merge:- - host- - placeholder: http://localhost- required: true- - merge:- - port- - display-name: Broker node port- placeholder: 8082- required: true- - cloud-ip-address-info- - group:- container-style: ["component", "backdrop"]- fields:- - ssh-tunnel- - advanced-options-start- - name: auth-enabled- display-name: Authentication header- default: false- type: boolean- visible-if:- advanced-options: true- - name: auth-username- display-name: Username- type: string- visible-if:- auth-enabled: true- - name: auth-token- display-name: Token- type: secret- secret-kind: password- visible-if:- auth-enabled: true- - default-advanced-options-init:- - step: load-namespace- namespace: metabase.driver.druiddiff --git a/modules/drivers/druid/src/metabase/driver/druid.clj b/modules/drivers/druid/src/metabase/driver/druid.cljdeleted file mode 100644index 95f02446343..00000000000--- a/modules/drivers/druid/src/metabase/driver/druid.clj+++ /dev/null@@ -1,68 +0,0 @@-(ns metabase.driver.druid- "Druid driver."- (:require- [clj-http.client :as http]- [metabase.driver :as driver]- [metabase.driver-api.core :as driver-api]- [metabase.driver.druid.client :as druid.client]- [metabase.driver.druid.execute :as druid.execute]- [metabase.driver.druid.query-processor :as druid.qp]- [metabase.driver.druid.sync :as druid.sync]- [metabase.driver.settings :as driver.settings]- [metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh]- [metabase.util.json :as json]))--(driver/register! :druid)--(doseq [[feature supported?] {:expression-aggregations true- :expression-literals true- :schemas false- :set-timezone true- :temporal/requires-default-unit true- :database-routing true}]- (defmethod driver/database-supports? [:druid feature] [_driver _feature _db] supported?))--(defmethod driver/can-connect? :druid- [_ details]- {:pre [(map? details)]}- (ssh/with-ssh-tunnel [details-with-tunnel details]- (let [{:keys [auth-enabled auth-username auth-token-value]} details]- (= 200 (:status (http/get (druid.client/details->url details-with-tunnel "/status")- (cond-> {}- auth-enabled (assoc :basic-auth (str auth-username ":" auth-token-value)))))))))--(defmethod driver/describe-table :druid- [_ database table]- (druid.sync/describe-table database table))--(defmethod driver/dbms-version :druid- [_ database]- (druid.sync/dbms-version database))--(defmethod driver/describe-database* :druid- [_ database]- (druid.sync/describe-database database))--(defmethod driver/mbql->native :druid- [_ query]- (druid.qp/mbql->native query))--(defn- add-timeout-to-query [query timeout]- (let [parsed (if (string? query)- (json/decode+kw query)- query)]- (assoc-in parsed [:context :timeout] timeout)))--(defmethod driver/execute-reducible-query :druid- [_driver query _context respond]- (druid.execute/execute-reducible-query- (partial druid.client/do-query-with-cancellation (driver-api/canceled-chan))- (update-in query [:native :query] add-timeout-to-query driver.settings/*query-timeout-ms*)- respond))--(defmethod driver/db-start-of-week :druid- [_]- :sunday)--(defmethod driver/llm-sql-dialect-resource :druid [_]- "metabot/prompts/dialects/druid.md")diff --git a/modules/drivers/druid/src/metabase/driver/druid/client.clj b/modules/drivers/druid/src/metabase/driver/druid/client.cljdeleted file mode 100644index 9886df99f38..00000000000--- a/modules/drivers/druid/src/metabase/driver/druid/client.clj+++ /dev/null@@ -1,120 +0,0 @@-(ns metabase.driver.druid.client- (:refer-clojure :exclude [get-in])- (:require- [clj-http.client :as http]- [clojure.core.async :as a]- [metabase.driver-api.core :as driver-api]- [metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh]- [metabase.util :as u]- [metabase.util.i18n :refer [tru]]- [metabase.util.json :as json]- [metabase.util.log :as log]- [metabase.util.performance :refer [get-in]]))--(set! *warn-on-reflection* true)--(defn details->url- "Helper for building a Druid URL.-- (details->url {:host \"http://localhost\", :port 8082} \"druid/v2\") -> \"http://localhost:8082/druid/v2\""- [{:keys [host port]} & strs]- {:pre [(string? host) (seq host) (integer? port)]}- (apply str (format "%s:%d" host port) (map name strs)))--(defn- do-request- "Perform a JSON request using `request-fn` against `url`.-- (do-request http/get \"http://my-json-api.net\")"- [request-fn url & {:as options}]- {:pre [(fn? request-fn) (string? url)]}- ;; this is the way the `Content-Type` header is formatted in requests made by the Druid web interface- (let [{:keys [auth-enabled auth-username auth-token-value]} options- options (cond-> (merge {:content-type "application/json;charset=UTF-8"} options)- (:body options) (update :body json/encode)- auth-enabled (assoc :basic-auth (str auth-username ":" auth-token-value)))]- (try- (let [{:keys [status body]} (request-fn url options)]- (when (not= status 200)- (throw (ex-info (tru "Druid request error [{0}]: {1}" status (pr-str body))- {:type driver-api/qp.error-type.db})))- (try- (json/decode+kw body)- (catch Throwable e- (throw (ex-info (tru "Failed to parse Druid response body: {0}" (pr-str body))- {:type driver-api/qp.error-type.db}- e)))))- (catch Throwable e- (let [response (u/ignore-exceptions- (when-let [body (:body (ex-data e))]- (json/decode+kw body)))]- (throw (ex-info (or (:errorMessage response)- (.getMessage e))- (merge- {:type driver-api/qp.error-type.db- :request-url url- :request-options options}- (when response- {:response response}))- e)))))))--(def ^{:arglists '([url & {:as options}]), :style/indent [:form]} GET "Execute a GET request." (partial do-request http/get))-(def ^{:arglists '([url & {:as options}]), :style/indent [:form]} POST "Execute a POST request." (partial do-request http/post))-(def ^{:arglists '([url & {:as options}]), :style/indent [:form]} DELETE "Execute a DELETE request." (partial do-request http/delete))--(defn do-query- "Run a Druid `query` against database connection `details`."- [details query]- {:pre [(map? details) (map? query)]}- (ssh/with-ssh-tunnel [details-with-tunnel details]- (try- (POST (details->url details-with-tunnel "/druid/v2"),- :body query- :auth-enabled (:auth-enabled details)- :auth-username (:auth-username details)- :auth-token-value (driver-api/secret-value-as-string :druid details "auth-token"))- ;; don't need to do anything fancy if the query was killed- (catch InterruptedException e- (throw e))- (catch Throwable e- (let [e' (ex-info (.getMessage e)- {:type driver-api/qp.error-type.db- :query query}- e)]- (log/error e' "Error running query")- ;; Re-throw a new exception with `message` set to the extracted message- (throw e'))))))--(defn- undo-query-with-id! [details query-id]- (if-not query-id- (log/warn "Client closed connection, no queryId found, can't cancel query")- (ssh/with-ssh-tunnel [details-with-tunnel details]- (log/warnf "Client closed connection, canceling Druid queryId %s" query-id)- (try- (log/debugf "Canceling Druid query with ID %s" query-id)- (DELETE (details->url details-with-tunnel (format "/druid/v2/%s" query-id))- :auth-enabled (:auth-enabled details)- :auth-username (:auth-username details)- :auth-token-value (driver-api/secret-value-as-string :druid details "auth-token"))- (catch Exception cancel-e- (log/warnf cancel-e "Failed to cancel Druid query with queryId %s" query-id))))))--(defn do-query-with-cancellation- "Run a Druid `query`, canceling it if `canceled-chan` gets a message."- [canceled-chan details query]- {:pre [(map? details) (map? query)]}- (let [query-id (get-in query [:context :queryId])- ;; Run the query in a future so that we can interrupt it from the canceled-chan callback.- query-fut (future (do-query details query))- undo! #(undo-query-with-id! details query-id)]- (a/go- (when (a/<! canceled-chan)- (future-cancel query-fut)- (undo!)))- (try @query-fut- (catch java.util.concurrent.ExecutionException e- (undo!)- ;; Unwrap exceptions from the future.- (throw (.getCause e)))- (catch InterruptedException e- (undo!)- (throw e)))))diff --git a/modules/drivers/druid/src/metabase/driver/druid/execute.clj b/modules/drivers/druid/src/metabase/driver/druid/execute.cljdeleted file mode 100644index ecc551aa201..00000000000--- a/modules/drivers/druid/src/metabase/driver/druid/execute.clj+++ /dev/null@@ -1,185 +0,0 @@-(ns metabase.driver.druid.execute- (:require- [clojure.math.numeric-tower :as math]- [java-time.api :as t]- [medley.core :as m]- [metabase.driver-api.core :as driver-api]- [metabase.driver.connection :as driver.conn]- [metabase.driver.druid.query-processor :as druid.qp]- [metabase.util :as u]- [metabase.util.date-2 :as u.date]- [metabase.util.i18n :refer [tru]]- [metabase.util.json :as json]- [metabase.util.malli :as mu]))--(set! *warn-on-reflection* true)--(defn- resolve-timezone- "Returns the timezone object (either report-timezone or JVM timezone). Returns nil if the timezone is UTC as the- timestamps from Druid are already in UTC and don't need to be converted"- [_]- (when-not (= (t/zone-id (driver-api/results-timezone-id)) (t/zone-id "UTC"))- (driver-api/results-timezone-id)))--(defmulti ^:private post-process- "Do appropriate post-processing on the results of a query based on the `query-type`."- {:arglists '([query-type projections timezone-and-middleware-settings results])}- (fn [query-type _ _ _]- query-type))--(defmethod post-process ::druid.qp/scan- [_ projections _ results]- {:projections projections- :results (for [{:keys [events]} results- {timestamp :__time, :as event} events]- (if-not timestamp- event- (-> (assoc event :timestamp (t/instant timestamp))- (dissoc :__time))))})--;; SELECT queries are no longer supported as of Druid 0.17.0, and no longer generated by Metabase as of 0.35.0, but if-;; someone has a native select query on Druid < 0.17.0 we can continue to parse it the way we did in the past.-;;-;; We can remove this at some point in the future when everyone is using Druid 0.17.0 or above.-(defmethod post-process ::druid.qp/select- [_ projections _ [{{:keys [events]} :result}]]- {:projections projections- :results (for [event (map :event events)]- (update event :timestamp u.date/parse))})--(defmethod post-process ::druid.qp/total- [_ projections _ results]- {:projections projections- :results (map :result results)})--(defmethod post-process ::druid.qp/topN- [_ projections {:keys [middleware]} results]- {:projections projections- :results (let [results (-> results first :result)]- (if (:format-rows? middleware true)- results- (map #(m/update-existing % :timestamp u.date/parse) results)))})--(defmethod post-process ::druid.qp/groupBy- [_ projections {:keys [middleware]} results]- {:projections projections- :results (if (:format-rows? middleware true)- (map :event results)- (map (comp #(m/update-existing % :timestamp u.date/parse)- :event)- results))})--(defmethod post-process ::druid.qp/timeseries- [_ projections {:keys [middleware]} results]- {:projections (conj projections :timestamp)- :results (let [ts-getter (if (:format-rows? middleware true)- :timestamp- (comp u.date/parse :timestamp))]- (for [event results]- (merge {:timestamp (ts-getter event)} (:result event))))})--(mu/defn- col-names->getter-fns :- [:sequential [:or :keyword fn?]]- "Given a sequence of `columns` keywords, return a sequence of appropriate getter functions to get values from a single- result row. Normally, these are just the keyword column names themselves, but for `:timestamp___int`, we'll also- parse the result as an integer (for further explanation, see the docstring for- `units-that-need-post-processing-int-parsing`). We also round `:distinct___count` in order to return an integer- since Druid returns the approximate floating point value for cardinality queries (See Druid documentation regarding- cardinality and HLL)."- [actual-col-names :- [:sequential :keyword]- annotate-col-names :- [:sequential :keyword]]- (let [annotate-col-names (set annotate-col-names)]- (filter- some?- (for [k actual-col-names]- (case k- :distinct___count (fn get-distinct-count [row]- (some-> (get row :distinct___count) math/round))- :timestamp___int (fn get-timestamp-int [row]- (some-> (get row :timestamp___int) Integer/parseInt))- (when (contains? annotate-col-names k)- k))))))--(defn- result-metadata [col-names]- ;; rename any occurrences of `:timestamp___int` to `:timestamp` in the results so the user doesn't know about- ;; our behind-the-scenes conversion and apply any other post-processing on the value such as parsing some- ;; units to int and rounding up approximate cardinality values.- (let [fixed-col-names (for [col-name col-names]- (case col-name- :timestamp___int :timestamp- :distinct___count :count- col-name))]- {:cols (vec (for [col-name fixed-col-names]- {:name (u/qualified-name col-name)- ;; Placeholder so metadata is well formed, we'll infer actual types later- :base_type :type/*}))}))--(defn- result-rows [{rows :results, :as results} actual-col-names annotate-col-names]- (let [getters (vec (col-names->getter-fns actual-col-names annotate-col-names))]- (when-not (seq getters)- (throw (ex-info (tru "Don''t know how to retrieve results for columns {0}" (pr-str actual-col-names))- {:type driver-api/qp.error-type.driver- :results results})))- (map (apply juxt getters) rows)))--(defn- remove-bonus-keys- "Remove keys that start with `___` from the results -- they were temporary, and we don't want to return them."- [columns]- (vec (remove #(re-find #"^___" (name %)) columns)))--(defn- reduce-results- [{{:keys [mbql?]} :native, :as outer-query} {:keys [projections], :as result} respond]- (let [col-names (if mbql?- (->> projections- remove-bonus-keys- vec)- (-> result :results first keys))- metadata (result-metadata col-names)- annotate-col-names (map (comp keyword :name) (driver-api/merged-column-info outer-query (when-not mbql?- metadata)))- rows (result-rows result col-names annotate-col-names)- base-types (transduce identity (driver-api/base-type-inferer metadata) rows)- metadata (update metadata :cols (partial map (fn [col base-type]- (assoc col :base_type base-type)))- base-types)]- (respond metadata rows)))--(defn execute-reducible-query- "Execute a query for a Druid DB."- [execute*- {{:keys [query query-type projections]} :native- middleware :middleware- :as mbql-query}- respond]- {:pre [query]}- (let [database (-> (driver-api/metadata-provider) driver-api/database)- details (driver.conn/effective-details database)- query (if (string? query)- (json/decode+kw query)- query)- query-type (or query-type- (keyword (namespace ::druid.qp/query) (name (:queryType query))))- _ (driver.conn/track-connection-acquisition! details)- results (try- (execute* details query)- (catch Throwable e- (throw (ex-info (tru "Error executing query: {0}" (ex-message e))- {:type driver-api/qp.error-type.db- :query query}- e))))- result (try (post-process query-type projections- {:timezone (resolve-timezone mbql-query)- :middleware middleware}- results)- (catch Throwable e- (throw (ex-info (tru "Error post-processing Druid query results")- {:type driver-api/qp.error-type.driver- :results results}- e))))]- (try- (reduce-results mbql-query result respond)- (catch Throwable e- (throw (ex-info (tru "Error reducing Druid query results")- {:type driver-api/qp.error-type.driver- :results results- :post-processed result}- e))))))diff --git a/modules/drivers/druid/src/metabase/driver/druid/js.clj b/modules/drivers/druid/src/metabase/driver/druid/js.cljdeleted file mode 100644index 6f83e63499b..00000000000--- a/modules/drivers/druid/src/metabase/driver/druid/js.clj+++ /dev/null@@ -1,67 +0,0 @@-(ns metabase.driver.druid.js- "Util fns for creating Javascript functions."- (:refer-clojure :exclude [+ - * / or])- (:require- [clojure.string :as str]))--(defn- ->js [x]- {:pre [(not (coll? x))]}- (if (keyword? x)- (name x)- x))--(defn parens- "Wrap `s` (presumably a string) in parens."- ^String [s]- (str "(" s ")"))--(defn argslist- "Generate a list of arguments, e.g. for a function declaration or function call.- `args` are separated by commas and wrapped in parens.-- (argslist [:x :y]) -> \"(x, y)\""- ^String [args]- (parens (str/join ", " (map ->js args))))--(defn return- "Generate a javascript `return` statement. `statement-parts` are combined directly into a single string.-- (return :x :+ :y) -> \"return x+y;\""- ^String [& statement-parts]- (str "return " (apply str (map ->js statement-parts)) ";"))--(defn function- "Create a JavaScript function with `args` and `body`."- ^String [args & body]- (str "function" (argslist args) " { "- (apply str body)- " }"))--(defn- arithmetic-operator- "Interpose arithmetic `operator` between `args`, and wrap the entire expression in parens."- ^String [operator & args]- (parens (str/join (str " " (name operator) " ")- (map ->js args))))--(def ^{:arglists '([& args])} ^String + "Interpose `+` between `args`, and wrap the entire expression in parens." (partial arithmetic-operator :+))-(def ^{:arglists '([& args])} ^String - "Interpose `-` between `args`, and wrap the entire expression in parens." (partial arithmetic-operator :-))-(def ^{:arglists '([& args])} ^String * "Interpose `*` between `args`, and wrap the entire expression in parens." (partial arithmetic-operator :*))-(def ^{:arglists '([& args])} ^String / "Interpose `/` between `args`, and wrap the entire expression in parens." (partial arithmetic-operator :/))--(defn fn-call- "Generate a JavaScript function call.-- (fn-call :parseFloat :x :y) -> \"parseFloat(x, y)\""- ^String [fn-name & args]- (str (name fn-name) (argslist args)))--(def ^{:arglists '([x])} ^String parse-float- "Generate a call to the JavaScript `parseFloat` function."- (partial fn-call :parseFloat))--(defn or- "Interpose the JavaScript or operator (`||`) between `args`, and wrap the entire expression in parens.-- (or :x :y) -> \"(x || y)\""- ^String [& args]- (parens (str/join " || " (map ->js args))))diff --git a/modules/drivers/druid/src/metabase/driver/druid/query_processor.clj b/modules/drivers/druid/src/metabase/driver/druid/query_processor.cljdeleted file mode 100644index 28b294c2795..00000000000--- a/modules/drivers/druid/src/metabase/driver/druid/query_processor.clj+++ /dev/null@@ -1,1216 +0,0 @@-(ns metabase.driver.druid.query-processor- (:refer-clojure :exclude [every? mapv some get-in])- (:require- [clojure.string :as str]- [metabase.driver-api.core :as driver-api]- [metabase.driver.common :as driver.common]- [metabase.driver.druid.js :as druid.js]- [metabase.lib.core :as lib]- [metabase.util :as u]- [metabase.util.date-2 :as u.date]- [metabase.util.i18n :refer [tru]]- [metabase.util.log :as log]- [metabase.util.malli :as mu]- [metabase.util.match :as match]- [metabase.util.performance :refer [every? get-in mapv some]]))--(set! *warn-on-reflection* true)--(def ^:private ^:const topN-max-results- "Maximum number of rows the topN query in Druid should return. Huge values cause significant issues with the engine.-- Coming from the default value hardcoded in the Druid engine itself- http://druid.io/docs/latest/querying/topnquery.html"- 1000)--;; +-----> ::scan +----> :groupBy-;; ::query ----| |-;; +----> ::ag-query ----+----> ::topN-;; | +----> total-;; +----> ::timeseries ----|-;; +----> grouped-timeseries--(derive ::scan ::query)-(derive ::ag-query ::query)-(derive ::topN ::ag-query)-(derive ::groupBy ::ag-query)-(derive ::timeseries ::ag-query)-(derive ::total ::timeseries)-(derive ::grouped-timeseries ::timeseries)--(def ^:private ^:dynamic *query*- "The INNER part of the query currently being processed.- (`:settings` is merged in from the outer query as well so we can access timezone info)."- nil)--(defn- query-type-dispatch-fn- [query-type & _]- query-type)--(defmulti ^:private ->rvalue- "Convert something to an 'rvalue`, i.e. a value that could be used in the right-hand side of an assignment expression.-- (let [x 100] ...) ; x is the lvalue; 100 is the rvalue"- {:arglists '([x])}- driver-api/dispatch-by-clause-name-or-class)--(defmethod ->rvalue nil- [_]- nil)--(defmethod ->rvalue Object- [this]- this)--(defn- ag-clause->rvalue [[ag-type :as ag]]- (cond- (= [:count] ag)- :count-- (= ag-type :distinct)- :distinct___count-- (= ag-type :aggregation-options)- (let [[_ wrapped-ag options] ag]- (or (:name options) (recur wrapped-ag)))-- ag-type- ag-type-- :else- (throw (ex-info (tru "Unknown aggregation type!") {:aggregation ag}))))--(defmethod ->rvalue :aggregation- [[_ index]]- (ag-clause->rvalue (or (nth (:aggregation *query*) index)- (throw (ex-info (tru "No aggregation at index {0}" index)- {:index index, :query *query*})))))--(defmethod ->rvalue :field- [[_ id-or-name]]- (if (integer? id-or-name)- (:name (driver-api/field (driver-api/metadata-provider) id-or-name))- id-or-name))--(defmethod ->rvalue :absolute-datetime- [[_ t unit]]- (u.date/format- (if (= unit :default)- t- (u.date/truncate t unit))))--;; TODO - not 100% sure how to handle times here, just treating it exactly like a date will have to do for now-(defmethod ->rvalue :time- [[_ t unit]]- (u.date/format (u.date/truncate t unit)))--(defmethod ->rvalue :relative-datetime- [[_ amount unit]]- (u.date/format (u.date/truncate (u.date/add unit amount) unit)))--(defmethod ->rvalue :value- [[_ value]]- (->rvalue value))--(defmulti ^:private dimension-or-metric?- "Is this field clause a `:dimension` or `:metric`?"- {:arglists '([field-clause])}- driver-api/dispatch-by-clause-name-or-class)--(defmethod dimension-or-metric? :field- [[_ id-or-name options]]- (let [{:keys [base-type database-type]} (if (integer? id-or-name)- (driver-api/field (driver-api/metadata-provider) id-or-name)- options)]- (cond- (str/includes? database-type "[metric]") :metric- (isa? base-type :type/DruidHyperUnique) :metric- :else :dimension)))--(defn- ^:dynamic *random-query-id* []- (str (random-uuid)))--(defn- query-type->default-query [query-type]- (merge- {:intervals ["1900-01-01/2100-01-01"]- :granularity :all- :context {:queryId (*random-query-id*)}}- (case query-type- ::scan {:queryType :scan- :limit driver-api/absolute-max-results}- ::total {:queryType :timeseries}- ::grouped-timeseries {:queryType :timeseries}- ::topN {:queryType :topN- :threshold topN-max-results}- ::groupBy {:queryType :groupBy})))--;;; ---------------------------------------------- handle-source-table -------------------------------------------------(defn- handle-source-table- [_query-type {source-table-id :source-table} druid-query]- (let [{source-table-name :name} (driver-api/table (driver-api/metadata-provider) source-table-id)]- (assoc-in druid-query [:query :dataSource] source-table-name)))--;;; ---------------------- handle-filter. See http://druid.io/docs/latest/querying/filters.html ------------------------(def ^:private ^{:arglists '([clause])} field?- (partial driver-api/is-clause? :field))--(defn- filter:and- [filters]- {:type :and- :fields filters})--(defn- filter:not- [filtr]- {:pre [filtr]}- (if (= (:type filtr) :not) ; it looks like "two nots don't make an identity" with druid- (:field filtr)- {:type :not, :field filtr}))--(defn- filter:=- [field value]- {:type :selector- :dimension (->rvalue field)- :value (->rvalue value)})--(defn- filter:nil?- [clause-or-field]- (if (driver-api/is-clause? #{:+ :- :/ :*} clause-or-field)- (filter:and (vec (for [arg (rest clause-or-field)- :when (field? arg)]- (filter:nil? arg))))- (filter:= clause-or-field (case (dimension-or-metric? clause-or-field)- :dimension nil- :metric 0))))--(defn- filter:like- "Build a `like` filter clause, which is almost just like a SQL `LIKE` clause."- [field pattern case-sensitive?]- {:type :like- :dimension (->rvalue field)- ;; tell Druid to use backslash as an escape character- :escape "\\"- ;; if this is a case-insensitive search we'll lower-case the search pattern and add an extraction function to- ;; lower-case the dimension values we're matching against- :pattern (cond-> pattern- (not case-sensitive?) u/lower-case-en)- :extractionFn (when-not case-sensitive?- {:type :lower})})--(defn- escape-like-filter-pattern- "Escape `%`, `_`, and backslash symbols that aren't meant to have special meaning in `like` filters- patterns. Backslashes wouldn't normally have a special meaning, but we specify backslash as our escape character in- the `filter:like` function above, so they need to be escaped as well."- [s]- (str/replace s #"([%_\\])" "\\\\$1"))--(defn- filter:bound- "Numeric `bound` filter, for finding values of `field` that are less than some value-or-field, greater than some- value-or-field, or both. Defaults to being `inclusive` (e.g. `<=` instead of `<`) but specify option `inclusive?` to- change this."- [field & {:keys [lower upper inclusive?]- :or {inclusive? true}}]- {:type :bound- :ordering :numeric- :dimension (->rvalue field)- :lower (num (->rvalue lower))- :upper (num (->rvalue upper))- :lowerStrict (not inclusive?)- :upperStrict (not inclusive?)})--(defmulti ^:private parse-filter*- "Parse an MBQL `filter-clause` and generate an appropriate Druid filter map.-- (parse-filter* [:= [:field 1 nil] 2]) ; -> {:type :selector, :dimension \"venue_price\", :value 2}"- {:arglists '([filter-clause])}- driver-api/dispatch-by-clause-name-or-class)--(defmethod parse-filter* nil- [_]- nil)--(defmethod parse-filter* :between- [[_ field min-val max-val]]- (filter:bound field, :lower min-val, :upper max-val))--(defmethod parse-filter* :contains- [[_ field pattern options]]- (if (and (sequential? pattern) (= :value (first pattern)))- {:type :search- :dimension (->rvalue field)- :query {:type :contains- :value (->rvalue pattern)- :caseSensitive (get options :case-sensitive true)}}- (throw (ex-info (tru "Dynamic patterns are not supported.")- {:type driver-api/qp.error-type.invalid-query- :field field :pattern pattern :options options}))))--(defmethod parse-filter* :starts-with- [[_ field pattern options]]- (if (and (sequential? pattern) (= :value (first pattern)))- (filter:like field- (str (escape-like-filter-pattern (->rvalue pattern)) \%)- (get options :case-sensitive true))- (throw (ex-info (tru "Dynamic patterns are not supported.")- {:type driver-api/qp.error-type.invalid-query- :field field :pattern pattern :options options}))))--(defmethod parse-filter* :ends-with- [[_ field pattern options]]- (if (and (sequential? pattern) (= :value (first pattern)))- (filter:like field- (str \% (escape-like-filter-pattern (->rvalue pattern)))- (get options :case-sensitive true))- (throw (ex-info (tru "Dynamic patterns are not supported.")- {:type driver-api/qp.error-type.invalid-query- :field field :pattern pattern :options options}))))--(defmethod parse-filter* :=- [[_ field value-or-field]]- (filter:= field value-or-field))--(defmethod parse-filter* :!=- [[_ field value-or-field]]- (filter:not (filter:= field value-or-field)))--(defmethod parse-filter* :<- [[_ field value-or-field]]- (filter:bound field, :upper value-or-field, :inclusive? false))--(defmethod parse-filter* :>- [[_ field value-or-field]]- (filter:bound field, :lower value-or-field, :inclusive? false))--(defmethod parse-filter* :<=- [[_ field value-or-field]]- (filter:bound field, :upper value-or-field))--(defmethod parse-filter* :>=- [[_ field value-or-field]]- (filter:bound field, :lower value-or-field))--(defmethod parse-filter* :and- [[_ & args]]- (when-let [fields (seq (keep identity (map parse-filter* args)))]- {:type :and, :fields (vec fields)}))--(defmethod parse-filter* :or- [[_ & args]]- (when-let [fields (seq (keep identity (map parse-filter* args)))]- {:type :or, :fields (vec fields)}))--(defmethod parse-filter* :not- [[_ subclause]]- (when-let [subclause (parse-filter* subclause)]- (filter:not subclause)))--(defn- parse-filter [filter-clause]- ;; strip out all the filters against temporal fields. Those are handled separately, as intervals- (-> (match/replace filter-clause- [_ [:field _ {:temporal-unit &truthy}] & _]- nil)- lib/->mbql5- lib/simplify-compound-filter- lib/->legacy-MBQL- parse-filter*))--(mu/defn- add-datetime-units* :- driver-api/mbql.schema.DateTimeValue- "Return a `relative-datetime` clause with `n` units added to it."- [absolute-or-relative-datetime :- driver-api/mbql.schema.DateTimeValue- n :- number?]- (if (driver-api/is-clause? :relative-datetime absolute-or-relative-datetime)- (let [[_ original-n unit] absolute-or-relative-datetime]- [:relative-datetime (+ n original-n) unit])- (let [[_ t unit] absolute-or-relative-datetime]- [:absolute-datetime (u.date/add t unit n) unit])))--(defn- add-datetime-units- "Adding `n` `:default` units doesn't make sense. So if an `:absoulte-datetime` has `:default` as its unit, add `n`- milliseconds, because that is the smallest unit Druid supports."- [clause n]- (match/replace clause- [:absolute-datetime t :default]- [:absolute-datetime (u.date/add t :millisecond n) :millisecond]-- _- (add-datetime-units* clause n)))--(defn- ->absolute-timestamp ^java.time.temporal.Temporal [clause]- (match/match-one clause- [:absolute-datetime t :default]- t-- [:absolute-datetime t unit]- (u.date/truncate t unit)-- [:relative-datetime amount unit]- (u.date/truncate (u.date/add unit amount) unit)-- _- nil))--(defmulti ^:private filter-clause->intervals- "Generate query intervals as appropriate from a `filter-clause` containing a temporal `:field`. `:intervals` are- specified separately from other things we think of as filter clauses in Druid. For temporal filter clauses, this- returns a sequence of min/max datetime tuples; like `[#t 2019-01-01 #t 2019-10-01]`; for irrelevant filter- clauses, the methods are skipped entirely."- {:arglists '([filter-clause])}- (fn [filter-clause]- (when (match/match-one filter-clause [:field _ {:temporal-unit &truthy}] true)- (driver-api/dispatch-by-clause-name-or-class filter-clause))))--(defmethod filter-clause->intervals :default- [_]- nil)--;; BETWEEN "2015-12-09", "2015-12-11" -> ["2015-12-09/2015-12-12"], because BETWEEN is inclusive-(defmethod filter-clause->intervals :between- [[_ _ min-value max-value]]- [[(->absolute-timestamp min-value) (->absolute-timestamp (add-datetime-units max-value 1))]])--(defmethod filter-clause->intervals :=- [[_ _ v]]- [[(->absolute-timestamp v) (->absolute-timestamp (add-datetime-units v 1))]])--(defmethod filter-clause->intervals :!=- [[_ _ v]]- [[nil (->absolute-timestamp v)] [(->absolute-timestamp (add-datetime-units v 1)) nil]])--(defmethod filter-clause->intervals :>- [[_ _ v]]- [[(->absolute-timestamp (add-datetime-units v 1)) nil]])--(defmethod filter-clause->intervals :>=- [[_ _ v]]- [[(->absolute-timestamp v) nil]])--(defmethod filter-clause->intervals :<- [[_ _ v]]- [[nil (->absolute-timestamp v)]])--(defmethod filter-clause->intervals :<=- [[_ _ v]]- [[nil (->absolute-timestamp (add-datetime-units v 1))]])--;; When you're anding together multiple intervals we have to combine them into a single interval that is the-;; logical equivalent of all the intervals. e.g.-;;-;; `[:and [:>= x 2018] [:< x 2019]]` should get converted to a `2018/2019` interval.-(defn- combine-intervals [[min-1 max-1] [min-2 max-2]]- (let [datetime-max (fn [x y] (if (pos? (compare x y)) x y))- datetime-min (fn [x y] (if (neg? (compare x y)) x y))]- [(if (and min-1 min-2)- (datetime-max min-1 min-2)- (or min-1 min-2))- (if (and max-1 max-2)- (datetime-min max-1 max-2)- (or max-1 max-2))]))--(defmethod filter-clause->intervals :and- [[_ & subclauses]]- (let [subclause-intervals (map filter-clause->intervals subclauses)- flattened-subclause-intervals (apply concat (filter #(= (count %) 1) subclause-intervals))]- ;; log a warning about all the intervals we filtered out above- (doseq [intervals subclause-intervals- :when (> (count intervals) 1)]- (log/warnf "WARNING: Don't know how to combine these intervals into a single interval.\nIgnoring intervals: %s"- intervals))- (reduce- (fn [[acc] interval]- [(combine-intervals acc interval)])- nil- flattened-subclause-intervals)))--(defmethod filter-clause->intervals :or- [[_ & subclauses]]- (mapcat filter-clause->intervals subclauses))--(defmethod filter-clause->intervals :not- [[_ subclause]]- ;; first, check and see if the subclause is actually something that will produce intervals (i.e., if it is a- ;; temporal filter). If it is, then negate the logic and use the intervals for that. We don't want to call negate- ;; without checking first because some filters like string `:contains` can't be negated without using a `:not`- ;; filter and we don't want to stack overflow- (when (seq (filter-clause->intervals subclause))- (-> subclause- lib/->mbql5- lib/negate-boolean-expression- lib/->legacy-MBQL- filter-clause->intervals)))--(defn- compile-intervals- "Compile the interval pairs generated by `filter-clause->intervals` into the format expected by Druid (`min/max`- strings)."- [intervals]- (when-let [intervals (seq (filter some? intervals))]- (vec (for [[min-value max-value] intervals]- (format "%s/%s"- (or (some-> min-value u.date/format) "-5000")- (or (some-> max-value u.date/format) "5000"))))))--(defn- handle-filter- [_ {filter-clause :filter} druid-query]- (if-not filter-clause- druid-query- (let [filter (parse-filter filter-clause)- intervals (compile-intervals (filter-clause->intervals filter-clause))]- (cond-> druid-query- (seq filter) (assoc-in [:query :filter] filter)- (seq intervals) (assoc-in [:query :intervals] intervals)))))--;;; ----------------------------------------------- handle-aggregation -------------------------------------------------(defn- expression->field-names- [[_ & args]]- {:post [(every? (some-fn keyword? string?) %)]}- (flatten (for [arg args- :when (or (field? arg)- (driver-api/is-clause? #{:+ :- :/ :*} arg))]- (cond- (driver-api/is-clause? #{:+ :- :/ :*} arg) (expression->field-names arg)- (field? arg) (->rvalue arg)))))--(defn- expression-arg->js- [arg default-value]- (if-not (field? arg)- arg- (druid.js/or (druid.js/parse-float (->rvalue arg))- default-value)))--(defn- expression->js- [[operator & args] default-value]- (apply (case operator- :+ druid.js/+- :- druid.js/-- :* druid.js/*- :/ druid.js//)- (for [arg args]- (expression-arg->js arg default-value))))--(defn- ag:doubleSum:expression- [[operator :as expression] output-name]- (let [field-names (expression->field-names expression)]- {:type :javascript- :name output-name- :fieldNames field-names- :fnReset (druid.js/function []- (druid.js/return 0))- :fnAggregate (druid.js/function (cons :current field-names)- (druid.js/return (druid.js/+ :current (expression->js expression (if (= operator :/) 1 0)))))- :fnCombine (druid.js/function [:x :y]- (druid.js/return (druid.js/+ :x :y)))}))--(defn- ag:doubleSum- [field-clause output-name]- (if (driver-api/is-clause? #{:+ :- :/ :*} field-clause)- (ag:doubleSum:expression field-clause output-name)- ;; metrics can use the built-in :doubleSum aggregator, but for dimensions we have to roll something that does the- ;; same thing in JS- (case (dimension-or-metric? field-clause)- :metric {:type :doubleSum- :name output-name- :fieldName (->rvalue field-clause)}- :dimension {:type :javascript- :name output-name- :fieldNames [(->rvalue field-clause)]- :fnReset "function() { return 0 ; }"- :fnAggregate "function(current, x) { return current + (parseFloat(x) || 0); }"- :fnCombine "function(x, y) { return x + y; }"})))--(defn- ag:doubleMin:expression- [expression output-name]- (let [field-names (expression->field-names expression)]- {:type :javascript- :name output-name- :fieldNames field-names- :fnReset (druid.js/function []- (druid.js/return "Number.MAX_VALUE"))- :fnAggregate (druid.js/function (cons :current field-names)- (druid.js/return (druid.js/fn-call :Math.min :current- (expression->js expression :Number.MAX_VALUE))))- :fnCombine (druid.js/function [:x :y]- (druid.js/return (druid.js/fn-call :Math.min :x :y)))}))--(defn- ag:doubleMin- [field-clause output-name]- (if (driver-api/is-clause? #{:+ :- :/ :*} field-clause)- (ag:doubleMin:expression field-clause output-name)- (case (dimension-or-metric? field-clause)- :metric {:type :doubleMin- :name output-name- :fieldName (->rvalue field-clause)}- :dimension {:type :javascript- :name output-name- :fieldNames [(->rvalue field-clause)]- :fnReset "function() { return Number.MAX_VALUE ; }"- :fnAggregate "function(current, x) { return Math.min(current, (parseFloat(x) || Number.MAX_VALUE)); }"- :fnCombine "function(x, y) { return Math.min(x, y); }"})))--(defn- ag:doubleMax:expression- [expression output-name]- (let [field-names (expression->field-names expression)]- {:type :javascript- :name output-name- :fieldNames field-names- :fnReset (druid.js/function []- (druid.js/return "Number.MIN_VALUE"))- :fnAggregate (druid.js/function (cons :current field-names)- (druid.js/return (druid.js/fn-call :Math.max :current- (expression->js expression :Number.MIN_VALUE))))- :fnCombine (druid.js/function [:x :y]- (druid.js/return (druid.js/fn-call :Math.max :x :y)))}))--(defn- ag:doubleMax- [field output-name]- (if (driver-api/is-clause? #{:+ :- :/ :*} field)- (ag:doubleMax:expression field output-name)- (case (dimension-or-metric? field)- :metric {:type :doubleMax- :name output-name- :fieldName (->rvalue field)}- :dimension {:type :javascript- :name output-name- :fieldNames [(->rvalue field)]- :fnReset "function() { return Number.MIN_VALUE ; }"- :fnAggregate "function(current, x) { return Math.max(current, (parseFloat(x) || Number.MIN_VALUE)); }"- :fnCombine "function(x, y) { return Math.max(x, y); }"})))--(defn- ag:filtered- [filtr aggregator]- {:pre [(map? filtr)]}- {:type :filtered, :filter filtr, :aggregator aggregator})--(defn- hyper-unique?- [[_ field-id]]- {:pre [(pos-int? field-id)]}- (isa? (:base-type (driver-api/field (driver-api/metadata-provider) field-id))- :type/DruidHyperUnique))--(defn- ag:distinct- [field output-name]- (cond- (driver-api/is-clause? #{:+} field)- {:type :cardinality- :name output-name- :fieldNames (mapv ->rvalue (rest field))- :byRow true- :round true}- (hyper-unique? field)- {:type :hyperUnique- :name output-name- :fieldName (->rvalue field)}- :else- {:type :cardinality- :name output-name- :fieldNames [(->rvalue field)]- :byRow true- :round true}))--(defn- ag:count- ([output-name]- {:type :count, :name output-name})- ([field output-name]- (if (and (driver-api/is-clause? #{:field-id} field)- (hyper-unique? field))- {:type :hyperUnique- :name output-name- :fieldName (->rvalue field)}- (ag:filtered (filter:not (filter:nil? field)) (ag:count output-name)))))--(defn- ag:countWhere- [pred output-name]- (ag:filtered (parse-filter pred) (ag:count output-name)))--(defn- ag:sumWhere- [field pred output-name]- (ag:filtered (parse-filter pred) (ag:doubleSum field output-name)))--(def ^:private ^{:arglists '([prefix])} genname- (comp name gensym))--(defn- create-aggregation-clause- [output-name ag-type ag-field args]- (let [output-name-kwd (keyword output-name)]- (match/match-one [ag-type ag-field]- ;; For 'distinct values' queries (queries with a breakout by no aggregation) just aggregate by count, but name- ;; it :___count so it gets discarded automatically- [nil nil] [[(or output-name-kwd :___count)] {:aggregations [(ag:count (or output-name :___count))]}]-- [:count nil] [[(or output-name-kwd :count)] {:aggregations [(ag:count (or output-name :count))]}]-- [:count _] [[(or output-name-kwd :count)] {:aggregations [(ag:count ag-field (or (name output-name) :count))]}]-- [:avg _] (let [count-name (genname "___count_")- sum-name (genname "___sum_")]- [[(keyword count-name) (keyword sum-name) (or output-name-kwd :avg)]- {:aggregations [(ag:count ag-field count-name)- (ag:doubleSum ag-field sum-name)]- :postAggregations [{:type :arithmetic- :name (or output-name :avg)- :fn :/- :fields [{:type :fieldAccess, :fieldName sum-name}- {:type :fieldAccess, :fieldName count-name}]}]}])-- [:sum-where _] (let [[pred] args]- [[(or output-name-kwd :sum-where)]- {:aggregations [(ag:sumWhere ag-field pred output-name-kwd)]}])-- [:count-where _] [[(or output-name-kwd :count-where)]- {:aggregations [(ag:countWhere ag-field output-name-kwd)]}]-- [:share _] (let [total-count-name (genname "___total_count_")- true-count-name (genname "___true_count_")]- [[(keyword total-count-name) (keyword true-count-name) (or output-name-kwd :share)]- {:aggregations [(ag:count total-count-name)- (ag:countWhere ag-field true-count-name)]- :postAggregations [{:type :arithmetic- :name (or output-name :share)- :fn :/- :fields [{:type :fieldAccess, :fieldName true-count-name}- {:type :fieldAccess, :fieldName total-count-name}]}]}])-- [:distinct _] [[(or output-name-kwd :distinct___count)]- {:aggregations [(ag:distinct ag-field (or output-name :distinct___count))]}]- [:sum _] [[(or output-name-kwd :sum)]- {:aggregations [(ag:doubleSum ag-field (or (name output-name) :sum))]}]- [:min _] [[(or output-name-kwd :min)]- {:aggregations [(ag:doubleMin ag-field (or output-name :min))]}]- [:max _] [[(or output-name-kwd :max)]- {:aggregations [(ag:doubleMax ag-field (or output-name :max))]}]- _ (throw (ex-info (str "Unknown aggregation tuple: " [ag-type ag-field]) {})))))--(defn ^:private ^:dynamic *query-unique-name-fn*- [& _]- (throw (ex-info "Must bind metabase.driver.druid.query-processor/*query-unique-name-fn*. See handle-aggregation for an example." {})))--(mu/defn- handle-aggregation- [query-type- ag-clause :- driver-api/mbql.schema.Aggregation- druid-query]- (let [output-name (*query-unique-name-fn* (driver-api/aggregation-name *query* ag-clause))- [ag-type ag-field & args] (match/match-one ag-clause- [:aggregation-options ag & _] (&recur ag)- _ &match)]- (if-not (isa? query-type ::ag-query)- druid-query- (let [[projections ag-clauses] (try- (create-aggregation-clause output-name ag-type ag-field args)- (catch Throwable e- (throw (ex-info (tru "Error creating aggregation clause")- {:type driver-api/qp.error-type.driver- :clause-name output-name- :ag-type ag-type- :ag-field ag-field- :args args}- e))))]- (-> druid-query- (update :projections into projections)- (update :query (partial merge-with concat) ag-clauses))))))--(defn- deduplicate-aggregation-options [expression]- (match/replace expression- [:aggregation-options [:aggregation-options ag options-1] options-2]- [:aggregation-options ag (merge options-1 options-2)]))--(defn- aggregation-unique-identifier [clause]- (format "__%s" (*query-unique-name-fn* (name clause))))--(defn- add-expression-aggregation-output-names- [expression]- (match/replace expression- [:aggregation-options ag options]- (deduplicate-aggregation-options [:aggregation-options (add-expression-aggregation-output-names ag) options])-- [(clause :guard #{:count :avg :distinct :stddev :sum :min :max}) & _]- [:aggregation-options &match {:name (aggregation-unique-identifier clause)}]))--(defn- post-aggregator-type- "Complex aggregators like `cardinality` and ``hyperUnique` (which we use to implement MBQL- `:distinct`) require finalizing their return value.- https://druid.apache.org/docs/latest/querying/post-aggregations.html"- [[op & _]]- (if (= :distinct op)- :finalizingFieldAccess- :fieldAccess))--(defn- expression-post-aggregation- [[operator & args, :as expression]]- (match/match-one expression- ;; If it's a named expression, we want to preserve the included name, so recurse, but merge in the name- [:aggregation-options ag _]- (assoc (expression-post-aggregation ag) :name (driver-api/aggregation-name *query* expression))-- _- {:type :arithmetic- :name (driver-api/aggregation-name *query* expression)- :fn operator- :fields (vec (for [arg args]- (match/match-one arg- (_ :guard number?)- {:type :constant, :name (str &match), :value &match}-- [:aggregation-options ag {:name name}]- {:type (post-aggregator-type ag), :fieldName name}-- [#{:+ :- :/ :*} & _]- (expression-post-aggregation &match)-- ;; we should never get here unless our code is B U S T E D- _- (throw (ex-info (tru "Expected :aggregation-options, constant, or expression.")- {:type :bug, :input arg})))))}))--(declare handle-aggregations)--(defn- expression->actual-ags- "Return a flattened list of actual aggregations that are needed for `expression`."- [[_ & args]]- (into []- (comp (remove number?)- (map (fn [arg]- (if (driver-api/is-clause? #{:+ :- :/ :*} arg)- (expression->actual-ags arg)- [arg])))- cat)- args))--(defn- unwrap-name- [x]- (if (driver-api/is-clause? :aggregation-options x)- (second x)- x))--(defn- handle-expression-aggregation- [query-type expression druid-query]- ;; filter out constants from the args list- (let [expression (add-expression-aggregation-output-names expression)- ;; The QP will automatically add a generated name to the expression, if it's there, unwrap it before looking- ;; for the aggregation- ags (expression->actual-ags (unwrap-name expression))- druid-query (handle-aggregations query-type {:aggregation ags} druid-query)- post-agg (expression-post-aggregation expression)]- (-> druid-query- (update :projections conj (keyword (:name post-agg)))- (update-in [:query :postAggregations] concat [post-agg]))))--(defn- handle-aggregations- [query-type {aggregations :aggregation} druid-query]- (binding [*query-unique-name-fn* (driver-api/unique-name-generator)]- ;; Load unique name generator with projections from breakouts- (doseq [projection (:projections druid-query)]- (*query-unique-name-fn* (name projection)))- (reduce- (fn [druid-query aggregation]- (match/match-one aggregation- [:aggregation-options [#{:+ :- :/ :*} & _] _]- (handle-expression-aggregation query-type &match druid-query)-- [#{:+ :- :/ :*} & _]- (handle-expression-aggregation query-type &match druid-query)-- _- (handle-aggregation query-type aggregation druid-query)))- druid-query- aggregations)))--;;; ------------------------------------------------ handle-breakout ---------------------------------------------------(defmulti ^:private ->dimension-rvalue- "Format `Field` for use in a `:dimension` or `:dimensions` clause."- {:arglists '([field-clause])}- driver-api/dispatch-by-clause-name-or-class)--(defn- extract:timeFormat- "Create a time format extraction. Returns a string. See- http://druid.io/docs/0.9.1.1/querying/dimensionspecs.html#time-format-extraction-function"- [format-str]- {:pre [(string? format-str)]}- {:type :timeFormat- :format format-str- :timeZone (or (get-in *query* [:settings :report-timezone])- "UTC")- :locale "en-US"})--(defn- extract:js- "Create an extraction function from JavaScript -- see- http://druid.io/docs/0.9.1.1/querying/dimensionspecs.html#javascript-extraction-function"- [& function-str-parts]- {:pre [(every? string? function-str-parts)]}- {:type :javascript- :function (str/replace (apply str function-str-parts) #"\s+" " ")})--;; don't try to make this a ^:const map -- extract:timeFormat looks up timezone info at query time-(defn- unit->extraction-fn [unit]- (case unit- :default (extract:timeFormat "yyyy-MM-dd'T'HH:mm:ssZZ")- :minute (extract:timeFormat "yyyy-MM-dd'T'HH:mm:00ZZ")- :minute-of-hour (extract:timeFormat "mm")- :hour (extract:timeFormat "yyyy-MM-dd'T'HH:00:00ZZ")- :hour-of-day (extract:timeFormat "HH")- :day (extract:timeFormat "yyyy-MM-dd'T'00:00:00ZZ")- :day-of-week (extract:js "function (timestamp) {"- " var date = new Date(timestamp);"- (format " var dayOfWeek = (date.getDay() + 1 + %s) %% 7;"- (driver.common/start-of-week-offset :druid))- " return (dayOfWeek == 0) ? 7 : dayOfWeek;"- "}")- :day-of-month (extract:timeFormat "dd")- :day-of-year (extract:timeFormat "DDD")- :week (extract:js "function (timestamp) {"- " var date = new Date(timestamp);"- (format " var firstDOW = new Date(date - ((date.getDay() + %s) * 86400000));"- (driver.common/start-of-week-offset :druid))- " var month = firstDOW.getMonth() + 1;"- " var day = firstDOW.getDate();"- " return '' + firstDOW.getFullYear() + '-' + (month < 10 ? '0' : '') + month + '-' + (day < 10 ? '0' : '') + day;"- "}")- :week-of-year (extract:js "function (timestamp) {"- " var date = new Date(timestamp);"- (format " var firstDOW = new Date(date - ((date.getDay() + %s) * 86400000));"- (driver.common/start-of-week-offset :druid))- " var dayOfYear = (Date.UTC(firstDOW.getFullYear(), firstDOW.getMonth(), firstDOW.getDate()) - Date.UTC(firstDOW.getFullYear(), 0, 0)) / 24 / 60 / 60 / 1000;"- " return Math.floor(dayOfYear / 7) + 1;"- "}")- :month (extract:timeFormat "yyyy-MM-01")- :month-of-year (extract:timeFormat "MM")- :quarter (extract:js "function (timestamp) {"- " var date = new Date(timestamp);"- " var month = date.getMonth() + 1;" ; js months are 0 - 11- " var quarterMonth = month - ((month - 1) % 3);"- " return '' + date.getFullYear() + '-' + (quarterMonth < 10 ? '0' : '') + quarterMonth + '-01';"- "}")- :quarter-of-year (extract:js "function (timestamp) {"- " var date = new Date(timestamp);"- " return Math.floor((date.getMonth() + 3) / 3);"- "}")- :year (extract:timeFormat "yyyy-01-01")))--(defn- unit->granularity- [unit]- (merge {:type "period"- :period (case unit- :minute "PT1M"- :hour "PT1H"- :day "P1D"- :week "P1W"- :month "P1M"- :quarter "P3M"- :year "P1Y")- :timeZone (driver-api/results-timezone-id)}- ;; Druid uses Monday for the start of its weekly calculations. Metabase uses Sundays. When grouping by week,- ;; the origin keypair will use the date specified as it's start of the week. The below date is the first- ;; Sunday after Epoch. The date itself isn't significant, it just uses it to figure out what day it should- ;; start from.- (when (= :week unit)- {:origin "1970-01-04T00:00:00Z"})))--(def ^:private units-that-need-post-processing-int-parsing- "`extract:timeFormat` always returns a string; there are cases where we'd like to return an integer tead, such as- `:day-of-month`. There's no simple way to do this in Druid -- Druid 0.9.0+ *does* let you combine extraction- functions with `:cascade`, but we're still supporting 0.8.x. Instead, we will perform the conversions in- Clojure-land during post-processing. If we need to perform the extra post-processing step, we'll name the resulting- column `:timestamp___int`; otherwise we'll keep the name `:timestamp`."- #{:minute-of-hour- :hour-of-day- :day-of-week- :day-of-month- :day-of-year- :week-of-year- :month-of-year- :quarter-of-year})--(defmethod ->dimension-rvalue nil- [_]- (->rvalue nil))--(defmethod ->dimension-rvalue Object- [this]- (->rvalue this))--(defn- temporal-dimension-rvalue [unit]- {:type :extraction- :dimension :__time- ;; :timestamp is a special case, and we need to do an 'extraction' agat the secret special value :__time to get- ;; at it- :outputName (if (contains? units-that-need-post-processing-int-parsing unit)- :timestamp___int- :timestamp)- :extractionFn (unit->extraction-fn unit)})--(defmethod ->dimension-rvalue :field- [[_ _ {:keys [base-type temporal-unit]} :as clause]]- (if (or temporal-unit- (isa? base-type :type/Temporal))- (temporal-dimension-rvalue (or temporal-unit :default))- (->rvalue clause)))--(defmulti ^:private handle-breakout- {:arglists '([query-type original-query druid-query])}- query-type-dispatch-fn)--;; only topN , grouped-timeseries & groupBy handle breakouts-(defmethod handle-breakout ::query- [_ _ druid-query]- druid-query)--(defmethod handle-breakout ::grouped-timeseries- [_ {[breakout-field] :breakout} druid-query]- (assoc-in druid-query [:query :granularity] (unit->granularity (:unit breakout-field))))--(defn- field-clause->name- [field-clause]- (match/match-one field-clause- [:field (id :guard integer?) _]- (:name (driver-api/field (driver-api/metadata-provider) id))-- [:field (field-name :guard string?) _]- field-name))--(defmethod handle-breakout ::topN- [_ {[breakout-field] :breakout} druid-query]- (let [dim-rvalue (->dimension-rvalue breakout-field)]- (-> druid-query- (update :projections conj (keyword (if (and (map? dim-rvalue)- (contains? dim-rvalue :outputName))- (:outputName dim-rvalue)- (field-clause->name breakout-field))))- (assoc-in [:query :dimension] dim-rvalue))))--(defmethod handle-breakout ::groupBy- [_ {breakout-fields :breakout} druid-query]- (-> druid-query- (update :projections into (for [breakout-field breakout-fields]- (let [dim-rvalue (->dimension-rvalue breakout-field)]- (keyword- (if (and (map? dim-rvalue)- (contains? dim-rvalue :outputName))- (:outputName dim-rvalue)- (field-clause->name breakout-field))))))- (assoc-in [:query :dimensions] (mapv ->dimension-rvalue breakout-fields))))--;;; ------------------------------------------------ handle-order-by ---------------------------------------------------(defmulti ^:private handle-order-by- {:arglists '([query-type original-query druid-query])}- query-type-dispatch-fn)--(defmethod handle-order-by ::query- [_ _ druid-query]- (log/warn- (u/format-color- 'red- "Sorting with Druid is only allowed in queries that have one or more breakout columns. Ignoring :order-by clause."))- druid-query)--(defmethod handle-order-by ::topN- [_ {[ag] :aggregation, [breakout-field] :breakout, [[direction field]] :order-by} druid-query]- (let [field (->rvalue field)- breakout-field (->rvalue breakout-field)- sort-by-breakout? (= field breakout-field)- ag-field (match/match-one ag- [:distinct & _] :distinct___count- [:aggregation-options _ {:name name}] name- [:aggregation-options wrapped-ag _] (&recur wrapped-ag)- [(ag-type :guard keyword?) & _] ag-type)]- (when-not sort-by-breakout?- (assert ag-field))- (assoc-in druid-query [:query :metric] (match/match-one [sort-by-breakout? direction]- [true :asc] {:type :alphaNumeric}- [true :desc] {:type :inverted, :metric {:type :alphaNumeric}}- [false :asc] {:type :inverted, :metric ag-field}- [false :desc] ag-field))))--(defmethod handle-order-by ::groupBy- [_ {:keys [order-by]} druid-query]- (assoc-in druid-query [:query :limitSpec :columns] (vec (for [[direction field] order-by]- {:dimension (->rvalue field)- :direction (case direction- :desc :descending- :asc :ascending)}))))-(defn- temporal-field?- "Similar to `types/temporal-field?` but works on field ids wrapped in a datetime or on fields that happen to be a- datetime"- [field]- (when field- (match/match-one field- [:field _id-or-name {:temporal-unit &truthy}]- true-- [:field (id :guard pos-int?) _opts]- (driver-api/temporal? (driver-api/field (driver-api/metadata-provider) id)))))--;; Handle order by timestamp field-(defmethod handle-order-by ::grouped-timeseries- [_ {[[direction field]] :order-by} druid-query]- (let [can-sort? (if (temporal-field? field)- true- (log/warn "grouped timeseries queries can only be sorted by the 'timestamp' column."))]- (cond-> druid-query- can-sort? (assoc-in [:query :descending] (= direction :desc)))))--(defmethod handle-order-by ::scan- [_ {[[direction field]] :order-by, fields :fields} druid-query]- (let [can-sort? (cond- (not (some temporal-field? fields))- (log/warn "scan queries can only be sorted if they include the 'timestamp' column.")-- (not (temporal-field? field))- (log/warn "scan queries can only be sorted by the 'timestamp' column.")-- :else- true)]- (cond-> druid-query- can-sort? (assoc-in [:query :order] (case direction- :desc :descending- :asc :ascending)))))--;;; ------------------------------------------------- handle-fields ----------------------------------------------------(defmulti ^:private handle-fields- {:arglists '([query-type original-query druid-query])}- query-type-dispatch-fn)--(defmethod handle-fields ::query- [_ {fields :fields} druid-query]- (when fields- (log/warn- (u/format-color 'red- ;; TODO - this is not really true, is it- "WARNING: It only makes sense to specify :fields for a query with no aggregation. Ignoring the clause.")))- druid-query)--(defmethod handle-fields ::scan- [_ {fields :fields} druid-query]- (transduce- identity- (fn- ([druid-query]- ;; If you specify nil or empty `:columns` Druid will just return all of the ones available. In cases where- ;; we don't want anything to be returned in one or the other, we'll ask for a `:___dummy` column instead.- ;; Druid happily returns `nil` for the column in every row, and it will get auto-filtered out of the results- ;; so the User will never see it.- (update-in druid-query [:query :columns] #(or (seq %) [:___dummy])))-- ([druid-query field]- (if (and (temporal-field? field)- (= (keyword (field-clause->name field)) :timestamp))- (-> druid-query- (update :projections conj :timestamp)- (update-in [:query :columns] conj :__time))- (-> druid-query- (update :projections conj (keyword (field-clause->name field)))- (update-in [:query :columns] conj (->rvalue field))))))- druid-query- fields))--;;; -------------------------------------------------- handle-limit ----------------------------------------------------(defmulti ^:private handle-limit- {:arglists '([query-type original-query druid-query])}- query-type-dispatch-fn)--(defn- adjust-limit- "No joke, Druid queries do not work if limit is `1048575`, but they work if limit is `1048576`. They fail with an- 'Invalid type marker byte 0x3c for expected value token` error. So if we see the updated `absolute-max-results` from- #15414, adjust it back to the old known working value. had to work around."- [limit]- (cond-> limit- (= limit driver-api/absolute-max-results) inc))--(defmethod handle-limit ::scan- [_ {limit :limit} druid-query]- (cond-> druid-query- limit (assoc-in [:query :limit] (adjust-limit limit))))--(defmethod handle-limit ::timeseries- [_ {limit :limit} druid-query]- (when limit- (log/warn- (u/format-color 'red "WARNING: Druid does not allow limitSpec in time series queries. Ignoring the LIMIT clause.")))- druid-query)--(defmethod handle-limit ::topN- [_ {limit :limit} druid-query]- (cond-> druid-query- limit (assoc-in [:query :threshold] (adjust-limit limit))))--(defmethod handle-limit ::groupBy- [_ {limit :limit} druid-query]- (cond-> druid-query- true (assoc-in [:query :limitSpec :type] :default)- limit (assoc-in [:query :limitSpec :limit] (adjust-limit limit))))--;;; -------------------------------------------------- handle-page -----------------------------------------------------;; TODO - no real way to implement this DB side, probably have to do Clojure-side w/ `take`/`drop`--(defmulti ^:private handle-page- {:arglists '([query-type original-query druid-query])}- query-type-dispatch-fn)--(defmethod handle-page ::query- [_ {page-clause :page} druid-query]- (when page-clause- (log/warn (u/format-color 'red "WARNING: 'page' is not yet implemented.")))- druid-query)--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | Build + Log + Process Query |-;;; +----------------------------------------------------------------------------------------------------------------+--(def ^:private ^:const timeseries-units #{:minute :hour :day :week :month :quarter :year})--(defn- druid-query-type- "What type of Druid query type should we perform?"- [{breakout-fields :breakout, [[ag-type]] :aggregation, limit :limit}]- (let [breakouts (condp = (count breakout-fields)- 0 :none- 1 :one- :many)- agg? (boolean ag-type)- ts? (boolean- (and- ;; Checks whether the query is a timeseries- (match/match-one (first breakout-fields) [:field _ {:temporal-unit &truthy}] true)- ;; (excludes x-of-y type breakouts)- (contains? timeseries-units (:unit (first breakout-fields)))- ;; (excludes queries with LIMIT)- (nil? limit)))]- (match/match-one [breakouts agg? ts?]- [:none false _] ::scan- [:none true _] ::total- [:one _ true] ::grouped-timeseries- [:one _ false] ::topN- [:many _ _] ::groupBy)))--(defn- build-druid-query- [original-query]- {:pre [(map? original-query)]}- (let [query-type (druid-query-type original-query)]- (reduce (fn [druid-query f]- (f query-type original-query druid-query))- {:projections [], :query (query-type->default-query query-type), :query-type query-type, :mbql? true}- [handle-source-table- handle-breakout- handle-aggregations- handle-filter- handle-order-by- handle-fields- handle-limit- handle-page])))--(defn mbql->native- "Transpile an MBQL (inner) query into a native form suitable for a Druid DB."- [query]- ;; Merge `:settings` into the inner query dict so the QP has access to it- (let [query (driver-api/->legacy-MBQL query)- query (assoc (:query query) :settings (:settings query))]- (binding [*query* query]- (try- (build-druid-query query)- (catch Throwable e- (throw (ex-info (tru "Error generating Druid query")- {:type driver-api/qp.error-type.driver- :source-query query}- e)))))))diff --git a/modules/drivers/druid/src/metabase/driver/druid/sync.clj b/modules/drivers/druid/src/metabase/driver/druid/sync.cljdeleted file mode 100644index 8d6297a8525..00000000000--- a/modules/drivers/druid/src/metabase/driver/druid/sync.clj+++ /dev/null@@ -1,76 +0,0 @@-(ns metabase.driver.druid.sync- (:refer-clojure :exclude [select-keys])- (:require- [medley.core :as m]- [metabase.driver-api.core :as driver-api]- [metabase.driver.connection :as driver.conn]- [metabase.driver.druid.client :as druid.client]- [metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh]- [metabase.util.performance :refer [select-keys]]))--(defn- do-segment-metadata-query [details datasource]- {:pre [(map? details) (string? datasource)]}- (druid.client/do-query details {"queryType" :segmentMetadata- "dataSource" datasource- "intervals" ["1999-01-01/2114-01-01"]- "analysisTypes" [:aggregators]- "merge" true}))--(defn- druid-type->base-type [field-type]- (case field-type- "STRING" :type/Text- "FLOAT" :type/Float- "DOUBLE" :type/Float- "LONG" :type/Integer- "hyperUnique" :type/DruidHyperUnique- :type/Float))--(defn describe-table- "Impl of `driver/describe-table` for Druid."- [database table]- {:pre [(map? database) (map? table)]}- (ssh/with-ssh-tunnel [details-with-tunnel (driver.conn/effective-details database)]- (let [{:keys [columns aggregators]} (first (do-segment-metadata-query details-with-tunnel (:name table)))- metric-column-names (set (keys aggregators))]- {:schema nil- :name (:name table)- :fields (set (cons- ;; every Druid table is an event stream w/ a timestamp field- {:name "timestamp"- :database-type "timestamp"- :base-type :type/Instant- :pk? false- :database-position 0}- (for [[idx [field-name {field-type :type}]] (m/indexed (dissoc columns :__time))- :let [metric? (contains? metric-column-names field-name)]]- {:name (name field-name)- :base-type (druid-type->base-type field-type)- :database-type (if metric?- (format "%s [metric]" field-type)- field-type)- :database-position (inc idx)})))})))--(defn describe-database- "Impl of `driver/describe-database` for Druid."- [database]- {:pre [(map? (:details database))]}- (let [details (driver.conn/effective-details database)]- (ssh/with-ssh-tunnel [details-with-tunnel details]- (let [druid-datasources (druid.client/GET (druid.client/details->url details-with-tunnel "/druid/v2/datasources")- :auth-enabled (:auth-enabled details)- :auth-username (:auth-username details)- :auth-token-value (driver-api/secret-value-as-string :druid details "auth-token"))]- {:tables (set (for [table-name druid-datasources]- {:schema nil, :name table-name}))}))))--(defn dbms-version- "Impl of `driver/dbms-version` for Druid."- [database]- {:pre [(map? (:details database))]}- (let [details (driver.conn/effective-details database)]- (ssh/with-ssh-tunnel [details-with-tunnel details]- (-> (druid.client/GET (druid.client/details->url details-with-tunnel "/status")- :auth-enabled (:auth-enabled details)- :auth-username (:auth-username details)- :auth-token-value (driver-api/secret-value-as-string :druid details "auth-token"))- (select-keys [:version])))))diff --git a/modules/drivers/druid/test/metabase/driver/druid/client_test.clj b/modules/drivers/druid/test/metabase/driver/druid/client_test.cljdeleted file mode 100644index 4bd715a88b0..00000000000--- a/modules/drivers/druid/test/metabase/driver/druid/client_test.clj+++ /dev/null@@ -1,105 +0,0 @@-(ns ^:mb/driver-tests metabase.driver.druid.client-test- {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.driver.druid.client-test]}}}}}}- (:require- [clojure.core.async :as a]- [clojure.test :refer :all]- [metabase.driver.druid.client :as druid.client]- [metabase.driver.settings :as driver.settings]- [metabase.driver.util :as driver.u]- [metabase.query-processor.test :as qp]- [metabase.query-processor.timeseries-test.util :as tqpt]- [metabase.test :as mt]))--(set! *warn-on-reflection* true)--(deftest query-cancelation-test- (mt/test-driver :druid- (tqpt/with-flattened-dbdef- (let [query (mt/mbql-query checkins)]- (mt/with-open-channels [running-chan (a/promise-chan)- cancel-chan (a/promise-chan)]- (with-redefs [druid.client/DELETE (fn [& _]- (a/>!! cancel-chan ::cancel))- druid.client/do-query (fn [& _]- (a/>!! running-chan ::running)- (Thread/sleep 5000)- (throw (Exception. "Don't actually run!")))]- (let [futur (future (qp/process-query query))]- ;; wait for query to start running, then kill the thread running the query- (a/go- (a/<! running-chan)- (future-cancel futur)))- (is (= ::cancel- (mt/wait-for-result cancel-chan 2000)))))))))--(deftest query-timeout-test- (mt/test-driver :druid- (tqpt/with-flattened-dbdef- (let [query (mt/mbql-query checkins)- executed-query (atom nil)]- (mt/with-dynamic-fn-redefs [druid.client/do-query-with-cancellation (fn [_chan _details query]- (reset! executed-query query)- [])]- (qp/process-query query)- (is (partial= {:context {:timeout driver.settings/*query-timeout-ms*}}- @executed-query)))))))--(deftest ssh-tunnel-test- (mt/test-driver- :druid- (is (thrown?- java.net.ConnectException- (try- (let [engine :druid- details {:ssl false- :password "changeme"- :tunnel-host "localhost"- :tunnel-pass "BOGUS-BOGUS"- :port 5432- :dbname "test"- :host "http://localhost"- :tunnel-enabled true- ;; we want to use a bogus port here on purpose -- ;; so that locally, it gets a ConnectionRefused,- ;; and in CI it does too. Apache's SSHD library- ;; doesn't wrap every exception in an SshdException- :tunnel-port 21212- :tunnel-user "bogus"}]- (driver.u/can-connect-with-details? engine details :throw-exceptions))- (catch Throwable e- (loop [^Throwable e e]- (or (when (instance? java.net.ConnectException e)- (throw e))- (some-> (.getCause e) recur)))))))))--(defn- test-request- ([request-fn]- (test-request request-fn true))- ([request-fn basic-auth?]- (try- (request-fn "http://localhost:8082/druid/v2"- :auth-enabled basic-auth?- :auth-username "nbotelho"- :auth-token-value "12345678910")- (catch Exception e- (ex-data e)))))--(defn- get-auth-header [m]- (select-keys (:request-options m) [:basic-auth]))--(deftest basic-auth-test- (let [get-request (test-request druid.client/GET)- post-request (test-request druid.client/POST)- delete-request (test-request druid.client/DELETE)]- (is (= {:basic-auth "nbotelho:12345678910"}- (get-auth-header get-request)- (get-auth-header post-request)- (get-auth-header delete-request)) "basic auth header included with successfully"))- (let [no-auth-basic false- get-request (test-request druid.client/GET no-auth-basic)- post-request (test-request druid.client/POST no-auth-basic)- delete-request (test-request druid.client/DELETE no-auth-basic)]- (is (= no-auth-basic- (contains? (get-auth-header get-request) :basic-auth)- (contains? (get-auth-header post-request) :basic-auth)- (contains? (get-auth-header delete-request) :basic-auth)) "basic auth header not included")))diff --git a/modules/drivers/druid/test/metabase/driver/druid/execute_test.clj b/modules/drivers/druid/test/metabase/driver/druid/execute_test.cljdeleted file mode 100644index 73c5c161fd3..00000000000--- a/modules/drivers/druid/test/metabase/driver/druid/execute_test.clj+++ /dev/null@@ -1,60 +0,0 @@-(ns ^:mb/driver-tests metabase.driver.druid.execute-test- {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.driver.druid.execute-test]}- metabase.test.data/run-mbql-query {:namespaces [metabase.driver.druid.execute-test]}}}}}}- (:require- [clojure.java.io :as io]- [clojure.test :refer :all]- [metabase.driver.druid.execute :as druid.execute]- [metabase.query-processor.timeseries-test.util :as tqpt]- [metabase.test :as mt]- [metabase.util :as u]- [metabase.util.json :as json]))--(deftest ^:parallel results-order-test- (mt/test-driver :druid- (testing (str "Make sure Druid cols + columns come back in the same order and that that order is the expected MBQL "- "columns order (#9294)")- (tqpt/with-flattened-dbdef- (let [results (mt/run-mbql-query checkins- {:limit 1, :order-by [[:asc $timestamp]]})]- (assert (= (:status results) :completed)- (u/pprint-to-str 'red results))- (testing "cols"- (is (= ["timestamp"- "venue_name"- "venue_longitude"- "venue_latitude"- "venue_price"- "venue_category_name"- "id"- "count"- "unique_users"- "user_name"- "user_last_login"]- (->> results :data :cols (map :name)))))- (testing "rows"- (is (= [["2013-01-03T00:00:00Z"- "Kinaree Thai Bistro"- -118.344- 34.094- 1- "Thai"- 931- 1- "AQAAAQAAAAEBsA=="- "Simcha Yan"- "2014-01-01T08:30:00"]]- (-> results :data :rows)))))))))--(deftest ^:parallel post-process-select-query-test- (testing "Test that we can still return results from native :select queries, even if we no longer generate them"- ;; example results adapted from https://github.com/apache/druid/blob/d00747774208dbbfcb272ee7d1c30cf879887838/docs/querying/select-query.md- (let [results (with-open [r (io/reader "modules/drivers/druid/test/metabase/driver/druid/execute_test/select_results.json")]- (json/decode+kw r))]- (is (= {:projections [:a :b :c]- :results [{:timestamp #t "2013-01-01T00:00Z[UTC]", :robot "1", :namespace "article", :page "11._korpus_(NOVJ)"}- {:timestamp #t "2013-01-01T00:00Z[UTC]", :robot "0", :namespace "article", :page "112_U.S._580"}- {:timestamp #t "2013-01-01T00:00Z[UTC]", :robot "0", :namespace "article", :page "113_U.S._243"}- {:timestamp #t "2013-01-01T00:00Z[UTC]", :robot "0", :namespace "article", :page "113_U.S._73"}- {:timestamp #t "2013-01-01T00:00Z[UTC]", :robot "0", :namespace "article", :page "113_U.S._756"}]}- (#'druid.execute/post-process :metabase.driver.druid.query-processor/select [:a :b :c] nil results))))))diff --git a/modules/drivers/druid/test/metabase/driver/druid/execute_test/select_results.json b/modules/drivers/druid/test/metabase/driver/druid/execute_test/select_results.jsondeleted file mode 100644index f1e9a2e43c0..00000000000--- a/modules/drivers/druid/test/metabase/driver/druid/execute_test/select_results.json+++ /dev/null@@ -1,54 +0,0 @@-[{- "timestamp" : "2013-01-01T00:00:00.000Z",- "result" : {- "pagingIdentifiers" : {- "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9" : 4- },- "events" : [ {- "segmentId" : "wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9",- "offset" : 0,- "event" : {- "timestamp" : "2013-01-01T00:00:00.000Z",- "robot" : "1",- "namespace" : "article",- "page" : "11._korpus_(NOVJ)"- }- }, {- "segmentId" : "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9",- "offset" : 1,- "event" : {- "timestamp" : "2013-01-01T00:00:00.000Z",- "robot" : "0",- "namespace" : "article",- "page" : "112_U.S._580"- }- }, {- "segmentId" : "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9",- "offset" : 2,- "event" : {- "timestamp" : "2013-01-01T00:00:00.000Z",- "robot" : "0",- "namespace" : "article",- "page" : "113_U.S._243"- }- }, {- "segmentId" : "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9",- "offset" : 3,- "event" : {- "timestamp" : "2013-01-01T00:00:00.000Z",- "robot" : "0",- "namespace" : "article",- "page" : "113_U.S._73"- }- }, {- "segmentId" : "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9",- "offset" : 4,- "event" : {- "timestamp" : "2013-01-01T00:00:00.000Z",- "robot" : "0",- "namespace" : "article",- "page" : "113_U.S._756"- }- } ]- }-} ]diff --git a/modules/drivers/druid/test/metabase/driver/druid/query_processor_test.clj b/modules/drivers/druid/test/metabase/driver/druid/query_processor_test.cljdeleted file mode 100644index 9984ce4ec4b..00000000000--- a/modules/drivers/druid/test/metabase/driver/druid/query_processor_test.clj+++ /dev/null@@ -1,675 +0,0 @@-(ns ^:mb/driver-tests metabase.driver.druid.query-processor-test- "Some tests to make sure the Druid Query Processor is generating sane Druid queries when compiling MBQL."- {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.driver.druid.query-processor-test]}- metabase.test.data/run-mbql-query {:namespaces [metabase.driver.druid.query-processor-test]}}}}}}- (:require- [clojure.test :refer :all]- [clojure.tools.macro :as tools.macro]- [java-time.api :as t]- [medley.core :as m]- [metabase.driver :as driver]- [metabase.driver.common.table-rows-sample :as table-rows-sample]- [metabase.driver.druid.query-processor :as druid.qp]- [metabase.lib.test-metadata :as meta]- [metabase.lib.test-util.macros :as lib.tu.macros]- [metabase.query-processor.compile :as qp.compile]- [metabase.query-processor.test :as qp]- [metabase.query-processor.timeseries-test.util :as tqpt]- [metabase.test :as mt]- [metabase.util.date-2 :as u.date]- [metabase.util.json :as json]- [toucan2.core :as t2]))--(defn- str->absolute-dt [s]- [:absolute-datetime (u.date/parse s "UTC") :default])--(deftest ^:parallel filter-intervals-test- (let [dt-field [:field 1 {:temporal-unit :default}]- filter-clause->intervals (comp @#'druid.qp/compile-intervals @#'druid.qp/filter-clause->intervals)]- (testing :=- (is (= ["2015-10-04T00:00:00Z/2015-10-04T00:00:00.001Z"]- (filter-clause->intervals [:= dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]))- ":= filters should get converted to intervals like `v/v+1`")- (is (= nil- (filter-clause->intervals [:= [:field 1 nil] "toucan"]))- "Non-temporal filter clauses should return `nil` intervals"))- (testing :<- (is (= ["-5000/2015-10-11T00:00:00Z"]- (filter-clause->intervals [:< dt-field (str->absolute-dt "2015-10-11T00:00:00Z")]))- ":<, :<=, :>, and :>= should return an interval with -5000 or 5000 as min or max"))- (testing :between- (is (= ["2015-10-04T00:00:00Z/2015-10-20T00:00:00.001Z"]- (filter-clause->intervals- [:between dt-field (str->absolute-dt "2015-10-04T00:00:00Z") (str->absolute-dt "2015-10-20T00:00:00Z")]))))- (testing :and- (is (= ["2015-10-04T00:00:00Z/2015-10-11T00:00:00Z"]- (filter-clause->intervals- [:and- [:>= dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]- [:< dt-field (str->absolute-dt "2015-10-11T00:00:00Z")]]))- "The Druid QP should be able to combine compound `:and` filter clauses into a single datetime interval.")- (is (= ["2015-10-06T00:00:00Z/2015-10-20T00:00:00.001Z"]- (filter-clause->intervals- [:and- [:between dt-field (str->absolute-dt "2015-10-04T00:00:00Z") (str->absolute-dt "2015-10-20T00:00:00Z")]- [:between dt-field (str->absolute-dt "2015-10-06T00:00:00Z") (str->absolute-dt "2015-10-21T00:00:00Z")]]))- "When two filters have overlapping intervals it should generate a single logically equivalent interval")- (is (= nil- (filter-clause->intervals- [:and [:= [:field 1 nil] "toucan"] [:= [:field 2 nil] "threecan"]]))- ":and clause should ignore non-temporal filters")- (is (= ["2015-10-04T00:00:00Z/2015-10-04T00:00:00.001Z"]- (filter-clause->intervals- [:and- [:= [:field 1 nil] "toucan"] [:= dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]]))- ":and clause with no temporal filters should be compiled to `nil` interval")- (is (= ["2015-10-04T00:00:00Z/2015-10-04T00:00:00.001Z"]- (filter-clause->intervals- [:and- [:= dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]- [:or- [:>= dt-field (str->absolute-dt "2015-10-03T00:00:00Z")]- [:< dt-field (str->absolute-dt "2015-10-11T00:00:00Z")]]]))- ":and clause should ignore nested `:or` filters, since they can't be combined into a single filter"))- (testing :or- (is (= ["2015-10-04T00:00:00Z/5000" "-5000/2015-10-11T00:00:00Z"]- (filter-clause->intervals- [:or- [:>= dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]- [:< dt-field (str->absolute-dt "2015-10-11T00:00:00Z")]]))- ":or filters should be combined into multiple intervals")- (is (= ["2015-10-04T00:00:00Z/5000"]- (filter-clause->intervals- [:or- [:>= dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]- [:= [:field 1 nil] "toucan"]]))- ":or clauses should ignore non-temporal filters")- (is (= nil- (filter-clause->intervals- [:or- [:= [:field 1 nil] "toucan"]- [:= [:field 2 nil] "threecan"]]))- ":or filters with no temporal filters should return nil"))- (testing :not- (is (= ["2015-10-04T00:00:00Z/2015-10-11T00:00:00Z"]- (filter-clause->intervals- [:not- [:or- [:< dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]- [:>= dt-field (str->absolute-dt "2015-10-11T00:00:00Z")]]]))))))--(defn- do-query->native [query]- (driver/with-driver :druid- (tqpt/with-flattened-dbdef- (binding [druid.qp/*random-query-id* (constantly "<Query ID>")]- (qp.compile/compile query)))))--(defmacro ^:private query->native [query]- `(do-query->native- (mt/mbql-query ~'checkins- ~query)))--(deftest ^:parallel compile-topN-test- (mt/test-driver :druid- (tqpt/with-flattened-dbdef- (is (= {:projections [:venue_price :__count :expression]- :query {:queryType :topN- :threshold 1000- :granularity :all- :dataSource "checkins"- :dimension "venue_price"- :context {:queryId "<Query ID>"}- :postAggregations [{:type :arithmetic- :name "expression"- :fn :*- :fields [{:type :fieldAccess, :fieldName "__count"}- {:type :constant, :name "10", :value 10}]}]- :intervals ["1900-01-01/2100-01-01"]- :metric {:type :alphaNumeric}- :aggregations- [{:type :filtered- :filter {:type :not- :field {:type :selector, :dimension "id", :value nil}}- :aggregator {:type :count, :name "__count"}}]}- :query-type ::druid.qp/topN- :mbql? true}- (query->native- #_:clj-kondo/ignore- {:aggregation [[:* [:count $id] 10]]- :breakout [$venue_price]}))))))--(deftest ^:parallel compile-topN-with-order-by-test- (mt/test-driver :druid- (tqpt/with-flattened-dbdef- (is (= {:projections [:venue_category_name :__count_0]- :query {:queryType :topN- :threshold 1000- :granularity :all- :dataSource "checkins"- :dimension "venue_category_name"- :context {:queryId "<Query ID>"}- :intervals ["1900-01-01/2100-01-01"]- :metric "__count_0"- :aggregations [{:type :cardinality- :name "__count_0"- :fieldNames ["venue_name"]- :byRow true- :round true}]}- :query-type ::druid.qp/topN- :mbql? true}- (query->native- #_:clj-kondo/ignore- {:aggregation [[:aggregation-options [:distinct $checkins.venue_name] {:name "__count_0"}]]- :breakout [$venue_category_name]- :order-by [[:desc [:aggregation 0]] [:asc $checkins.venue_category_name]]}))))))--(deftest ^:parallel compile-groupBy-test- (mt/test-driver :druid- (tqpt/with-flattened-dbdef- (is (= {:projections [:venue_category_name :user_name :__count_0]- :query {:queryType :groupBy- :granularity :all- :dataSource "checkins"- :dimensions ["venue_category_name", "user_name"]- :context {:queryId "<Query ID>"}- :intervals ["1900-01-01/2100-01-01"]- :aggregations [{:type :cardinality- :name "__count_0"- :fieldNames ["venue_name"]- :byRow true- :round true}]- :limitSpec {:type :default- :columns [{:dimension "__count_0", :direction :descending}- {:dimension "venue_category_name", :direction :ascending}- {:dimension "user_name", :direction :ascending}]}}- :query-type ::druid.qp/groupBy- :mbql? true}- (query->native- #_:clj-kondo/ignore- {:aggregation [[:aggregation-options [:distinct $checkins.venue_name] {:name "__count_0"}]]- :breakout [$venue_category_name $user_name]- :order-by [[:desc [:aggregation 0]] [:asc $checkins.venue_category_name]]}))))))--(deftest ^:parallel compile-groupBy-with-limit-test- (mt/test-driver :druid- (tqpt/with-flattened-dbdef- (is (= {:projections [:venue_category_name :user_name :__count_0]- :query {:queryType :groupBy- :granularity :all- :dataSource "checkins"- :dimensions ["venue_category_name", "user_name"]- :context {:queryId "<Query ID>"}- :intervals ["1900-01-01/2100-01-01"]- :aggregations [{:type :cardinality- :name "__count_0"- :fieldNames ["venue_name"]- :byRow true- :round true}]- :limitSpec {:type :default- :columns [{:dimension "__count_0", :direction :descending}- {:dimension "venue_category_name", :direction :ascending}- {:dimension "user_name", :direction :ascending}]- :limit 5}}- :query-type ::druid.qp/groupBy- :mbql? true}- (query->native- {:aggregation [[:aggregation-options [:distinct $checkins.venue_name] {:name "__count_0"}]]- :breakout [$venue_category_name $user_name]- :order-by [[:desc [:aggregation 0]] [:asc $checkins.venue_category_name]]- :limit 5}))))))--(deftest ^:parallel finalizing-field-access-test- (mt/test-driver :druid- (tqpt/with-flattened-dbdef- (testing "`distinct` when used in post aggregations should have type `:finalizingFieldAccess`"- (is (= {:projections [:__distinct_0 :expression]- :query {:queryType :timeseries- :granularity :all- :dataSource "checkins"- :context {:queryId "<Query ID>"}- :intervals ["1900-01-01/2100-01-01"]- :aggregations [{:type :cardinality- :name "__distinct_0"- :fieldNames ["venue_name"]- :byRow true- :round true}]- :postAggregations [{:type :arithmetic,- :name "expression",- :fn :+,- :fields- [{:type :constant, :name "1", :value 1}- {:type :finalizingFieldAccess, :fieldName "__distinct_0"}]}]}- :query-type ::druid.qp/total- :mbql? true}- (query->native- {:aggregation [[:+ 1 [:aggregation-options [:distinct $checkins.venue_name] {:name "__distinct_0"}]]]})))))))--(defn- table-rows-sample []- (->> (table-rows-sample/table-rows-sample (t2/select-one :model/Table :id (mt/id :checkins))- [(t2/select-one :model/Field :id (mt/id :checkins :id))- (t2/select-one :model/Field :id (mt/id :checkins :venue_name))- (t2/select-one :model/Field :id (mt/id :checkins :timestamp))]- (constantly conj))- (sort-by first)- (take 5)))--(deftest table-rows-sample-test- (mt/test-driver :druid- (tqpt/with-flattened-dbdef- (testing "Druid driver doesn't need to convert results to the expected timezone for us. QP middleware can handle that."- (let [expected [[1 "The Misfit Restaurant + Bar" (t/instant "2014-04-07T00:00:00Z")]- [2 "Bludso's BBQ" (t/instant "2014-09-18T00:00:00Z")]- [3 "Philippe the Original" (t/instant "2014-09-15T00:00:00Z")]- [4 "Wurstküche" (t/instant "2014-03-11T00:00:00Z")]- [5 "Hotel Biron" (t/instant "2013-05-05T00:00:00Z")]]]- (testing "UTC timezone"- (is (= expected- (table-rows-sample))))- (mt/with-temporary-setting-values [report-timezone "America/Los_Angeles"]- (is (= expected- (table-rows-sample))))- (mt/with-system-timezone-id! "America/Chicago"- (is (= expected- (table-rows-sample)))))))))--(def ^:private native-query-1- (json/encode- {:queryType :scan- :dataSource :checkins- :intervals ["1900-01-01/2100-01-01"]- :granularity :all- :limit 2- :columns [:id- :user_name- :venue_price- :venue_name- :count]}))--(defn- process-native-query [query]- (driver/with-driver :druid- (tqpt/with-flattened-dbdef- (-> (qp/process-query {:native {:query query}- :type :native- :database (mt/id)})- (m/dissoc-in [:data :results_metadata])))))--(deftest ^:parallel native-query-test- (mt/test-driver :druid- (is (partial=- {:row_count 2- :status :completed- :data {:rows [[931 "Simcha Yan" 1 "Kinaree Thai Bistro" 1]- [285 "Kfir Caj" 2 "Ruen Pair Thai Restaurant" 1]]- :cols [{:name "id"- :source :native- :display_name "id"- :field_ref [:field "id" {:base-type :type/Integer}]- :base_type :type/Integer- :effective_type :type/Integer}- {:name "user_name"- :source :native- :display_name "user_name"- :base_type :type/Text- :effective_type :type/Text- :field_ref [:field "user_name" {:base-type :type/Text}]}- {:name "venue_price"- :source :native- :display_name "venue_price"- :base_type :type/Integer- :effective_type :type/Integer- :field_ref [:field "venue_price" {:base-type :type/Integer}]}- {:name "venue_name"- :source :native- :display_name "venue_name"- :base_type :type/Text- :effective_type :type/Text- :field_ref [:field "venue_name" {:base-type :type/Text}]}- {:name "count"- :source :native- :display_name "count"- :base_type :type/Integer- :effective_type :type/Integer- :field_ref [:field "count" {:base-type :type/Integer}]}]- :native_form {:query native-query-1}- :results_timezone "UTC"}}- (-> (process-native-query native-query-1)- (m/dissoc-in [:data :insights]))))))--(def ^:private native-query-2- (json/encode- {:intervals ["1900-01-01/2100-01-01"]- :granularity {:type :period- :period :P1M- :timeZone :UTC}- :queryType :timeseries- :dataSource :checkins- :aggregations [{:type :count- :name :count}]}))--(deftest ^:parallel native-query-test-2- (testing "make sure we can run a native :timeseries query. This was throwing an Exception -- see #3409"- (mt/test-driver :druid- (is (= :completed- (:status (process-native-query native-query-2)))))))--(defmacro ^:private druid-query {:style/indent 0} [& body]- `(tqpt/with-flattened-dbdef- (qp/process-query- (mt/mbql-query ~'checkins- ~@body))))--(defmacro ^:private druid-query-returning-rows {:style/indent 0} [& body]- `(mt/rows (druid-query ~@body)))--(deftest ^:parallel start-of-week-test- (mt/test-driver :druid- (testing "Count the number of events in the given week."- (is (= [["2015-10-04" 9]]- (druid-query-returning-rows- {:filter [:between !day.timestamp "2015-10-04" "2015-10-10"]- :aggregation [[:count $id]]- :breakout [!week.timestamp]}))))))--(deftest ^:parallel sum-aggregation-test- (mt/test-driver :druid- (testing "sum, *"- (is (= [["1" 110688.0]- ["2" 616708.0]- ["3" 179661.0]- ["4" 86284.0]]- (druid-query-returning-rows- {:aggregation [[:sum [:* $id $venue_price]]]- :breakout [$venue_price]}))))))--(deftest ^:parallel min-aggregation-test- (mt/test-driver :druid- (testing "min, +"- (is (= [["1" 4.0]- ["2" 3.0]- ["3" 8.0]- ["4" 12.0]]- (druid-query-returning-rows- {:aggregation [[:min [:+ $id $venue_price]]]- :breakout [$venue_price]}))))))--(deftest ^:parallel max-aggregation-test- (mt/test-driver :druid- (testing "max, /"- (is (= [["1" 1000.0]- ["2" 499.5]- ["3" 332.0]- ["4" 248.25]]- (druid-query-returning-rows- {:aggregation [[:max [:/ $id $venue_price]]]- :breakout [$venue_price]}))))))--(deftest ^:parallel avg-aggregation-test- (mt/test-driver :druid- (testing "avg, -"- (is (= [["1" 500.85067873303166]- ["2" 1002.7772357723577]- ["3" 1562.2695652173913]- ["4" 1760.8979591836735]]- (druid-query-returning-rows- {:aggregation [[:avg [:* $id $venue_price]]]- :breakout [$venue_price]}))))))--(deftest ^:parallel share-aggregation-test- (mt/test-driver :druid- (testing "share"- (is (= [[0.951]]- (druid-query-returning-rows- {:aggregation [[:share [:< $venue_price 4]]]}))))))--(deftest ^:parallel count-where-aggregation-test- (mt/test-driver :druid- (testing "count-where"- (is (= [[951]]- (druid-query-returning-rows- {:aggregation [[:count-where [:< $venue_price 4]]]}))))))--(deftest ^:parallel sum-where-aggregation-test- (mt/test-driver :druid- (testing "sum-where"- (is (= [[1796.0]]- (druid-query-returning-rows- {:aggregation [[:sum-where $venue_price [:< $venue_price 4]]]}))))))--(deftest ^:parallel count-aggregation-test- (mt/test-driver :druid- (testing "aggregation w/o field"- (is (= [["1" 222.0]- ["2" 616.0]- ["3" 116.0]- ["4" 50.0]]- (druid-query-returning-rows- {:aggregation [[:+ 1 [:count]]]- :breakout [$venue_price]}))))))--(deftest ^:parallel expression-aggregations-test- (mt/test-driver :druid- (testing "post-aggregation math w/ 2 args: count + sum"- (is (= [["1" 442.0]- ["2" 1845.0]- ["3" 460.0]- ["4" 245.0]]- (druid-query-returning-rows- {:aggregation [[:+ [:count $id] [:sum $venue_price]]]- :breakout [$venue_price]}))))- (testing "post-aggregation math w/ 3 args: count + sum + count"- (is (= [["1" 663.0]- ["2" 2460.0]- ["3" 575.0]- ["4" 294.0]]- (druid-query-returning-rows- {:aggregation [[:+- [:count $id]- [:sum $venue_price]- [:count $venue_price]]]- :breakout [$venue_price]}))))- (testing "post-aggregation math w/ a constant: count * 10"- (is (= [["1" 2210.0]- ["2" 6150.0]- ["3" 1150.0]- ["4" 490.0]]- (druid-query-returning-rows- {:aggregation [[:* [:count $id] 10]]- :breakout [$venue_price]}))))- (testing "nested post-aggregation math: count + (count * sum)"- (is (= [["1" 49062.0]- ["2" 757065.0]- ["3" 39790.0]- ["4" 9653.0]]- (druid-query-returning-rows- {:aggregation [[:+- [:count $id]- [:* [:count $id] [:sum $venue_price]]]]- :breakout [$venue_price]}))))- (testing "post-aggregation math w/ avg: count + avg"- (is (= [["1" 721.8506787330316]- ["2" 1116.388617886179]- ["3" 635.7565217391304]- ["4" 489.2244897959184]]- (druid-query-returning-rows- {:aggregation [[:+ [:count $id] [:avg $id]]]- :breakout [$venue_price]}))))- (testing "aggregation with math inside the aggregation :scream_cat:"- (is (= [["1" 442.0]- ["2" 1845.0]- ["3" 460.0]- ["4" 245.0]]- (druid-query-returning-rows- {:aggregation [[:sum [:+ $venue_price 1]]]- :breakout [$venue_price]}))))- (testing "post aggregation math + math inside aggregations: max(venue_price) + min(venue_price - id)"- (is (= [["1" -998.0]- ["2" -995.0]- ["3" -990.0]- ["4" -985.0]]- (druid-query-returning-rows- {:aggregation [[:+- [:max $venue_price]- [:min [:- $venue_price $id]]]]- :breakout [$venue_price]}))))))--(deftest ^:parallel named-top-level-aggregation-test- (mt/test-driver :druid- (testing "check that we can name an expression aggregation w/ aggregation at top-level"- (is (= [["1" 442.0]- ["2" 1845.0]- ["3" 460.0]- ["4" 245.0]]- (mt/rows- (druid-query- {:aggregation [[:aggregation-options [:sum [:+ $venue_price 1]] {:name "New Price"}]]- :breakout [$venue_price]})))))))--(deftest ^:parallel named-expression-aggregations-test- (mt/test-driver :druid- (testing "check that we can name an expression aggregation w/ expression at top-level"- (is (= {:rows [["1" 180.0]- ["2" 1189.0]- ["3" 304.0]- ["4" 155.0]]- :columns ["venue_price" "Sum-41"]}- (mt/rows+column-names- (druid-query- {:aggregation [[:aggregation-options [:- [:sum $venue_price] 41] {:name "Sum-41"}]]- :breakout [$venue_price]})))))))--(deftest ^:parallel distinct-count-of-two-dimensions-test- (mt/test-driver :druid- (is (= {:rows [[979]]- :columns ["count"]}- (mt/rows+column-names- (druid-query- {:aggregation [[:distinct [:+ $id $checkins.venue_price]]]}))))))--(deftest ^:parallel order-by-aggregation-test- (mt/test-driver :druid- (doseq [[direction expected-rows] {:desc [["Bar" "Felipinho Asklepios" 8]- ["Bar" "Spiros Teofil" 8]- ["Japanese" "Felipinho Asklepios" 7]- ["Japanese" "Frans Hevel" 7]- ["Mexican" "Shad Ferdynand" 7]]- :asc [["American" "Rüstem Hebel" 1]- ["Artisan" "Broen Olujimi" 1]- ["Artisan" "Conchúr Tihomir" 1]- ["Artisan" "Dwight Gresham" 1]- ["Artisan" "Plato Yeshua" 1]]}]- (testing direction- (is (= expected-rows- (druid-query-returning-rows- {:aggregation [[:aggregation-options [:distinct $checkins.venue_name] {:name "__count_0"}]]- :breakout [$venue_category_name $user_name]- :order-by [[direction [:aggregation 0]] [:asc $checkins.venue_category_name]]- :limit 5})))))))--(deftest ^:parallel hll-count-test- (mt/test-driver :druid- (testing "Do we generate the correct count clause for HLL fields?"- (is (= [["Bar" "Szymon Theutrich" 13]- ["Mexican" "Dwight Gresham" 12]- ["American" "Spiros Teofil" 10]- ["Bar" "Felipinho Asklepios" 10]- ["Bar" "Kaneonuskatew Eiran" 10]]- (druid-query-returning-rows- {:aggregation [[:aggregation-options [:count $checkins.user_name] {:name "unique_users"}]]- :breakout [$venue_category_name $user_name]- :order-by [[:desc [:aggregation 0]] [:asc $checkins.venue_category_name]]- :limit 5}))))))--(deftest ^:parallel numeric-filter-test- (mt/test-driver :druid- (tqpt/with-flattened-dbdef- (letfn [(compiled [query]- (-> (qp.compile/compile query) :query (select-keys [:filter :queryType])))]- (doseq [[message field] {"Make sure we can filter by numeric columns (#10935)" :venue_price- "We should be able to filter by Metrics (#11823)" :count}- :let [field-clause [:field (mt/id :checkins field) nil]- field-name (name field)]]- (testing message- (testing "scan query"- (let [query (mt/mbql-query checkins- {:fields [$id $venue_price $venue_name]- :filter [:= field-clause 1]- :order-by [[:desc $id]]- :limit 5})]- (is (= {:filter {:type :selector, :dimension field-name, :value 1}- :queryType :scan}- (compiled query)))- (is (= [931 1 "Kinaree Thai Bistro"]- (mt/first-row (qp/process-query query))))))- (testing "topN query"- (let [query (mt/mbql-query checkins- {:aggregation [[:count]]- :breakout [$venue_price]- :filter [:= field-clause 1]})]- (is (= {:filter {:type :selector, :dimension field-name, :value 1}- :queryType :topN}- (compiled query)))- (is (= ["1" 221]- (mt/first-row (qp/process-query query))))))- (testing "groupBy query"- (let [query (mt/mbql-query checkins- {:aggregation [[:aggregation-options [:distinct $checkins.venue_name] {:name "__count_0"}]]- :breakout [$venue_category_name $user_name]- :order-by [[:desc [:aggregation 0]] [:asc $checkins.venue_category_name]]- :filter [:= field-clause 1]})]- (is (= {:filter {:type :selector, :dimension field-name, :value 1}- :queryType :groupBy}- (compiled query)))- (is (= (case field- :count ["Bar" "Felipinho Asklepios" 8]- :venue_price ["Mexican" "Conchúr Tihomir" 4])- (mt/first-row (qp/process-query query))))))- (testing "timeseries query"- (let [query (mt/mbql-query checkins- {:aggregation [[:count]]- :filter [:= field-clause 1]})]- (is (= {:queryType :timeseries- :filter {:type :selector, :dimension field-name, :value 1}}- (compiled query)))- (is (= (case field- :count [1000]- :venue_price [221])- (mt/first-row (qp/process-query query))))))))))))--(deftest ^:parallel parse-filter-test- (testing "parse-filter should generate the correct filter clauses"- (mt/with-metadata-provider meta/metadata-provider- (tools.macro/macrolet [(parse-filter [filter-clause]- `(#'druid.qp/parse-filter (lib.tu.macros/$ids :checkins ~filter-clause)))]- (testing "normal non-compound filters should work as expected"- (is (= {:type :selector, :dimension "PRICE", :value 2}- (parse-filter [:= $venues.price [:value 2 {:base_type :type/Integer}]]))))- (testing "temporal filters should get stripped out"- (is (= nil- (parse-filter [:>= !checkins.date [:absolute-datetime #t "2015-09-01T00:00Z[UTC]" :default]])))- (is (= {:type :selector, :dimension "NAME", :value "Mexican"}- (parse-filter- [:and- [:= $categories.name [:value "Mexican" {:base_type :type/Text}]]- [:< !checkins.date [:absolute-datetime #t "2015-10-01T00:00Z[UTC]" :default]]]))))))))--(deftest ^:parallel multiple-filters-test- (mt/test-driver :druid- (testing "Should be able to filter by both a temporal and a non-temporal filter (#15903)"- (tqpt/with-flattened-dbdef- (is (= [4]- (mt/first-row- (mt/run-mbql-query checkins- {:aggregation [[:count]]- :filter [:and- [:= $venue_category_name "Mexican"]- [:= !month.timestamp "2015-09"]]}))))))))--(deftest ^:parallel open-ended-temporal-filter-test- (mt/test-driver :druid- (testing "Should be able to filter by an open-ended absolute temporal moment (#15902)"- (tqpt/with-flattened-dbdef- (is (= [58]- (mt/first-row- (mt/run-mbql-query checkins- {:aggregation [[:count]]- :filter [:> $timestamp "2015-10-01T00:00:00Z"]}))))))))diff --git a/modules/drivers/druid/test/metabase/driver/druid/sync_test.clj b/modules/drivers/druid/test/metabase/driver/druid/sync_test.cljdeleted file mode 100644index 6dd16223db3..00000000000--- a/modules/drivers/druid/test/metabase/driver/druid/sync_test.clj+++ /dev/null@@ -1,82 +0,0 @@-(ns ^:mb/driver-tests metabase.driver.druid.sync-test- (:require- [clojure.test :refer :all]- [malli.error :as me]- [metabase.driver :as driver]- [metabase.driver.druid.client :as druid.client]- [metabase.driver.druid.sync :as druid.sync]- [metabase.query-processor.timeseries-test.util :as tqpt]- [metabase.secrets.models.secret :as secret]- [metabase.sync.sync-metadata.dbms-version :as sync-dbms-ver]- [metabase.test :as mt]- [metabase.util :as u]- [metabase.util.malli.registry :as mr]- [toucan2.core :as t2]))--(deftest ^:parallel sync-test- (mt/test-driver :druid- (tqpt/with-flattened-dbdef- (testing "describe-database"- (is (= {:tables #{{:schema nil, :name "checkins"}}}- (driver/describe-database :druid (mt/db)))))- (testing "describe-table"- (is (= {:schema nil- :name "checkins"- :fields [{:name "timestamp", :base-type :type/Instant, :database-type "timestamp", :database-position 0, :pk? false}- {:name "venue_name", :base-type :type/Text, :database-type "STRING", :database-position 1}- {:name "user_password", :base-type :type/Text, :database-type "STRING", :database-position 2}- {:name "venue_longitude", :base-type :type/Float, :database-type "DOUBLE", :database-position 3}- {:name "venue_latitude", :base-type :type/Float, :database-type "DOUBLE", :database-position 4}- {:name "venue_price", :base-type :type/Integer, :database-type "LONG", :database-position 5}- {:name "venue_category_name", :base-type :type/Text, :database-type "STRING", :database-position 6}- {:name "id", :base-type :type/Integer, :database-type "LONG", :database-position 7}- {:name "count", :base-type :type/Integer, :database-type "LONG [metric]", :database-position 8}- {:name "unique_users", :base-type :type/DruidHyperUnique, :database-type "hyperUnique [metric]", :database-position 9}- {:name "user_name", :base-type :type/Text, :database-type "STRING", :database-position 10}- {:name "user_last_login", :base-type :type/Text, :database-type "STRING", :database-position 11}]}- (-> (driver/describe-table :druid (mt/db) {:name "checkins"})- (update :fields (partial sort-by :database-position)))))))))--(defn- db-dbms-version [db-or-id]- (t2/select-one-fn :dbms_version :model/Database :id (u/the-id db-or-id)))--(defn- check-dbms-version [dbms-version]- (me/humanize (mr/explain sync-dbms-ver/DBMSVersion dbms-version)))--(deftest dbms-version-test- (mt/test-driver :druid- (testing (str "This tests populating the dbms_version field for a given database."- " The sync happens automatically, so this test removes it first"- " to ensure that it gets set when missing.")- (tqpt/with-flattened-dbdef- (let [db (mt/db)- version-on-load (db-dbms-version db)- _ (t2/update! :model/Database (u/the-id db) {:dbms_version nil})- db (t2/select-one :model/Database :id (u/the-id db))- version-after-update (db-dbms-version db)- _ (sync-dbms-ver/sync-dbms-version! db)]- (testing "On startup is the dbms-version specified?"- (is (nil? (check-dbms-version version-on-load))))- (testing "Check to make sure the test removed the timezone"- (is (nil? version-after-update)))- (testing "Check that the value was set again after sync"- (is (nil? (check-dbms-version (db-dbms-version db))))))))))--(deftest ^:synchronized auth-dbms-version-test- (mt/test-driver- :druid- (testing "`dbms-version` uses auth parameters (#41579)"- (tqpt/with-flattened-dbdef- (let [get-args (volatile! nil)- db-with-auth-details (update (mt/db) :details assoc- :auth-enabled true- :auth-username "admin"- :auth-token-value "password1")]- ;; Following ensures that auth parameters are passed to GET during version sync if present.- (with-redefs [secret/value-as-string (constantly "password1")- druid.client/GET (fn [_url & params] (vreset! get-args (apply hash-map params)) nil)]- ;; Just fill in the params with internally modified `dbms-version`.- (druid.sync/dbms-version db-with-auth-details)- (is (true? (:auth-enabled @get-args)))- (is (= "admin" (:auth-username @get-args)))- (is (= "password1" (:auth-token-value @get-args)))))))))diff --git a/modules/drivers/druid/test/metabase/test/data/druid.clj b/modules/drivers/druid/test/metabase/test/data/druid.cljdeleted file mode 100644index 8e9988a4465..00000000000--- a/modules/drivers/druid/test/metabase/test/data/druid.clj+++ /dev/null@@ -1,60 +0,0 @@-(ns metabase.test.data.druid- (:require- [clojure.string :as str]- [metabase.driver.druid.client :as druid.client]- [metabase.test.data.impl :as data.impl]- [metabase.test.data.interface :as tx]- [metabase.util :as u]))--(set! *warn-on-reflection* true)--(tx/add-test-extensions! :druid)--(defn- host []- ;; force Druid to use localhost for now -- this is until we remove the env var in CircleCI so it doesn't override- ;; this value.- "http://localhost"- #_(let [host (tx/db-test-env-var-or-throw :druid :host "localhost")]- (cond->> host- (not (str/starts-with? host "http")) (str "http://"))))--(defn- broker-port []- (Integer/parseUnsignedInt (tx/db-test-env-var-or-throw :druid :port "8082")))--(defn- broker-port-up? []- (u/host-port-up? (str/replace (host) #"^https?://" "") (broker-port)))--(defmethod tx/dbdef->connection-details :druid- [& _]- {:host (host)- :port (broker-port)})--(defn- datasources-endpoint []- (format "%s:%d/druid/v2/datasources" (host) (broker-port)))--(defn- datasources []- {:pre [(broker-port-up?)]}- (druid.client/GET (datasources-endpoint)))--(defn- already-loaded []- (set (datasources)))--(defmethod tx/create-db! :druid- [_ dbdef & _]- (let [{:keys [database-name], :as _dbdef} (tx/get-dataset-definition dbdef)]- (assert (= database-name "checkins")- "Druid tests currently only support the flattened test-data dataset.")- (assert (contains? (already-loaded) "checkins")- "Expected 'checkins' dataset to be present in Druid datasources. (This should be loaded as part of building Docker image)")- nil))--;; NO-OP-(defmethod tx/destroy-db! :druid- [& _]- nil)--;; no-op -- because the names of the columns actually loaded by Druid differ from ones in the database definition, the-;; default impl will fail. TODO -- we should write an implementation that works for Druid-(defmethod data.impl/verify-data-loaded-correctly :druid- [_ _ _]- nil)diff --git a/resources/migrations/063/20260602_migrate_legacy_druid_driver.yaml b/resources/migrations/063/20260602_migrate_legacy_druid_driver.yamlnew file mode 100644index 00000000000..2cd50c83fd8--- /dev/null+++ b/resources/migrations/063/20260602_migrate_legacy_druid_driver.yaml@@ -0,0 +1,14 @@+## TODO: quick description of what feature/bug this migration is for++databaseChangeLog:+ - changeSet:+ id: v63.9lrcl8+ author: camsaul+ comment: Migrate databases using legacy druid driver to druid-jdbc+ changes:+ - sql:+ sql: >-+ UPDATE metabase_database+ SET engine = 'druid-jdbc'+ WHERE engine = 'druid';+ rollback: # nothing to dodiff --git a/src/metabase/analyze/classifiers/name.clj b/src/metabase/analyze/classifiers/name.cljindex 8f74c9752bf..f101ec5958b 100644--- a/src/metabase/analyze/classifiers/name.clj+++ b/src/metabase/analyze/classifiers/name.clj@@ -4,7 +4,6 @@[clojure.string :as str][metabase.analyze.schema :as analyze.schema][metabase.config.core :as config]- [metabase.driver.util :as driver.u][metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint][metabase.sync.util :as sync-util][metabase.util :as u]@@ -189,7 +188,4 @@(when (re-find pattern table-name)type))entity-types-patterns)- (case (some-> (:db_id table) driver.u/database->driver)- :druid :entity/EventTable- nil):entity/GenericTable))))diff --git a/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj b/test/metabase/driver/sql_jdbc/sync/describe_table_test.cljindex e96bf366b9f..12ce3db1ef5 100644--- a/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj+++ b/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj@@ -780,7 +780,6 @@(doseq [driver [:presto-jdbc:h2 ;; TODO- :druid:druid-jdbc]](defmethod driver/database-supports? [driver ::describe-view-fields][_driver _feature _database]@@ -790,7 +789,6 @@:h2 ;; TODO:snowflake ;; Requires enterprise account:presto-jdbc- :druid:druid-jdbc:mysql:sqlserverdiff --git a/test/metabase/query_processor/alternative_date_test.clj b/test/metabase/query_processor/alternative_date_test.cljindex ca657a18ac5..efa98ac9c23 100644--- a/test/metabase/query_processor/alternative_date_test.clj+++ b/test/metabase/query_processor/alternative_date_test.clj@@ -425,7 +425,6 @@:bigquery-cloud-sdk:clickhouse:databricks- :druid:snowflake:sparksql:sqlserverdiff --git a/test/metabase/query_processor/timeseries_test/util.clj b/test/metabase/query_processor/timeseries_test/util.cljindex 65283ecabc6..8deeb9a9f6d 100644--- a/test/metabase/query_processor/timeseries_test/util.clj+++ b/test/metabase/query_processor/timeseries_test/util.clj@@ -2,11 +2,12 @@"Utility functions and macros for testing timeseries database drivers, such as Druid."(:require[metabase.test :as mt]+ [metabase.test.data :as test.data][metabase.test.data.dataset-definitions :as defs][metabase.test.data.interface :as tx]))(defn timeseries-drivers []- #{:druid :druid-jdbc})+ test.data/timeseries-drivers)(def flattened-db-def"The normal test-data DB definition as a flattened, single-table DB definition."diff --git a/test/metabase/test/data.clj b/test/metabase/test/data.cljindex 13a0d2c9358..22062e95f28 100644--- a/test/metabase/test/data.clj+++ b/test/metabase/test/data.clj@@ -322,7 +322,7 @@;; Non-"normal" timeseries drivers are tested in [[metabase.query-processor.timeseries-test]] and elsewhere(def timeseries-drivers"Drivers that are so weird that we can't use the standard dataset loading against them."- #{:druid :druid-jdbc})+ #{:druid-jdbc})(mr/def ::driver-selector[:map {:closed true}