Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Remove legacy `:druid` driver / migrate to `:druid-jdbc` (#75143)

Cam Saul4eb1408f48cd
diff --git a/.claude/agents/drivers-and-sync-backend-expert.md b/.claude/agents/drivers-and-sync-backend-expert.md
index 80bb2ee43bc..85f00b9aa4a 100644
--- a/.claude/agents/drivers-and-sync-backend-expert.md
+++ b/.claude/agents/drivers-and-sync-backend-expert.md
@@ -22,7 +22,6 @@ You understand Metabase's driver architecture built on Clojure multimethods with
:bigquery-cloud-sdk, :databricks, :clickhouse, :athena, :sparksql,
:presto-jdbc / :starburst, :vertica, :sqlite, :h2
:mongo (non-SQL, custom protocol)
-:druid (non-SQL, REST-based)
```
Key driver extension points you know intimately:
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index c5bc1c167ce..d635e274342 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -249,7 +249,6 @@
{:team "Graphy"
:api #{metabase.analyze.core}
:uses #{config
- driver
lib
models
query-processor
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index 501b96243cd..3971e4556e3 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -296,54 +296,6 @@ jobs:
upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
- be-tests-druid-ee:
- needs: determine-driver-skips
- if: ${{ needs.determine-driver-skips.outputs.druid-should-run == 'true' || needs.determine-driver-skips.outputs.druid-quarantine-conflict == 'true' }}
- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 40
- env:
- CI: 'true'
- DRIVERS: druid
- services:
- druid:
- image: metabase/druid:0.20.2
- ports:
- - "8082:8082"
- env:
- CLUSTER_SIZE: nano-quickstart
- name: Druid (Legacy)
- steps:
- - name: Fail on quarantine conflict
- if: ${{ needs.determine-driver-skips.outputs.druid-quarantine-conflict == 'true' }}
- run: |
- echo "::error::Druid driver is quarantined but you changed files in modules/drivers/druid/"
- echo ""
- echo "Your changes will NOT be tested because the driver is currently quarantined."
- echo "To run Druid tests, add the label: break-quarantine-druid"
- echo ""
- echo "If you merge without testing, you risk breaking the Druid driver!"
- exit 1
- - uses: actions/checkout@v6
- - name: Test Druid driver
- id: test-driver
- uses: ./.github/actions/test-driver
- with:
- junit-name: 'be-tests-druid-ee'
- test-args: >-
- :only-tags [:mb/driver-tests]
- :exclude-tags [:mb/transforms-python-test]
- - name: Upload Test Results
- uses: ./.github/actions/upload-test-results
- if: always()
- with:
- input-path: ./target/junit/
- output-name: ${{ github.job }}
- bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}
- role-to-assume: ${{ secrets.CI_TEST_RESULTS_IAM_ROLE }}
- aws-region: ${{ vars.AWS_REGION }}
- trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
- previous-step-outcome: ${{ steps.test-driver.outputs.test-outcome }}
-
be-tests-druid-jdbc-ee:
needs: determine-driver-skips
if: ${{ needs.determine-driver-skips.outputs.druid-jdbc-should-run == 'true' || needs.determine-driver-skips.outputs.druid-jdbc-quarantine-conflict == 'true' }}
@@ -1629,7 +1581,6 @@ jobs:
# these are not reliable, but we want to run them anyway to gather data about failures
# - be-tests-snowflake-ee
# - be-tests-bigquery-cloud-sdk-ee
- - be-tests-druid-ee
- be-tests-databricks-ee
- be-tests-druid-jdbc-ee
- be-tests-mysql-mariadb
diff --git a/.github/workflows/snyk.yml b/.github/workflows/snyk.yml
index 50f830c3c1a..9291dbcde67 100644
--- a/.github/workflows/snyk.yml
+++ b/.github/workflows/snyk.yml
@@ -107,9 +107,6 @@ jobs:
- name: Driver - Databricks
file: modules/drivers/databricks/pom.xml
category: driver-databricks
- - name: Driver - Druid
- file: modules/drivers/druid/pom.xml
- category: driver-druid
- name: Driver - Druid JDBC
file: modules/drivers/druid-jdbc/pom.xml
category: driver-druid-jdbc
diff --git a/bin/build/src/i18n/enumerate.clj b/bin/build/src/i18n/enumerate.clj
index eca36140616..0b41f70d9c3 100644
--- a/bin/build/src/i18n/enumerate.clj
+++ b/bin/build/src/i18n/enumerate.clj
@@ -19,7 +19,6 @@
["/src"
"/enterprise/backend/src"
"/modules/drivers/bigquery-cloud-sdk/src"
- "/modules/drivers/druid/src"
"/modules/drivers/druid-jdbc/src"
"/modules/drivers/mongo/src"
"/modules/drivers/oracle/src"
diff --git a/deps.edn b/deps.edn
index 8ab8d42c7a9..45927a5eb83 100644
--- a/deps.edn
+++ b/deps.edn
@@ -497,7 +497,6 @@
"modules/drivers/bigquery-cloud-sdk/test"
"modules/drivers/clickhouse/test"
"modules/drivers/databricks/test"
- "modules/drivers/druid/test"
"modules/drivers/druid-jdbc/test"
"modules/drivers/hive-like/test"
"modules/drivers/mongo/test"
@@ -561,8 +560,6 @@
"modules/drivers/databricks/test"
"modules/drivers/druid-jdbc/src"
"modules/drivers/druid-jdbc/test"
- "modules/drivers/druid/src"
- "modules/drivers/druid/test"
"modules/drivers/hive-like/src"
"modules/drivers/hive-like/test"
"modules/drivers/mongo/src"
@@ -600,7 +597,6 @@
"modules/drivers/bigquery-cloud-sdk/src"
"modules/drivers/clickhouse/src"
"modules/drivers/databricks/src"
- "modules/drivers/druid/src"
"modules/drivers/druid-jdbc/src"
"modules/drivers/hive-like/src"
"modules/drivers/mongo/src"
@@ -626,7 +622,6 @@
"modules/drivers/bigquery-cloud-sdk/src"
"modules/drivers/clickhouse/src"
"modules/drivers/databricks/src"
- "modules/drivers/druid/src"
"modules/drivers/druid-jdbc/src"
"modules/drivers/hive-like/src"
"modules/drivers/mongo/src"
diff --git a/docs/databases/connections/druid.md b/docs/databases/connections/druid.md
index 4536edce8c6..e0894ae8cdd 100644
--- a/docs/databases/connections/druid.md
+++ b/docs/databases/connections/druid.md
@@ -4,8 +4,6 @@ title: Druid
# Druid
-> Prefer the Druid JDBC connection.
-
To add a database connection, click the **grid** icon in the top right, and navigate to **Admin** > **Databases** > **Add a database**.
## Edit connection details
@@ -34,16 +32,12 @@ See our [guide to SSH tunneling](../ssh-tunnel.md).
### Allow unfolding of JSON columns
-> Only available for the Druid JDBC connection.
-
This enables unfolding of JSON columns into their component fields.
See [JSON unfolding](../../data-modeling/json-unfolding.md).
### Authentication
-> Only available for the Druid JDBC connection.
-
Toggle on to enable basic authentication with a username and password.
### Re-run queries for simple explorations
diff --git a/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts b/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts
index 1f9c53fd52e..ddca0407faa 100644
--- a/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts
+++ b/e2e/test/scenarios/admin/database-connection-strings.cy.spec.ts
@@ -51,12 +51,12 @@ const databaseTestCases = [
],
},
{
- engine: "Druid",
+ engine: "Druid JDBC",
connectionString:
"jdbc:avatica:remote:url=http://localhost:8888/druid/v2/sql/avatica/;transparent_reconnection=true",
expectedFields: [
{ label: "Host", value: "localhost" },
- { label: "Broker node port", value: "8888" },
+ { label: "Router node port", value: "8888" },
],
},
{
diff --git a/e2e/test/scenarios/admin/databases.cy.spec.js b/e2e/test/scenarios/admin/databases.cy.spec.js
index d0011b6efae..e0228f5a548 100644
--- a/e2e/test/scenarios/admin/databases.cy.spec.js
+++ b/e2e/test/scenarios/admin/databases.cy.spec.js
@@ -449,7 +449,7 @@ describe("database page > side panel", () => {
{ name: "Amazon Redshift", file: "redshift" },
{ name: "ClickHouse", file: "clickhouse" },
{ name: "Databricks", file: "databricks" },
- { name: "Druid", file: "druid" },
+ { name: "Druid JDBC", file: "druid" },
{ name: "MongoDB", file: "mongo" },
{ name: "MySQL", file: "mysql" },
{ name: "PostgreSQL", file: "postgresql" },
diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.clj
index 4eb0033c361..957f456fcc4 100644
--- a/mage/src/mage/modules.clj
+++ b/mage/src/mage/modules.clj
@@ -266,7 +266,6 @@
:bigquery
:clickhouse
:databricks
- :druid
:druid-jdbc
:mongo
:mongo-ssl
@@ -289,7 +288,6 @@
"bigquery-cloud-sdk" [:bigquery]
"clickhouse" [:clickhouse]
"databricks" [:databricks]
- "druid" [:druid]
"druid-jdbc" [:druid-jdbc]
"mongo" [:mongo :mongo-ssl :mongo-sharded-cluster]
"oracle" [:oracle]
diff --git a/modules/drivers/deps.edn b/modules/drivers/deps.edn
index 98f65a210d0..b23239e9ae5 100644
--- a/modules/drivers/deps.edn
+++ b/modules/drivers/deps.edn
@@ -11,7 +11,6 @@
metabase/bigquery-cloud-sdk {:local/root "bigquery-cloud-sdk"}
metabase/clickhouse {:local/root "clickhouse"}
metabase/databricks {:local/root "databricks"}
- metabase/druid {:local/root "druid"}
metabase/druid-jdbc {:local/root "druid-jdbc"}
metabase/hive-like {:local/root "hive-like"}
metabase/mongo {:local/root "mongo"}
diff --git a/modules/drivers/druid/deps.edn b/modules/drivers/druid/deps.edn
deleted file mode 100644
index aa7725e18a5..00000000000
--- a/modules/drivers/druid/deps.edn
+++ /dev/null
@@ -1 +0,0 @@
-{:paths ["src" "resources"]}
diff --git a/modules/drivers/druid/resources/metabase/druid/metabase-plugin.yaml b/modules/drivers/druid/resources/metabase/druid/metabase-plugin.yaml
deleted file mode 100644
index dd892dcf94a..00000000000
--- a/modules/drivers/druid/resources/metabase/druid/metabase-plugin.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
-info:
- name: Metabase Druid Driver
- version: 1.0.0-SNAPSHOT
- description: Allows Metabase to connect to Druid databases.
-driver:
- name: druid
- display-name: Druid
- lazy-load: true
- connection-properties:
- - group:
- container-style: ["grid", "3fr 1fr"]
- fields:
- - merge:
- - host
- - placeholder: http://localhost
- required: true
- - merge:
- - port
- - display-name: Broker node port
- placeholder: 8082
- required: true
- - cloud-ip-address-info
- - group:
- container-style: ["component", "backdrop"]
- fields:
- - ssh-tunnel
- - advanced-options-start
- - name: auth-enabled
- display-name: Authentication header
- default: false
- type: boolean
- visible-if:
- advanced-options: true
- - name: auth-username
- display-name: Username
- type: string
- visible-if:
- auth-enabled: true
- - name: auth-token
- display-name: Token
- type: secret
- secret-kind: password
- visible-if:
- auth-enabled: true
- - default-advanced-options
-init:
- - step: load-namespace
- namespace: metabase.driver.druid
diff --git a/modules/drivers/druid/src/metabase/driver/druid.clj b/modules/drivers/druid/src/metabase/driver/druid.clj
deleted file mode 100644
index 95f02446343..00000000000
--- a/modules/drivers/druid/src/metabase/driver/druid.clj
+++ /dev/null
@@ -1,68 +0,0 @@
-(ns metabase.driver.druid
- "Druid driver."
- (:require
- [clj-http.client :as http]
- [metabase.driver :as driver]
- [metabase.driver-api.core :as driver-api]
- [metabase.driver.druid.client :as druid.client]
- [metabase.driver.druid.execute :as druid.execute]
- [metabase.driver.druid.query-processor :as druid.qp]
- [metabase.driver.druid.sync :as druid.sync]
- [metabase.driver.settings :as driver.settings]
- [metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh]
- [metabase.util.json :as json]))
-
-(driver/register! :druid)
-
-(doseq [[feature supported?] {:expression-aggregations true
- :expression-literals true
- :schemas false
- :set-timezone true
- :temporal/requires-default-unit true
- :database-routing true}]
- (defmethod driver/database-supports? [:druid feature] [_driver _feature _db] supported?))
-
-(defmethod driver/can-connect? :druid
- [_ details]
- {:pre [(map? details)]}
- (ssh/with-ssh-tunnel [details-with-tunnel details]
- (let [{:keys [auth-enabled auth-username auth-token-value]} details]
- (= 200 (:status (http/get (druid.client/details->url details-with-tunnel "/status")
- (cond-> {}
- auth-enabled (assoc :basic-auth (str auth-username ":" auth-token-value)))))))))
-
-(defmethod driver/describe-table :druid
- [_ database table]
- (druid.sync/describe-table database table))
-
-(defmethod driver/dbms-version :druid
- [_ database]
- (druid.sync/dbms-version database))
-
-(defmethod driver/describe-database* :druid
- [_ database]
- (druid.sync/describe-database database))
-
-(defmethod driver/mbql->native :druid
- [_ query]
- (druid.qp/mbql->native query))
-
-(defn- add-timeout-to-query [query timeout]
- (let [parsed (if (string? query)
- (json/decode+kw query)
- query)]
- (assoc-in parsed [:context :timeout] timeout)))
-
-(defmethod driver/execute-reducible-query :druid
- [_driver query _context respond]
- (druid.execute/execute-reducible-query
- (partial druid.client/do-query-with-cancellation (driver-api/canceled-chan))
- (update-in query [:native :query] add-timeout-to-query driver.settings/*query-timeout-ms*)
- respond))
-
-(defmethod driver/db-start-of-week :druid
- [_]
- :sunday)
-
-(defmethod driver/llm-sql-dialect-resource :druid [_]
- "metabot/prompts/dialects/druid.md")
diff --git a/modules/drivers/druid/src/metabase/driver/druid/client.clj b/modules/drivers/druid/src/metabase/driver/druid/client.clj
deleted file mode 100644
index 9886df99f38..00000000000
--- a/modules/drivers/druid/src/metabase/driver/druid/client.clj
+++ /dev/null
@@ -1,120 +0,0 @@
-(ns metabase.driver.druid.client
- (:refer-clojure :exclude [get-in])
- (:require
- [clj-http.client :as http]
- [clojure.core.async :as a]
- [metabase.driver-api.core :as driver-api]
- [metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh]
- [metabase.util :as u]
- [metabase.util.i18n :refer [tru]]
- [metabase.util.json :as json]
- [metabase.util.log :as log]
- [metabase.util.performance :refer [get-in]]))
-
-(set! *warn-on-reflection* true)
-
-(defn details->url
- "Helper for building a Druid URL.
-
- (details->url {:host \"http://localhost\", :port 8082} \"druid/v2\") -> \"http://localhost:8082/druid/v2\""
- [{:keys [host port]} & strs]
- {:pre [(string? host) (seq host) (integer? port)]}
- (apply str (format "%s:%d" host port) (map name strs)))
-
-(defn- do-request
- "Perform a JSON request using `request-fn` against `url`.
-
- (do-request http/get \"http://my-json-api.net\")"
- [request-fn url & {:as options}]
- {:pre [(fn? request-fn) (string? url)]}
- ;; this is the way the `Content-Type` header is formatted in requests made by the Druid web interface
- (let [{:keys [auth-enabled auth-username auth-token-value]} options
- options (cond-> (merge {:content-type "application/json;charset=UTF-8"} options)
- (:body options) (update :body json/encode)
- auth-enabled (assoc :basic-auth (str auth-username ":" auth-token-value)))]
- (try
- (let [{:keys [status body]} (request-fn url options)]
- (when (not= status 200)
- (throw (ex-info (tru "Druid request error [{0}]: {1}" status (pr-str body))
- {:type driver-api/qp.error-type.db})))
- (try
- (json/decode+kw body)
- (catch Throwable e
- (throw (ex-info (tru "Failed to parse Druid response body: {0}" (pr-str body))
- {:type driver-api/qp.error-type.db}
- e)))))
- (catch Throwable e
- (let [response (u/ignore-exceptions
- (when-let [body (:body (ex-data e))]
- (json/decode+kw body)))]
- (throw (ex-info (or (:errorMessage response)
- (.getMessage e))
- (merge
- {:type driver-api/qp.error-type.db
- :request-url url
- :request-options options}
- (when response
- {:response response}))
- e)))))))
-
-(def ^{:arglists '([url & {:as options}]), :style/indent [:form]} GET "Execute a GET request." (partial do-request http/get))
-(def ^{:arglists '([url & {:as options}]), :style/indent [:form]} POST "Execute a POST request." (partial do-request http/post))
-(def ^{:arglists '([url & {:as options}]), :style/indent [:form]} DELETE "Execute a DELETE request." (partial do-request http/delete))
-
-(defn do-query
- "Run a Druid `query` against database connection `details`."
- [details query]
- {:pre [(map? details) (map? query)]}
- (ssh/with-ssh-tunnel [details-with-tunnel details]
- (try
- (POST (details->url details-with-tunnel "/druid/v2"),
- :body query
- :auth-enabled (:auth-enabled details)
- :auth-username (:auth-username details)
- :auth-token-value (driver-api/secret-value-as-string :druid details "auth-token"))
- ;; don't need to do anything fancy if the query was killed
- (catch InterruptedException e
- (throw e))
- (catch Throwable e
- (let [e' (ex-info (.getMessage e)
- {:type driver-api/qp.error-type.db
- :query query}
- e)]
- (log/error e' "Error running query")
- ;; Re-throw a new exception with `message` set to the extracted message
- (throw e'))))))
-
-(defn- undo-query-with-id! [details query-id]
- (if-not query-id
- (log/warn "Client closed connection, no queryId found, can't cancel query")
- (ssh/with-ssh-tunnel [details-with-tunnel details]
- (log/warnf "Client closed connection, canceling Druid queryId %s" query-id)
- (try
- (log/debugf "Canceling Druid query with ID %s" query-id)
- (DELETE (details->url details-with-tunnel (format "/druid/v2/%s" query-id))
- :auth-enabled (:auth-enabled details)
- :auth-username (:auth-username details)
- :auth-token-value (driver-api/secret-value-as-string :druid details "auth-token"))
- (catch Exception cancel-e
- (log/warnf cancel-e "Failed to cancel Druid query with queryId %s" query-id))))))
-
-(defn do-query-with-cancellation
- "Run a Druid `query`, canceling it if `canceled-chan` gets a message."
- [canceled-chan details query]
- {:pre [(map? details) (map? query)]}
- (let [query-id (get-in query [:context :queryId])
- ;; Run the query in a future so that we can interrupt it from the canceled-chan callback.
- query-fut (future (do-query details query))
- undo! #(undo-query-with-id! details query-id)]
- (a/go
- (when (a/<! canceled-chan)
- (future-cancel query-fut)
- (undo!)))
- (try @query-fut
- (catch java.util.concurrent.ExecutionException e
- (undo!)
- ;; Unwrap exceptions from the future.
- (throw (.getCause e)))
- (catch InterruptedException e
- (undo!)
- (throw e)))))
diff --git a/modules/drivers/druid/src/metabase/driver/druid/execute.clj b/modules/drivers/druid/src/metabase/driver/druid/execute.clj
deleted file mode 100644
index ecc551aa201..00000000000
--- a/modules/drivers/druid/src/metabase/driver/druid/execute.clj
+++ /dev/null
@@ -1,185 +0,0 @@
-(ns metabase.driver.druid.execute
- (:require
- [clojure.math.numeric-tower :as math]
- [java-time.api :as t]
- [medley.core :as m]
- [metabase.driver-api.core :as driver-api]
- [metabase.driver.connection :as driver.conn]
- [metabase.driver.druid.query-processor :as druid.qp]
- [metabase.util :as u]
- [metabase.util.date-2 :as u.date]
- [metabase.util.i18n :refer [tru]]
- [metabase.util.json :as json]
- [metabase.util.malli :as mu]))
-
-(set! *warn-on-reflection* true)
-
-(defn- resolve-timezone
- "Returns the timezone object (either report-timezone or JVM timezone). Returns nil if the timezone is UTC as the
- timestamps from Druid are already in UTC and don't need to be converted"
- [_]
- (when-not (= (t/zone-id (driver-api/results-timezone-id)) (t/zone-id "UTC"))
- (driver-api/results-timezone-id)))
-
-(defmulti ^:private post-process
- "Do appropriate post-processing on the results of a query based on the `query-type`."
- {:arglists '([query-type projections timezone-and-middleware-settings results])}
- (fn [query-type _ _ _]
- query-type))
-
-(defmethod post-process ::druid.qp/scan
- [_ projections _ results]
- {:projections projections
- :results (for [{:keys [events]} results
- {timestamp :__time, :as event} events]
- (if-not timestamp
- event
- (-> (assoc event :timestamp (t/instant timestamp))
- (dissoc :__time))))})
-
-;; SELECT queries are no longer supported as of Druid 0.17.0, and no longer generated by Metabase as of 0.35.0, but if
-;; someone has a native select query on Druid < 0.17.0 we can continue to parse it the way we did in the past.
-;;
-;; We can remove this at some point in the future when everyone is using Druid 0.17.0 or above.
-(defmethod post-process ::druid.qp/select
- [_ projections _ [{{:keys [events]} :result}]]
- {:projections projections
- :results (for [event (map :event events)]
- (update event :timestamp u.date/parse))})
-
-(defmethod post-process ::druid.qp/total
- [_ projections _ results]
- {:projections projections
- :results (map :result results)})
-
-(defmethod post-process ::druid.qp/topN
- [_ projections {:keys [middleware]} results]
- {:projections projections
- :results (let [results (-> results first :result)]
- (if (:format-rows? middleware true)
- results
- (map #(m/update-existing % :timestamp u.date/parse) results)))})
-
-(defmethod post-process ::druid.qp/groupBy
- [_ projections {:keys [middleware]} results]
- {:projections projections
- :results (if (:format-rows? middleware true)
- (map :event results)
- (map (comp #(m/update-existing % :timestamp u.date/parse)
- :event)
- results))})
-
-(defmethod post-process ::druid.qp/timeseries
- [_ projections {:keys [middleware]} results]
- {:projections (conj projections :timestamp)
- :results (let [ts-getter (if (:format-rows? middleware true)
- :timestamp
- (comp u.date/parse :timestamp))]
- (for [event results]
- (merge {:timestamp (ts-getter event)} (:result event))))})
-
-(mu/defn- col-names->getter-fns :- [:sequential [:or :keyword fn?]]
- "Given a sequence of `columns` keywords, return a sequence of appropriate getter functions to get values from a single
- result row. Normally, these are just the keyword column names themselves, but for `:timestamp___int`, we'll also
- parse the result as an integer (for further explanation, see the docstring for
- `units-that-need-post-processing-int-parsing`). We also round `:distinct___count` in order to return an integer
- since Druid returns the approximate floating point value for cardinality queries (See Druid documentation regarding
- cardinality and HLL)."
- [actual-col-names :- [:sequential :keyword]
- annotate-col-names :- [:sequential :keyword]]
- (let [annotate-col-names (set annotate-col-names)]
- (filter
- some?
- (for [k actual-col-names]
- (case k
- :distinct___count (fn get-distinct-count [row]
- (some-> (get row :distinct___count) math/round))
- :timestamp___int (fn get-timestamp-int [row]
- (some-> (get row :timestamp___int) Integer/parseInt))
- (when (contains? annotate-col-names k)
- k))))))
-
-(defn- result-metadata [col-names]
- ;; rename any occurrences of `:timestamp___int` to `:timestamp` in the results so the user doesn't know about
- ;; our behind-the-scenes conversion and apply any other post-processing on the value such as parsing some
- ;; units to int and rounding up approximate cardinality values.
- (let [fixed-col-names (for [col-name col-names]
- (case col-name
- :timestamp___int :timestamp
- :distinct___count :count
- col-name))]
- {:cols (vec (for [col-name fixed-col-names]
- {:name (u/qualified-name col-name)
- ;; Placeholder so metadata is well formed, we'll infer actual types later
- :base_type :type/*}))}))
-
-(defn- result-rows [{rows :results, :as results} actual-col-names annotate-col-names]
- (let [getters (vec (col-names->getter-fns actual-col-names annotate-col-names))]
- (when-not (seq getters)
- (throw (ex-info (tru "Don''t know how to retrieve results for columns {0}" (pr-str actual-col-names))
- {:type driver-api/qp.error-type.driver
- :results results})))
- (map (apply juxt getters) rows)))
-
-(defn- remove-bonus-keys
- "Remove keys that start with `___` from the results -- they were temporary, and we don't want to return them."
- [columns]
- (vec (remove #(re-find #"^___" (name %)) columns)))
-
-(defn- reduce-results
- [{{:keys [mbql?]} :native, :as outer-query} {:keys [projections], :as result} respond]
- (let [col-names (if mbql?
- (->> projections
- remove-bonus-keys
- vec)
- (-> result :results first keys))
- metadata (result-metadata col-names)
- annotate-col-names (map (comp keyword :name) (driver-api/merged-column-info outer-query (when-not mbql?
- metadata)))
- rows (result-rows result col-names annotate-col-names)
- base-types (transduce identity (driver-api/base-type-inferer metadata) rows)
- metadata (update metadata :cols (partial map (fn [col base-type]
- (assoc col :base_type base-type)))
- base-types)]
- (respond metadata rows)))
-
-(defn execute-reducible-query
- "Execute a query for a Druid DB."
- [execute*
- {{:keys [query query-type projections]} :native
- middleware :middleware
- :as mbql-query}
- respond]
- {:pre [query]}
- (let [database (-> (driver-api/metadata-provider) driver-api/database)
- details (driver.conn/effective-details database)
- query (if (string? query)
- (json/decode+kw query)
- query)
- query-type (or query-type
- (keyword (namespace ::druid.qp/query) (name (:queryType query))))
- _ (driver.conn/track-connection-acquisition! details)
- results (try
- (execute* details query)
- (catch Throwable e
- (throw (ex-info (tru "Error executing query: {0}" (ex-message e))
- {:type driver-api/qp.error-type.db
- :query query}
- e))))
- result (try (post-process query-type projections
- {:timezone (resolve-timezone mbql-query)
- :middleware middleware}
- results)
- (catch Throwable e
- (throw (ex-info (tru "Error post-processing Druid query results")
- {:type driver-api/qp.error-type.driver
- :results results}
- e))))]
- (try
- (reduce-results mbql-query result respond)
- (catch Throwable e
- (throw (ex-info (tru "Error reducing Druid query results")
- {:type driver-api/qp.error-type.driver
- :results results
- :post-processed result}
- e))))))
diff --git a/modules/drivers/druid/src/metabase/driver/druid/js.clj b/modules/drivers/druid/src/metabase/driver/druid/js.clj
deleted file mode 100644
index 6f83e63499b..00000000000
--- a/modules/drivers/druid/src/metabase/driver/druid/js.clj
+++ /dev/null
@@ -1,67 +0,0 @@
-(ns metabase.driver.druid.js
- "Util fns for creating Javascript functions."
- (:refer-clojure :exclude [+ - * / or])
- (:require
- [clojure.string :as str]))
-
-(defn- ->js [x]
- {:pre [(not (coll? x))]}
- (if (keyword? x)
- (name x)
- x))
-
-(defn parens
- "Wrap `s` (presumably a string) in parens."
- ^String [s]
- (str "(" s ")"))
-
-(defn argslist
- "Generate a list of arguments, e.g. for a function declaration or function call.
- `args` are separated by commas and wrapped in parens.
-
- (argslist [:x :y]) -> \"(x, y)\""
- ^String [args]
- (parens (str/join ", " (map ->js args))))
-
-(defn return
- "Generate a javascript `return` statement. `statement-parts` are combined directly into a single string.
-
- (return :x :+ :y) -> \"return x+y;\""
- ^String [& statement-parts]
- (str "return " (apply str (map ->js statement-parts)) ";"))
-
-(defn function
- "Create a JavaScript function with `args` and `body`."
- ^String [args & body]
- (str "function" (argslist args) " { "
- (apply str body)
- " }"))
-
-(defn- arithmetic-operator
- "Interpose arithmetic `operator` between `args`, and wrap the entire expression in parens."
- ^String [operator & args]
- (parens (str/join (str " " (name operator) " ")
- (map ->js args))))
-
-(def ^{:arglists '([& args])} ^String + "Interpose `+` between `args`, and wrap the entire expression in parens." (partial arithmetic-operator :+))
-(def ^{:arglists '([& args])} ^String - "Interpose `-` between `args`, and wrap the entire expression in parens." (partial arithmetic-operator :-))
-(def ^{:arglists '([& args])} ^String * "Interpose `*` between `args`, and wrap the entire expression in parens." (partial arithmetic-operator :*))
-(def ^{:arglists '([& args])} ^String / "Interpose `/` between `args`, and wrap the entire expression in parens." (partial arithmetic-operator :/))
-
-(defn fn-call
- "Generate a JavaScript function call.
-
- (fn-call :parseFloat :x :y) -> \"parseFloat(x, y)\""
- ^String [fn-name & args]
- (str (name fn-name) (argslist args)))
-
-(def ^{:arglists '([x])} ^String parse-float
- "Generate a call to the JavaScript `parseFloat` function."
- (partial fn-call :parseFloat))
-
-(defn or
- "Interpose the JavaScript or operator (`||`) between `args`, and wrap the entire expression in parens.
-
- (or :x :y) -> \"(x || y)\""
- ^String [& args]
- (parens (str/join " || " (map ->js args))))
diff --git a/modules/drivers/druid/src/metabase/driver/druid/query_processor.clj b/modules/drivers/druid/src/metabase/driver/druid/query_processor.clj
deleted file mode 100644
index 28b294c2795..00000000000
--- a/modules/drivers/druid/src/metabase/driver/druid/query_processor.clj
+++ /dev/null
@@ -1,1216 +0,0 @@
-(ns metabase.driver.druid.query-processor
- (:refer-clojure :exclude [every? mapv some get-in])
- (:require
- [clojure.string :as str]
- [metabase.driver-api.core :as driver-api]
- [metabase.driver.common :as driver.common]
- [metabase.driver.druid.js :as druid.js]
- [metabase.lib.core :as lib]
- [metabase.util :as u]
- [metabase.util.date-2 :as u.date]
- [metabase.util.i18n :refer [tru]]
- [metabase.util.log :as log]
- [metabase.util.malli :as mu]
- [metabase.util.match :as match]
- [metabase.util.performance :refer [every? get-in mapv some]]))
-
-(set! *warn-on-reflection* true)
-
-(def ^:private ^:const topN-max-results
- "Maximum number of rows the topN query in Druid should return. Huge values cause significant issues with the engine.
-
- Coming from the default value hardcoded in the Druid engine itself
- http://druid.io/docs/latest/querying/topnquery.html"
- 1000)
-
-;; +-----> ::scan +----> :groupBy
-;; ::query ----| |
-;; +----> ::ag-query ----+----> ::topN
-;; | +----> total
-;; +----> ::timeseries ----|
-;; +----> grouped-timeseries
-
-(derive ::scan ::query)
-(derive ::ag-query ::query)
-(derive ::topN ::ag-query)
-(derive ::groupBy ::ag-query)
-(derive ::timeseries ::ag-query)
-(derive ::total ::timeseries)
-(derive ::grouped-timeseries ::timeseries)
-
-(def ^:private ^:dynamic *query*
- "The INNER part of the query currently being processed.
- (`:settings` is merged in from the outer query as well so we can access timezone info)."
- nil)
-
-(defn- query-type-dispatch-fn
- [query-type & _]
- query-type)
-
-(defmulti ^:private ->rvalue
- "Convert something to an 'rvalue`, i.e. a value that could be used in the right-hand side of an assignment expression.
-
- (let [x 100] ...) ; x is the lvalue; 100 is the rvalue"
- {:arglists '([x])}
- driver-api/dispatch-by-clause-name-or-class)
-
-(defmethod ->rvalue nil
- [_]
- nil)
-
-(defmethod ->rvalue Object
- [this]
- this)
-
-(defn- ag-clause->rvalue [[ag-type :as ag]]
- (cond
- (= [:count] ag)
- :count
-
- (= ag-type :distinct)
- :distinct___count
-
- (= ag-type :aggregation-options)
- (let [[_ wrapped-ag options] ag]
- (or (:name options) (recur wrapped-ag)))
-
- ag-type
- ag-type
-
- :else
- (throw (ex-info (tru "Unknown aggregation type!") {:aggregation ag}))))
-
-(defmethod ->rvalue :aggregation
- [[_ index]]
- (ag-clause->rvalue (or (nth (:aggregation *query*) index)
- (throw (ex-info (tru "No aggregation at index {0}" index)
- {:index index, :query *query*})))))
-
-(defmethod ->rvalue :field
- [[_ id-or-name]]
- (if (integer? id-or-name)
- (:name (driver-api/field (driver-api/metadata-provider) id-or-name))
- id-or-name))
-
-(defmethod ->rvalue :absolute-datetime
- [[_ t unit]]
- (u.date/format
- (if (= unit :default)
- t
- (u.date/truncate t unit))))
-
-;; TODO - not 100% sure how to handle times here, just treating it exactly like a date will have to do for now
-(defmethod ->rvalue :time
- [[_ t unit]]
- (u.date/format (u.date/truncate t unit)))
-
-(defmethod ->rvalue :relative-datetime
- [[_ amount unit]]
- (u.date/format (u.date/truncate (u.date/add unit amount) unit)))
-
-(defmethod ->rvalue :value
- [[_ value]]
- (->rvalue value))
-
-(defmulti ^:private dimension-or-metric?
- "Is this field clause a `:dimension` or `:metric`?"
- {:arglists '([field-clause])}
- driver-api/dispatch-by-clause-name-or-class)
-
-(defmethod dimension-or-metric? :field
- [[_ id-or-name options]]
- (let [{:keys [base-type database-type]} (if (integer? id-or-name)
- (driver-api/field (driver-api/metadata-provider) id-or-name)
- options)]
- (cond
- (str/includes? database-type "[metric]") :metric
- (isa? base-type :type/DruidHyperUnique) :metric
- :else :dimension)))
-
-(defn- ^:dynamic *random-query-id* []
- (str (random-uuid)))
-
-(defn- query-type->default-query [query-type]
- (merge
- {:intervals ["1900-01-01/2100-01-01"]
- :granularity :all
- :context {:queryId (*random-query-id*)}}
- (case query-type
- ::scan {:queryType :scan
- :limit driver-api/absolute-max-results}
- ::total {:queryType :timeseries}
- ::grouped-timeseries {:queryType :timeseries}
- ::topN {:queryType :topN
- :threshold topN-max-results}
- ::groupBy {:queryType :groupBy})))
-
-;;; ---------------------------------------------- handle-source-table -----------------------------------------------
-
-(defn- handle-source-table
- [_query-type {source-table-id :source-table} druid-query]
- (let [{source-table-name :name} (driver-api/table (driver-api/metadata-provider) source-table-id)]
- (assoc-in druid-query [:query :dataSource] source-table-name)))
-
-;;; ---------------------- handle-filter. See http://druid.io/docs/latest/querying/filters.html ----------------------
-
-(def ^:private ^{:arglists '([clause])} field?
- (partial driver-api/is-clause? :field))
-
-(defn- filter:and
- [filters]
- {:type :and
- :fields filters})
-
-(defn- filter:not
- [filtr]
- {:pre [filtr]}
- (if (= (:type filtr) :not) ; it looks like "two nots don't make an identity" with druid
- (:field filtr)
- {:type :not, :field filtr}))
-
-(defn- filter:=
- [field value]
- {:type :selector
- :dimension (->rvalue field)
- :value (->rvalue value)})
-
-(defn- filter:nil?
- [clause-or-field]
- (if (driver-api/is-clause? #{:+ :- :/ :*} clause-or-field)
- (filter:and (vec (for [arg (rest clause-or-field)
- :when (field? arg)]
- (filter:nil? arg))))
- (filter:= clause-or-field (case (dimension-or-metric? clause-or-field)
- :dimension nil
- :metric 0))))
-
-(defn- filter:like
- "Build a `like` filter clause, which is almost just like a SQL `LIKE` clause."
- [field pattern case-sensitive?]
- {:type :like
- :dimension (->rvalue field)
- ;; tell Druid to use backslash as an escape character
- :escape "\\"
- ;; if this is a case-insensitive search we'll lower-case the search pattern and add an extraction function to
- ;; lower-case the dimension values we're matching against
- :pattern (cond-> pattern
- (not case-sensitive?) u/lower-case-en)
- :extractionFn (when-not case-sensitive?
- {:type :lower})})
-
-(defn- escape-like-filter-pattern
- "Escape `%`, `_`, and backslash symbols that aren't meant to have special meaning in `like` filters
- patterns. Backslashes wouldn't normally have a special meaning, but we specify backslash as our escape character in
- the `filter:like` function above, so they need to be escaped as well."
- [s]
- (str/replace s #"([%_\\])" "\\\\$1"))
-
-(defn- filter:bound
- "Numeric `bound` filter, for finding values of `field` that are less than some value-or-field, greater than some
- value-or-field, or both. Defaults to being `inclusive` (e.g. `<=` instead of `<`) but specify option `inclusive?` to
- change this."
- [field & {:keys [lower upper inclusive?]
- :or {inclusive? true}}]
- {:type :bound
- :ordering :numeric
- :dimension (->rvalue field)
- :lower (num (->rvalue lower))
- :upper (num (->rvalue upper))
- :lowerStrict (not inclusive?)
- :upperStrict (not inclusive?)})
-
-(defmulti ^:private parse-filter*
- "Parse an MBQL `filter-clause` and generate an appropriate Druid filter map.
-
- (parse-filter* [:= [:field 1 nil] 2]) ; -> {:type :selector, :dimension \"venue_price\", :value 2}"
- {:arglists '([filter-clause])}
- driver-api/dispatch-by-clause-name-or-class)
-
-(defmethod parse-filter* nil
- [_]
- nil)
-
-(defmethod parse-filter* :between
- [[_ field min-val max-val]]
- (filter:bound field, :lower min-val, :upper max-val))
-
-(defmethod parse-filter* :contains
- [[_ field pattern options]]
- (if (and (sequential? pattern) (= :value (first pattern)))
- {:type :search
- :dimension (->rvalue field)
- :query {:type :contains
- :value (->rvalue pattern)
- :caseSensitive (get options :case-sensitive true)}}
- (throw (ex-info (tru "Dynamic patterns are not supported.")
- {:type driver-api/qp.error-type.invalid-query
- :field field :pattern pattern :options options}))))
-
-(defmethod parse-filter* :starts-with
- [[_ field pattern options]]
- (if (and (sequential? pattern) (= :value (first pattern)))
- (filter:like field
- (str (escape-like-filter-pattern (->rvalue pattern)) \%)
- (get options :case-sensitive true))
- (throw (ex-info (tru "Dynamic patterns are not supported.")
- {:type driver-api/qp.error-type.invalid-query
- :field field :pattern pattern :options options}))))
-
-(defmethod parse-filter* :ends-with
- [[_ field pattern options]]
- (if (and (sequential? pattern) (= :value (first pattern)))
- (filter:like field
- (str \% (escape-like-filter-pattern (->rvalue pattern)))
- (get options :case-sensitive true))
- (throw (ex-info (tru "Dynamic patterns are not supported.")
- {:type driver-api/qp.error-type.invalid-query
- :field field :pattern pattern :options options}))))
-
-(defmethod parse-filter* :=
- [[_ field value-or-field]]
- (filter:= field value-or-field))
-
-(defmethod parse-filter* :!=
- [[_ field value-or-field]]
- (filter:not (filter:= field value-or-field)))
-
-(defmethod parse-filter* :<
- [[_ field value-or-field]]
- (filter:bound field, :upper value-or-field, :inclusive? false))
-
-(defmethod parse-filter* :>
- [[_ field value-or-field]]
- (filter:bound field, :lower value-or-field, :inclusive? false))
-
-(defmethod parse-filter* :<=
- [[_ field value-or-field]]
- (filter:bound field, :upper value-or-field))
-
-(defmethod parse-filter* :>=
- [[_ field value-or-field]]
- (filter:bound field, :lower value-or-field))
-
-(defmethod parse-filter* :and
- [[_ & args]]
- (when-let [fields (seq (keep identity (map parse-filter* args)))]
- {:type :and, :fields (vec fields)}))
-
-(defmethod parse-filter* :or
- [[_ & args]]
- (when-let [fields (seq (keep identity (map parse-filter* args)))]
- {:type :or, :fields (vec fields)}))
-
-(defmethod parse-filter* :not
- [[_ subclause]]
- (when-let [subclause (parse-filter* subclause)]
- (filter:not subclause)))
-
-(defn- parse-filter [filter-clause]
- ;; strip out all the filters against temporal fields. Those are handled separately, as intervals
- (-> (match/replace filter-clause
- [_ [:field _ {:temporal-unit &truthy}] & _]
- nil)
- lib/->mbql5
- lib/simplify-compound-filter
- lib/->legacy-MBQL
- parse-filter*))
-
-(mu/defn- add-datetime-units* :- driver-api/mbql.schema.DateTimeValue
- "Return a `relative-datetime` clause with `n` units added to it."
- [absolute-or-relative-datetime :- driver-api/mbql.schema.DateTimeValue
- n :- number?]
- (if (driver-api/is-clause? :relative-datetime absolute-or-relative-datetime)
- (let [[_ original-n unit] absolute-or-relative-datetime]
- [:relative-datetime (+ n original-n) unit])
- (let [[_ t unit] absolute-or-relative-datetime]
- [:absolute-datetime (u.date/add t unit n) unit])))
-
-(defn- add-datetime-units
- "Adding `n` `:default` units doesn't make sense. So if an `:absoulte-datetime` has `:default` as its unit, add `n`
- milliseconds, because that is the smallest unit Druid supports."
- [clause n]
- (match/replace clause
- [:absolute-datetime t :default]
- [:absolute-datetime (u.date/add t :millisecond n) :millisecond]
-
- _
- (add-datetime-units* clause n)))
-
-(defn- ->absolute-timestamp ^java.time.temporal.Temporal [clause]
- (match/match-one clause
- [:absolute-datetime t :default]
- t
-
- [:absolute-datetime t unit]
- (u.date/truncate t unit)
-
- [:relative-datetime amount unit]
- (u.date/truncate (u.date/add unit amount) unit)
-
- _
- nil))
-
-(defmulti ^:private filter-clause->intervals
- "Generate query intervals as appropriate from a `filter-clause` containing a temporal `:field`. `:intervals` are
- specified separately from other things we think of as filter clauses in Druid. For temporal filter clauses, this
- returns a sequence of min/max datetime tuples; like `[#t 2019-01-01 #t 2019-10-01]`; for irrelevant filter
- clauses, the methods are skipped entirely."
- {:arglists '([filter-clause])}
- (fn [filter-clause]
- (when (match/match-one filter-clause [:field _ {:temporal-unit &truthy}] true)
- (driver-api/dispatch-by-clause-name-or-class filter-clause))))
-
-(defmethod filter-clause->intervals :default
- [_]
- nil)
-
-;; BETWEEN "2015-12-09", "2015-12-11" -> ["2015-12-09/2015-12-12"], because BETWEEN is inclusive
-(defmethod filter-clause->intervals :between
- [[_ _ min-value max-value]]
- [[(->absolute-timestamp min-value) (->absolute-timestamp (add-datetime-units max-value 1))]])
-
-(defmethod filter-clause->intervals :=
- [[_ _ v]]
- [[(->absolute-timestamp v) (->absolute-timestamp (add-datetime-units v 1))]])
-
-(defmethod filter-clause->intervals :!=
- [[_ _ v]]
- [[nil (->absolute-timestamp v)] [(->absolute-timestamp (add-datetime-units v 1)) nil]])
-
-(defmethod filter-clause->intervals :>
- [[_ _ v]]
- [[(->absolute-timestamp (add-datetime-units v 1)) nil]])
-
-(defmethod filter-clause->intervals :>=
- [[_ _ v]]
- [[(->absolute-timestamp v) nil]])
-
-(defmethod filter-clause->intervals :<
- [[_ _ v]]
- [[nil (->absolute-timestamp v)]])
-
-(defmethod filter-clause->intervals :<=
- [[_ _ v]]
- [[nil (->absolute-timestamp (add-datetime-units v 1))]])
-
-;; When you're anding together multiple intervals we have to combine them into a single interval that is the
-;; logical equivalent of all the intervals. e.g.
-;;
-;; `[:and [:>= x 2018] [:< x 2019]]` should get converted to a `2018/2019` interval.
-(defn- combine-intervals [[min-1 max-1] [min-2 max-2]]
- (let [datetime-max (fn [x y] (if (pos? (compare x y)) x y))
- datetime-min (fn [x y] (if (neg? (compare x y)) x y))]
- [(if (and min-1 min-2)
- (datetime-max min-1 min-2)
- (or min-1 min-2))
- (if (and max-1 max-2)
- (datetime-min max-1 max-2)
- (or max-1 max-2))]))
-
-(defmethod filter-clause->intervals :and
- [[_ & subclauses]]
- (let [subclause-intervals (map filter-clause->intervals subclauses)
- flattened-subclause-intervals (apply concat (filter #(= (count %) 1) subclause-intervals))]
- ;; log a warning about all the intervals we filtered out above
- (doseq [intervals subclause-intervals
- :when (> (count intervals) 1)]
- (log/warnf "WARNING: Don't know how to combine these intervals into a single interval.\nIgnoring intervals: %s"
- intervals))
- (reduce
- (fn [[acc] interval]
- [(combine-intervals acc interval)])
- nil
- flattened-subclause-intervals)))
-
-(defmethod filter-clause->intervals :or
- [[_ & subclauses]]
- (mapcat filter-clause->intervals subclauses))
-
-(defmethod filter-clause->intervals :not
- [[_ subclause]]
- ;; first, check and see if the subclause is actually something that will produce intervals (i.e., if it is a
- ;; temporal filter). If it is, then negate the logic and use the intervals for that. We don't want to call negate
- ;; without checking first because some filters like string `:contains` can't be negated without using a `:not`
- ;; filter and we don't want to stack overflow
- (when (seq (filter-clause->intervals subclause))
- (-> subclause
- lib/->mbql5
- lib/negate-boolean-expression
- lib/->legacy-MBQL
- filter-clause->intervals)))
-
-(defn- compile-intervals
- "Compile the interval pairs generated by `filter-clause->intervals` into the format expected by Druid (`min/max`
- strings)."
- [intervals]
- (when-let [intervals (seq (filter some? intervals))]
- (vec (for [[min-value max-value] intervals]
- (format "%s/%s"
- (or (some-> min-value u.date/format) "-5000")
- (or (some-> max-value u.date/format) "5000"))))))
-
-(defn- handle-filter
- [_ {filter-clause :filter} druid-query]
- (if-not filter-clause
- druid-query
- (let [filter (parse-filter filter-clause)
- intervals (compile-intervals (filter-clause->intervals filter-clause))]
- (cond-> druid-query
- (seq filter) (assoc-in [:query :filter] filter)
- (seq intervals) (assoc-in [:query :intervals] intervals)))))
-
-;;; ----------------------------------------------- handle-aggregation -----------------------------------------------
-
-(defn- expression->field-names
- [[_ & args]]
- {:post [(every? (some-fn keyword? string?) %)]}
- (flatten (for [arg args
- :when (or (field? arg)
- (driver-api/is-clause? #{:+ :- :/ :*} arg))]
- (cond
- (driver-api/is-clause? #{:+ :- :/ :*} arg) (expression->field-names arg)
- (field? arg) (->rvalue arg)))))
-
-(defn- expression-arg->js
- [arg default-value]
- (if-not (field? arg)
- arg
- (druid.js/or (druid.js/parse-float (->rvalue arg))
- default-value)))
-
-(defn- expression->js
- [[operator & args] default-value]
- (apply (case operator
- :+ druid.js/+
- :- druid.js/-
- :* druid.js/*
- :/ druid.js//)
- (for [arg args]
- (expression-arg->js arg default-value))))
-
-(defn- ag:doubleSum:expression
- [[operator :as expression] output-name]
- (let [field-names (expression->field-names expression)]
- {:type :javascript
- :name output-name
- :fieldNames field-names
- :fnReset (druid.js/function []
- (druid.js/return 0))
- :fnAggregate (druid.js/function (cons :current field-names)
- (druid.js/return (druid.js/+ :current (expression->js expression (if (= operator :/) 1 0)))))
- :fnCombine (druid.js/function [:x :y]
- (druid.js/return (druid.js/+ :x :y)))}))
-
-(defn- ag:doubleSum
- [field-clause output-name]
- (if (driver-api/is-clause? #{:+ :- :/ :*} field-clause)
- (ag:doubleSum:expression field-clause output-name)
- ;; metrics can use the built-in :doubleSum aggregator, but for dimensions we have to roll something that does the
- ;; same thing in JS
- (case (dimension-or-metric? field-clause)
- :metric {:type :doubleSum
- :name output-name
- :fieldName (->rvalue field-clause)}
- :dimension {:type :javascript
- :name output-name
- :fieldNames [(->rvalue field-clause)]
- :fnReset "function() { return 0 ; }"
- :fnAggregate "function(current, x) { return current + (parseFloat(x) || 0); }"
- :fnCombine "function(x, y) { return x + y; }"})))
-
-(defn- ag:doubleMin:expression
- [expression output-name]
- (let [field-names (expression->field-names expression)]
- {:type :javascript
- :name output-name
- :fieldNames field-names
- :fnReset (druid.js/function []
- (druid.js/return "Number.MAX_VALUE"))
- :fnAggregate (druid.js/function (cons :current field-names)
- (druid.js/return (druid.js/fn-call :Math.min :current
- (expression->js expression :Number.MAX_VALUE))))
- :fnCombine (druid.js/function [:x :y]
- (druid.js/return (druid.js/fn-call :Math.min :x :y)))}))
-
-(defn- ag:doubleMin
- [field-clause output-name]
- (if (driver-api/is-clause? #{:+ :- :/ :*} field-clause)
- (ag:doubleMin:expression field-clause output-name)
- (case (dimension-or-metric? field-clause)
- :metric {:type :doubleMin
- :name output-name
- :fieldName (->rvalue field-clause)}
- :dimension {:type :javascript
- :name output-name
- :fieldNames [(->rvalue field-clause)]
- :fnReset "function() { return Number.MAX_VALUE ; }"
- :fnAggregate "function(current, x) { return Math.min(current, (parseFloat(x) || Number.MAX_VALUE)); }"
- :fnCombine "function(x, y) { return Math.min(x, y); }"})))
-
-(defn- ag:doubleMax:expression
- [expression output-name]
- (let [field-names (expression->field-names expression)]
- {:type :javascript
- :name output-name
- :fieldNames field-names
- :fnReset (druid.js/function []
- (druid.js/return "Number.MIN_VALUE"))
- :fnAggregate (druid.js/function (cons :current field-names)
- (druid.js/return (druid.js/fn-call :Math.max :current
- (expression->js expression :Number.MIN_VALUE))))
- :fnCombine (druid.js/function [:x :y]
- (druid.js/return (druid.js/fn-call :Math.max :x :y)))}))
-
-(defn- ag:doubleMax
- [field output-name]
- (if (driver-api/is-clause? #{:+ :- :/ :*} field)
- (ag:doubleMax:expression field output-name)
- (case (dimension-or-metric? field)
- :metric {:type :doubleMax
- :name output-name
- :fieldName (->rvalue field)}
- :dimension {:type :javascript
- :name output-name
- :fieldNames [(->rvalue field)]
- :fnReset "function() { return Number.MIN_VALUE ; }"
- :fnAggregate "function(current, x) { return Math.max(current, (parseFloat(x) || Number.MIN_VALUE)); }"
- :fnCombine "function(x, y) { return Math.max(x, y); }"})))
-
-(defn- ag:filtered
- [filtr aggregator]
- {:pre [(map? filtr)]}
- {:type :filtered, :filter filtr, :aggregator aggregator})
-
-(defn- hyper-unique?
- [[_ field-id]]
- {:pre [(pos-int? field-id)]}
- (isa? (:base-type (driver-api/field (driver-api/metadata-provider) field-id))
- :type/DruidHyperUnique))
-
-(defn- ag:distinct
- [field output-name]
- (cond
- (driver-api/is-clause? #{:+} field)
- {:type :cardinality
- :name output-name
- :fieldNames (mapv ->rvalue (rest field))
- :byRow true
- :round true}
- (hyper-unique? field)
- {:type :hyperUnique
- :name output-name
- :fieldName (->rvalue field)}
- :else
- {:type :cardinality
- :name output-name
- :fieldNames [(->rvalue field)]
- :byRow true
- :round true}))
-
-(defn- ag:count
- ([output-name]
- {:type :count, :name output-name})
- ([field output-name]
- (if (and (driver-api/is-clause? #{:field-id} field)
- (hyper-unique? field))
- {:type :hyperUnique
- :name output-name
- :fieldName (->rvalue field)}
- (ag:filtered (filter:not (filter:nil? field)) (ag:count output-name)))))
-
-(defn- ag:countWhere
- [pred output-name]
- (ag:filtered (parse-filter pred) (ag:count output-name)))
-
-(defn- ag:sumWhere
- [field pred output-name]
- (ag:filtered (parse-filter pred) (ag:doubleSum field output-name)))
-
-(def ^:private ^{:arglists '([prefix])} genname
- (comp name gensym))
-
-(defn- create-aggregation-clause
- [output-name ag-type ag-field args]
- (let [output-name-kwd (keyword output-name)]
- (match/match-one [ag-type ag-field]
- ;; For 'distinct values' queries (queries with a breakout by no aggregation) just aggregate by count, but name
- ;; it :___count so it gets discarded automatically
- [nil nil] [[(or output-name-kwd :___count)] {:aggregations [(ag:count (or output-name :___count))]}]
-
- [:count nil] [[(or output-name-kwd :count)] {:aggregations [(ag:count (or output-name :count))]}]
-
- [:count _] [[(or output-name-kwd :count)] {:aggregations [(ag:count ag-field (or (name output-name) :count))]}]
-
- [:avg _] (let [count-name (genname "___count_")
- sum-name (genname "___sum_")]
- [[(keyword count-name) (keyword sum-name) (or output-name-kwd :avg)]
- {:aggregations [(ag:count ag-field count-name)
- (ag:doubleSum ag-field sum-name)]
- :postAggregations [{:type :arithmetic
- :name (or output-name :avg)
- :fn :/
- :fields [{:type :fieldAccess, :fieldName sum-name}
- {:type :fieldAccess, :fieldName count-name}]}]}])
-
- [:sum-where _] (let [[pred] args]
- [[(or output-name-kwd :sum-where)]
- {:aggregations [(ag:sumWhere ag-field pred output-name-kwd)]}])
-
- [:count-where _] [[(or output-name-kwd :count-where)]
- {:aggregations [(ag:countWhere ag-field output-name-kwd)]}]
-
- [:share _] (let [total-count-name (genname "___total_count_")
- true-count-name (genname "___true_count_")]
- [[(keyword total-count-name) (keyword true-count-name) (or output-name-kwd :share)]
- {:aggregations [(ag:count total-count-name)
- (ag:countWhere ag-field true-count-name)]
- :postAggregations [{:type :arithmetic
- :name (or output-name :share)
- :fn :/
- :fields [{:type :fieldAccess, :fieldName true-count-name}
- {:type :fieldAccess, :fieldName total-count-name}]}]}])
-
- [:distinct _] [[(or output-name-kwd :distinct___count)]
- {:aggregations [(ag:distinct ag-field (or output-name :distinct___count))]}]
- [:sum _] [[(or output-name-kwd :sum)]
- {:aggregations [(ag:doubleSum ag-field (or (name output-name) :sum))]}]
- [:min _] [[(or output-name-kwd :min)]
- {:aggregations [(ag:doubleMin ag-field (or output-name :min))]}]
- [:max _] [[(or output-name-kwd :max)]
- {:aggregations [(ag:doubleMax ag-field (or output-name :max))]}]
- _ (throw (ex-info (str "Unknown aggregation tuple: " [ag-type ag-field]) {})))))
-
-(defn ^:private ^:dynamic *query-unique-name-fn*
- [& _]
- (throw (ex-info "Must bind metabase.driver.druid.query-processor/*query-unique-name-fn*. See handle-aggregation for an example." {})))
-
-(mu/defn- handle-aggregation
- [query-type
- ag-clause :- driver-api/mbql.schema.Aggregation
- druid-query]
- (let [output-name (*query-unique-name-fn* (driver-api/aggregation-name *query* ag-clause))
- [ag-type ag-field & args] (match/match-one ag-clause
- [:aggregation-options ag & _] (&recur ag)
- _ &match)]
- (if-not (isa? query-type ::ag-query)
- druid-query
- (let [[projections ag-clauses] (try
- (create-aggregation-clause output-name ag-type ag-field args)
- (catch Throwable e
- (throw (ex-info (tru "Error creating aggregation clause")
- {:type driver-api/qp.error-type.driver
- :clause-name output-name
- :ag-type ag-type
- :ag-field ag-field
- :args args}
- e))))]
- (-> druid-query
- (update :projections into projections)
- (update :query (partial merge-with concat) ag-clauses))))))
-
-(defn- deduplicate-aggregation-options [expression]
- (match/replace expression
- [:aggregation-options [:aggregation-options ag options-1] options-2]
- [:aggregation-options ag (merge options-1 options-2)]))
-
-(defn- aggregation-unique-identifier [clause]
- (format "__%s" (*query-unique-name-fn* (name clause))))
-
-(defn- add-expression-aggregation-output-names
- [expression]
- (match/replace expression
- [:aggregation-options ag options]
- (deduplicate-aggregation-options [:aggregation-options (add-expression-aggregation-output-names ag) options])
-
- [(clause :guard #{:count :avg :distinct :stddev :sum :min :max}) & _]
- [:aggregation-options &match {:name (aggregation-unique-identifier clause)}]))
-
-(defn- post-aggregator-type
- "Complex aggregators like `cardinality` and ``hyperUnique` (which we use to implement MBQL
- `:distinct`) require finalizing their return value.
- https://druid.apache.org/docs/latest/querying/post-aggregations.html"
- [[op & _]]
- (if (= :distinct op)
- :finalizingFieldAccess
- :fieldAccess))
-
-(defn- expression-post-aggregation
- [[operator & args, :as expression]]
- (match/match-one expression
- ;; If it's a named expression, we want to preserve the included name, so recurse, but merge in the name
- [:aggregation-options ag _]
- (assoc (expression-post-aggregation ag) :name (driver-api/aggregation-name *query* expression))
-
- _
- {:type :arithmetic
- :name (driver-api/aggregation-name *query* expression)
- :fn operator
- :fields (vec (for [arg args]
- (match/match-one arg
- (_ :guard number?)
- {:type :constant, :name (str &match), :value &match}
-
- [:aggregation-options ag {:name name}]
- {:type (post-aggregator-type ag), :fieldName name}
-
- [#{:+ :- :/ :*} & _]
- (expression-post-aggregation &match)
-
- ;; we should never get here unless our code is B U S T E D
- _
- (throw (ex-info (tru "Expected :aggregation-options, constant, or expression.")
- {:type :bug, :input arg})))))}))
-
-(declare handle-aggregations)
-
-(defn- expression->actual-ags
- "Return a flattened list of actual aggregations that are needed for `expression`."
- [[_ & args]]
- (into []
- (comp (remove number?)
- (map (fn [arg]
- (if (driver-api/is-clause? #{:+ :- :/ :*} arg)
- (expression->actual-ags arg)
- [arg])))
- cat)
- args))
-
-(defn- unwrap-name
- [x]
- (if (driver-api/is-clause? :aggregation-options x)
- (second x)
- x))
-
-(defn- handle-expression-aggregation
- [query-type expression druid-query]
- ;; filter out constants from the args list
- (let [expression (add-expression-aggregation-output-names expression)
- ;; The QP will automatically add a generated name to the expression, if it's there, unwrap it before looking
- ;; for the aggregation
- ags (expression->actual-ags (unwrap-name expression))
- druid-query (handle-aggregations query-type {:aggregation ags} druid-query)
- post-agg (expression-post-aggregation expression)]
- (-> druid-query
- (update :projections conj (keyword (:name post-agg)))
- (update-in [:query :postAggregations] concat [post-agg]))))
-
-(defn- handle-aggregations
- [query-type {aggregations :aggregation} druid-query]
- (binding [*query-unique-name-fn* (driver-api/unique-name-generator)]
- ;; Load unique name generator with projections from breakouts
- (doseq [projection (:projections druid-query)]
- (*query-unique-name-fn* (name projection)))
- (reduce
- (fn [druid-query aggregation]
- (match/match-one aggregation
- [:aggregation-options [#{:+ :- :/ :*} & _] _]
- (handle-expression-aggregation query-type &match druid-query)
-
- [#{:+ :- :/ :*} & _]
- (handle-expression-aggregation query-type &match druid-query)
-
- _
- (handle-aggregation query-type aggregation druid-query)))
- druid-query
- aggregations)))
-
-;;; ------------------------------------------------ handle-breakout -------------------------------------------------
-
-(defmulti ^:private ->dimension-rvalue
- "Format `Field` for use in a `:dimension` or `:dimensions` clause."
- {:arglists '([field-clause])}
- driver-api/dispatch-by-clause-name-or-class)
-
-(defn- extract:timeFormat
- "Create a time format extraction. Returns a string. See
- http://druid.io/docs/0.9.1.1/querying/dimensionspecs.html#time-format-extraction-function"
- [format-str]
- {:pre [(string? format-str)]}
- {:type :timeFormat
- :format format-str
- :timeZone (or (get-in *query* [:settings :report-timezone])
- "UTC")
- :locale "en-US"})
-
-(defn- extract:js
- "Create an extraction function from JavaScript -- see
- http://druid.io/docs/0.9.1.1/querying/dimensionspecs.html#javascript-extraction-function"
- [& function-str-parts]
- {:pre [(every? string? function-str-parts)]}
- {:type :javascript
- :function (str/replace (apply str function-str-parts) #"\s+" " ")})
-
-;; don't try to make this a ^:const map -- extract:timeFormat looks up timezone info at query time
-(defn- unit->extraction-fn [unit]
- (case unit
- :default (extract:timeFormat "yyyy-MM-dd'T'HH:mm:ssZZ")
- :minute (extract:timeFormat "yyyy-MM-dd'T'HH:mm:00ZZ")
- :minute-of-hour (extract:timeFormat "mm")
- :hour (extract:timeFormat "yyyy-MM-dd'T'HH:00:00ZZ")
- :hour-of-day (extract:timeFormat "HH")
- :day (extract:timeFormat "yyyy-MM-dd'T'00:00:00ZZ")
- :day-of-week (extract:js "function (timestamp) {"
- " var date = new Date(timestamp);"
- (format " var dayOfWeek = (date.getDay() + 1 + %s) %% 7;"
- (driver.common/start-of-week-offset :druid))
- " return (dayOfWeek == 0) ? 7 : dayOfWeek;"
- "}")
- :day-of-month (extract:timeFormat "dd")
- :day-of-year (extract:timeFormat "DDD")
- :week (extract:js "function (timestamp) {"
- " var date = new Date(timestamp);"
- (format " var firstDOW = new Date(date - ((date.getDay() + %s) * 86400000));"
- (driver.common/start-of-week-offset :druid))
- " var month = firstDOW.getMonth() + 1;"
- " var day = firstDOW.getDate();"
- " return '' + firstDOW.getFullYear() + '-' + (month < 10 ? '0' : '') + month + '-' + (day < 10 ? '0' : '') + day;"
- "}")
- :week-of-year (extract:js "function (timestamp) {"
- " var date = new Date(timestamp);"
- (format " var firstDOW = new Date(date - ((date.getDay() + %s) * 86400000));"
- (driver.common/start-of-week-offset :druid))
- " var dayOfYear = (Date.UTC(firstDOW.getFullYear(), firstDOW.getMonth(), firstDOW.getDate()) - Date.UTC(firstDOW.getFullYear(), 0, 0)) / 24 / 60 / 60 / 1000;"
- " return Math.floor(dayOfYear / 7) + 1;"
- "}")
- :month (extract:timeFormat "yyyy-MM-01")
- :month-of-year (extract:timeFormat "MM")
- :quarter (extract:js "function (timestamp) {"
- " var date = new Date(timestamp);"
- " var month = date.getMonth() + 1;" ; js months are 0 - 11
- " var quarterMonth = month - ((month - 1) % 3);"
- " return '' + date.getFullYear() + '-' + (quarterMonth < 10 ? '0' : '') + quarterMonth + '-01';"
- "}")
- :quarter-of-year (extract:js "function (timestamp) {"
- " var date = new Date(timestamp);"
- " return Math.floor((date.getMonth() + 3) / 3);"
- "}")
- :year (extract:timeFormat "yyyy-01-01")))
-
-(defn- unit->granularity
- [unit]
- (merge {:type "period"
- :period (case unit
- :minute "PT1M"
- :hour "PT1H"
- :day "P1D"
- :week "P1W"
- :month "P1M"
- :quarter "P3M"
- :year "P1Y")
- :timeZone (driver-api/results-timezone-id)}
- ;; Druid uses Monday for the start of its weekly calculations. Metabase uses Sundays. When grouping by week,
- ;; the origin keypair will use the date specified as it's start of the week. The below date is the first
- ;; Sunday after Epoch. The date itself isn't significant, it just uses it to figure out what day it should
- ;; start from.
- (when (= :week unit)
- {:origin "1970-01-04T00:00:00Z"})))
-
-(def ^:private units-that-need-post-processing-int-parsing
- "`extract:timeFormat` always returns a string; there are cases where we'd like to return an integer tead, such as
- `:day-of-month`. There's no simple way to do this in Druid -- Druid 0.9.0+ *does* let you combine extraction
- functions with `:cascade`, but we're still supporting 0.8.x. Instead, we will perform the conversions in
- Clojure-land during post-processing. If we need to perform the extra post-processing step, we'll name the resulting
- column `:timestamp___int`; otherwise we'll keep the name `:timestamp`."
- #{:minute-of-hour
- :hour-of-day
- :day-of-week
- :day-of-month
- :day-of-year
- :week-of-year
- :month-of-year
- :quarter-of-year})
-
-(defmethod ->dimension-rvalue nil
- [_]
- (->rvalue nil))
-
-(defmethod ->dimension-rvalue Object
- [this]
- (->rvalue this))
-
-(defn- temporal-dimension-rvalue [unit]
- {:type :extraction
- :dimension :__time
- ;; :timestamp is a special case, and we need to do an 'extraction' agat the secret special value :__time to get
- ;; at it
- :outputName (if (contains? units-that-need-post-processing-int-parsing unit)
- :timestamp___int
- :timestamp)
- :extractionFn (unit->extraction-fn unit)})
-
-(defmethod ->dimension-rvalue :field
- [[_ _ {:keys [base-type temporal-unit]} :as clause]]
- (if (or temporal-unit
- (isa? base-type :type/Temporal))
- (temporal-dimension-rvalue (or temporal-unit :default))
- (->rvalue clause)))
-
-(defmulti ^:private handle-breakout
- {:arglists '([query-type original-query druid-query])}
- query-type-dispatch-fn)
-
-;; only topN , grouped-timeseries & groupBy handle breakouts
-(defmethod handle-breakout ::query
- [_ _ druid-query]
- druid-query)
-
-(defmethod handle-breakout ::grouped-timeseries
- [_ {[breakout-field] :breakout} druid-query]
- (assoc-in druid-query [:query :granularity] (unit->granularity (:unit breakout-field))))
-
-(defn- field-clause->name
- [field-clause]
- (match/match-one field-clause
- [:field (id :guard integer?) _]
- (:name (driver-api/field (driver-api/metadata-provider) id))
-
- [:field (field-name :guard string?) _]
- field-name))
-
-(defmethod handle-breakout ::topN
- [_ {[breakout-field] :breakout} druid-query]
- (let [dim-rvalue (->dimension-rvalue breakout-field)]
- (-> druid-query
- (update :projections conj (keyword (if (and (map? dim-rvalue)
- (contains? dim-rvalue :outputName))
- (:outputName dim-rvalue)
- (field-clause->name breakout-field))))
- (assoc-in [:query :dimension] dim-rvalue))))
-
-(defmethod handle-breakout ::groupBy
- [_ {breakout-fields :breakout} druid-query]
- (-> druid-query
- (update :projections into (for [breakout-field breakout-fields]
- (let [dim-rvalue (->dimension-rvalue breakout-field)]
- (keyword
- (if (and (map? dim-rvalue)
- (contains? dim-rvalue :outputName))
- (:outputName dim-rvalue)
- (field-clause->name breakout-field))))))
- (assoc-in [:query :dimensions] (mapv ->dimension-rvalue breakout-fields))))
-
-;;; ------------------------------------------------ handle-order-by -------------------------------------------------
-
-(defmulti ^:private handle-order-by
- {:arglists '([query-type original-query druid-query])}
- query-type-dispatch-fn)
-
-(defmethod handle-order-by ::query
- [_ _ druid-query]
- (log/warn
- (u/format-color
- 'red
- "Sorting with Druid is only allowed in queries that have one or more breakout columns. Ignoring :order-by clause."))
- druid-query)
-
-(defmethod handle-order-by ::topN
- [_ {[ag] :aggregation, [breakout-field] :breakout, [[direction field]] :order-by} druid-query]
- (let [field (->rvalue field)
- breakout-field (->rvalue breakout-field)
- sort-by-breakout? (= field breakout-field)
- ag-field (match/match-one ag
- [:distinct & _] :distinct___count
- [:aggregation-options _ {:name name}] name
- [:aggregation-options wrapped-ag _] (&recur wrapped-ag)
- [(ag-type :guard keyword?) & _] ag-type)]
- (when-not sort-by-breakout?
- (assert ag-field))
- (assoc-in druid-query [:query :metric] (match/match-one [sort-by-breakout? direction]
- [true :asc] {:type :alphaNumeric}
- [true :desc] {:type :inverted, :metric {:type :alphaNumeric}}
- [false :asc] {:type :inverted, :metric ag-field}
- [false :desc] ag-field))))
-
-(defmethod handle-order-by ::groupBy
- [_ {:keys [order-by]} druid-query]
- (assoc-in druid-query [:query :limitSpec :columns] (vec (for [[direction field] order-by]
- {:dimension (->rvalue field)
- :direction (case direction
- :desc :descending
- :asc :ascending)}))))
-(defn- temporal-field?
- "Similar to `types/temporal-field?` but works on field ids wrapped in a datetime or on fields that happen to be a
- datetime"
- [field]
- (when field
- (match/match-one field
- [:field _id-or-name {:temporal-unit &truthy}]
- true
-
- [:field (id :guard pos-int?) _opts]
- (driver-api/temporal? (driver-api/field (driver-api/metadata-provider) id)))))
-
-;; Handle order by timestamp field
-(defmethod handle-order-by ::grouped-timeseries
- [_ {[[direction field]] :order-by} druid-query]
- (let [can-sort? (if (temporal-field? field)
- true
- (log/warn "grouped timeseries queries can only be sorted by the 'timestamp' column."))]
- (cond-> druid-query
- can-sort? (assoc-in [:query :descending] (= direction :desc)))))
-
-(defmethod handle-order-by ::scan
- [_ {[[direction field]] :order-by, fields :fields} druid-query]
- (let [can-sort? (cond
- (not (some temporal-field? fields))
- (log/warn "scan queries can only be sorted if they include the 'timestamp' column.")
-
- (not (temporal-field? field))
- (log/warn "scan queries can only be sorted by the 'timestamp' column.")
-
- :else
- true)]
- (cond-> druid-query
- can-sort? (assoc-in [:query :order] (case direction
- :desc :descending
- :asc :ascending)))))
-
-;;; ------------------------------------------------- handle-fields --------------------------------------------------
-
-(defmulti ^:private handle-fields
- {:arglists '([query-type original-query druid-query])}
- query-type-dispatch-fn)
-
-(defmethod handle-fields ::query
- [_ {fields :fields} druid-query]
- (when fields
- (log/warn
- (u/format-color 'red
- ;; TODO - this is not really true, is it
- "WARNING: It only makes sense to specify :fields for a query with no aggregation. Ignoring the clause.")))
- druid-query)
-
-(defmethod handle-fields ::scan
- [_ {fields :fields} druid-query]
- (transduce
- identity
- (fn
- ([druid-query]
- ;; If you specify nil or empty `:columns` Druid will just return all of the ones available. In cases where
- ;; we don't want anything to be returned in one or the other, we'll ask for a `:___dummy` column instead.
- ;; Druid happily returns `nil` for the column in every row, and it will get auto-filtered out of the results
- ;; so the User will never see it.
- (update-in druid-query [:query :columns] #(or (seq %) [:___dummy])))
-
- ([druid-query field]
- (if (and (temporal-field? field)
- (= (keyword (field-clause->name field)) :timestamp))
- (-> druid-query
- (update :projections conj :timestamp)
- (update-in [:query :columns] conj :__time))
- (-> druid-query
- (update :projections conj (keyword (field-clause->name field)))
- (update-in [:query :columns] conj (->rvalue field))))))
- druid-query
- fields))
-
-;;; -------------------------------------------------- handle-limit --------------------------------------------------
-
-(defmulti ^:private handle-limit
- {:arglists '([query-type original-query druid-query])}
- query-type-dispatch-fn)
-
-(defn- adjust-limit
- "No joke, Druid queries do not work if limit is `1048575`, but they work if limit is `1048576`. They fail with an
- 'Invalid type marker byte 0x3c for expected value token` error. So if we see the updated `absolute-max-results` from
- #15414, adjust it back to the old known working value. had to work around."
- [limit]
- (cond-> limit
- (= limit driver-api/absolute-max-results) inc))
-
-(defmethod handle-limit ::scan
- [_ {limit :limit} druid-query]
- (cond-> druid-query
- limit (assoc-in [:query :limit] (adjust-limit limit))))
-
-(defmethod handle-limit ::timeseries
- [_ {limit :limit} druid-query]
- (when limit
- (log/warn
- (u/format-color 'red "WARNING: Druid does not allow limitSpec in time series queries. Ignoring the LIMIT clause.")))
- druid-query)
-
-(defmethod handle-limit ::topN
- [_ {limit :limit} druid-query]
- (cond-> druid-query
- limit (assoc-in [:query :threshold] (adjust-limit limit))))
-
-(defmethod handle-limit ::groupBy
- [_ {limit :limit} druid-query]
- (cond-> druid-query
- true (assoc-in [:query :limitSpec :type] :default)
- limit (assoc-in [:query :limitSpec :limit] (adjust-limit limit))))
-
-;;; -------------------------------------------------- handle-page ---------------------------------------------------
-
-;; TODO - no real way to implement this DB side, probably have to do Clojure-side w/ `take`/`drop`
-
-(defmulti ^:private handle-page
- {:arglists '([query-type original-query druid-query])}
- query-type-dispatch-fn)
-
-(defmethod handle-page ::query
- [_ {page-clause :page} druid-query]
- (when page-clause
- (log/warn (u/format-color 'red "WARNING: 'page' is not yet implemented.")))
- druid-query)
-
-;;; +----------------------------------------------------------------------------------------------------------------+
-;;; | Build + Log + Process Query |
-;;; +----------------------------------------------------------------------------------------------------------------+
-
-(def ^:private ^:const timeseries-units #{:minute :hour :day :week :month :quarter :year})
-
-(defn- druid-query-type
- "What type of Druid query type should we perform?"
- [{breakout-fields :breakout, [[ag-type]] :aggregation, limit :limit}]
- (let [breakouts (condp = (count breakout-fields)
- 0 :none
- 1 :one
- :many)
- agg? (boolean ag-type)
- ts? (boolean
- (and
- ;; Checks whether the query is a timeseries
- (match/match-one (first breakout-fields) [:field _ {:temporal-unit &truthy}] true)
- ;; (excludes x-of-y type breakouts)
- (contains? timeseries-units (:unit (first breakout-fields)))
- ;; (excludes queries with LIMIT)
- (nil? limit)))]
- (match/match-one [breakouts agg? ts?]
- [:none false _] ::scan
- [:none true _] ::total
- [:one _ true] ::grouped-timeseries
- [:one _ false] ::topN
- [:many _ _] ::groupBy)))
-
-(defn- build-druid-query
- [original-query]
- {:pre [(map? original-query)]}
- (let [query-type (druid-query-type original-query)]
- (reduce (fn [druid-query f]
- (f query-type original-query druid-query))
- {:projections [], :query (query-type->default-query query-type), :query-type query-type, :mbql? true}
- [handle-source-table
- handle-breakout
- handle-aggregations
- handle-filter
- handle-order-by
- handle-fields
- handle-limit
- handle-page])))
-
-(defn mbql->native
- "Transpile an MBQL (inner) query into a native form suitable for a Druid DB."
- [query]
- ;; Merge `:settings` into the inner query dict so the QP has access to it
- (let [query (driver-api/->legacy-MBQL query)
- query (assoc (:query query) :settings (:settings query))]
- (binding [*query* query]
- (try
- (build-druid-query query)
- (catch Throwable e
- (throw (ex-info (tru "Error generating Druid query")
- {:type driver-api/qp.error-type.driver
- :source-query query}
- e)))))))
diff --git a/modules/drivers/druid/src/metabase/driver/druid/sync.clj b/modules/drivers/druid/src/metabase/driver/druid/sync.clj
deleted file mode 100644
index 8d6297a8525..00000000000
--- a/modules/drivers/druid/src/metabase/driver/druid/sync.clj
+++ /dev/null
@@ -1,76 +0,0 @@
-(ns metabase.driver.druid.sync
- (:refer-clojure :exclude [select-keys])
- (:require
- [medley.core :as m]
- [metabase.driver-api.core :as driver-api]
- [metabase.driver.connection :as driver.conn]
- [metabase.driver.druid.client :as druid.client]
- [metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh]
- [metabase.util.performance :refer [select-keys]]))
-
-(defn- do-segment-metadata-query [details datasource]
- {:pre [(map? details) (string? datasource)]}
- (druid.client/do-query details {"queryType" :segmentMetadata
- "dataSource" datasource
- "intervals" ["1999-01-01/2114-01-01"]
- "analysisTypes" [:aggregators]
- "merge" true}))
-
-(defn- druid-type->base-type [field-type]
- (case field-type
- "STRING" :type/Text
- "FLOAT" :type/Float
- "DOUBLE" :type/Float
- "LONG" :type/Integer
- "hyperUnique" :type/DruidHyperUnique
- :type/Float))
-
-(defn describe-table
- "Impl of `driver/describe-table` for Druid."
- [database table]
- {:pre [(map? database) (map? table)]}
- (ssh/with-ssh-tunnel [details-with-tunnel (driver.conn/effective-details database)]
- (let [{:keys [columns aggregators]} (first (do-segment-metadata-query details-with-tunnel (:name table)))
- metric-column-names (set (keys aggregators))]
- {:schema nil
- :name (:name table)
- :fields (set (cons
- ;; every Druid table is an event stream w/ a timestamp field
- {:name "timestamp"
- :database-type "timestamp"
- :base-type :type/Instant
- :pk? false
- :database-position 0}
- (for [[idx [field-name {field-type :type}]] (m/indexed (dissoc columns :__time))
- :let [metric? (contains? metric-column-names field-name)]]
- {:name (name field-name)
- :base-type (druid-type->base-type field-type)
- :database-type (if metric?
- (format "%s [metric]" field-type)
- field-type)
- :database-position (inc idx)})))})))
-
-(defn describe-database
- "Impl of `driver/describe-database` for Druid."
- [database]
- {:pre [(map? (:details database))]}
- (let [details (driver.conn/effective-details database)]
- (ssh/with-ssh-tunnel [details-with-tunnel details]
- (let [druid-datasources (druid.client/GET (druid.client/details->url details-with-tunnel "/druid/v2/datasources")
- :auth-enabled (:auth-enabled details)
- :auth-username (:auth-username details)
- :auth-token-value (driver-api/secret-value-as-string :druid details "auth-token"))]
- {:tables (set (for [table-name druid-datasources]
- {:schema nil, :name table-name}))}))))
-
-(defn dbms-version
- "Impl of `driver/dbms-version` for Druid."
- [database]
- {:pre [(map? (:details database))]}
- (let [details (driver.conn/effective-details database)]
- (ssh/with-ssh-tunnel [details-with-tunnel details]
- (-> (druid.client/GET (druid.client/details->url details-with-tunnel "/status")
- :auth-enabled (:auth-enabled details)
- :auth-username (:auth-username details)
- :auth-token-value (driver-api/secret-value-as-string :druid details "auth-token"))
- (select-keys [:version])))))
diff --git a/modules/drivers/druid/test/metabase/driver/druid/client_test.clj b/modules/drivers/druid/test/metabase/driver/druid/client_test.clj
deleted file mode 100644
index 4bd715a88b0..00000000000
--- a/modules/drivers/druid/test/metabase/driver/druid/client_test.clj
+++ /dev/null
@@ -1,105 +0,0 @@
-(ns ^:mb/driver-tests metabase.driver.druid.client-test
- {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.driver.druid.client-test]}}}}}}
- (:require
- [clojure.core.async :as a]
- [clojure.test :refer :all]
- [metabase.driver.druid.client :as druid.client]
- [metabase.driver.settings :as driver.settings]
- [metabase.driver.util :as driver.u]
- [metabase.query-processor.test :as qp]
- [metabase.query-processor.timeseries-test.util :as tqpt]
- [metabase.test :as mt]))
-
-(set! *warn-on-reflection* true)
-
-(deftest query-cancelation-test
- (mt/test-driver :druid
- (tqpt/with-flattened-dbdef
- (let [query (mt/mbql-query checkins)]
- (mt/with-open-channels [running-chan (a/promise-chan)
- cancel-chan (a/promise-chan)]
- (with-redefs [druid.client/DELETE (fn [& _]
- (a/>!! cancel-chan ::cancel))
- druid.client/do-query (fn [& _]
- (a/>!! running-chan ::running)
- (Thread/sleep 5000)
- (throw (Exception. "Don't actually run!")))]
- (let [futur (future (qp/process-query query))]
- ;; wait for query to start running, then kill the thread running the query
- (a/go
- (a/<! running-chan)
- (future-cancel futur)))
- (is (= ::cancel
- (mt/wait-for-result cancel-chan 2000)))))))))
-
-(deftest query-timeout-test
- (mt/test-driver :druid
- (tqpt/with-flattened-dbdef
- (let [query (mt/mbql-query checkins)
- executed-query (atom nil)]
- (mt/with-dynamic-fn-redefs [druid.client/do-query-with-cancellation (fn [_chan _details query]
- (reset! executed-query query)
- [])]
- (qp/process-query query)
- (is (partial= {:context {:timeout driver.settings/*query-timeout-ms*}}
- @executed-query)))))))
-
-(deftest ssh-tunnel-test
- (mt/test-driver
- :druid
- (is (thrown?
- java.net.ConnectException
- (try
- (let [engine :druid
- details {:ssl false
- :password "changeme"
- :tunnel-host "localhost"
- :tunnel-pass "BOGUS-BOGUS"
- :port 5432
- :dbname "test"
- :host "http://localhost"
- :tunnel-enabled true
- ;; we want to use a bogus port here on purpose -
- ;; so that locally, it gets a ConnectionRefused,
- ;; and in CI it does too. Apache's SSHD library
- ;; doesn't wrap every exception in an SshdException
- :tunnel-port 21212
- :tunnel-user "bogus"}]
- (driver.u/can-connect-with-details? engine details :throw-exceptions))
- (catch Throwable e
- (loop [^Throwable e e]
- (or (when (instance? java.net.ConnectException e)
- (throw e))
- (some-> (.getCause e) recur)))))))))
-
-(defn- test-request
- ([request-fn]
- (test-request request-fn true))
- ([request-fn basic-auth?]
- (try
- (request-fn "http://localhost:8082/druid/v2"
- :auth-enabled basic-auth?
- :auth-username "nbotelho"
- :auth-token-value "12345678910")
- (catch Exception e
- (ex-data e)))))
-
-(defn- get-auth-header [m]
- (select-keys (:request-options m) [:basic-auth]))
-
-(deftest basic-auth-test
- (let [get-request (test-request druid.client/GET)
- post-request (test-request druid.client/POST)
- delete-request (test-request druid.client/DELETE)]
- (is (= {:basic-auth "nbotelho:12345678910"}
- (get-auth-header get-request)
- (get-auth-header post-request)
- (get-auth-header delete-request)) "basic auth header included with successfully"))
- (let [no-auth-basic false
- get-request (test-request druid.client/GET no-auth-basic)
- post-request (test-request druid.client/POST no-auth-basic)
- delete-request (test-request druid.client/DELETE no-auth-basic)]
- (is (= no-auth-basic
- (contains? (get-auth-header get-request) :basic-auth)
- (contains? (get-auth-header post-request) :basic-auth)
- (contains? (get-auth-header delete-request) :basic-auth)) "basic auth header not included")))
diff --git a/modules/drivers/druid/test/metabase/driver/druid/execute_test.clj b/modules/drivers/druid/test/metabase/driver/druid/execute_test.clj
deleted file mode 100644
index 73c5c161fd3..00000000000
--- a/modules/drivers/druid/test/metabase/driver/druid/execute_test.clj
+++ /dev/null
@@ -1,60 +0,0 @@
-(ns ^:mb/driver-tests metabase.driver.druid.execute-test
- {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.driver.druid.execute-test]}
- metabase.test.data/run-mbql-query {:namespaces [metabase.driver.druid.execute-test]}}}}}}
- (:require
- [clojure.java.io :as io]
- [clojure.test :refer :all]
- [metabase.driver.druid.execute :as druid.execute]
- [metabase.query-processor.timeseries-test.util :as tqpt]
- [metabase.test :as mt]
- [metabase.util :as u]
- [metabase.util.json :as json]))
-
-(deftest ^:parallel results-order-test
- (mt/test-driver :druid
- (testing (str "Make sure Druid cols + columns come back in the same order and that that order is the expected MBQL "
- "columns order (#9294)")
- (tqpt/with-flattened-dbdef
- (let [results (mt/run-mbql-query checkins
- {:limit 1, :order-by [[:asc $timestamp]]})]
- (assert (= (:status results) :completed)
- (u/pprint-to-str 'red results))
- (testing "cols"
- (is (= ["timestamp"
- "venue_name"
- "venue_longitude"
- "venue_latitude"
- "venue_price"
- "venue_category_name"
- "id"
- "count"
- "unique_users"
- "user_name"
- "user_last_login"]
- (->> results :data :cols (map :name)))))
- (testing "rows"
- (is (= [["2013-01-03T00:00:00Z"
- "Kinaree Thai Bistro"
- -118.344
- 34.094
- 1
- "Thai"
- 931
- 1
- "AQAAAQAAAAEBsA=="
- "Simcha Yan"
- "2014-01-01T08:30:00"]]
- (-> results :data :rows)))))))))
-
-(deftest ^:parallel post-process-select-query-test
- (testing "Test that we can still return results from native :select queries, even if we no longer generate them"
- ;; example results adapted from https://github.com/apache/druid/blob/d00747774208dbbfcb272ee7d1c30cf879887838/docs/querying/select-query.md
- (let [results (with-open [r (io/reader "modules/drivers/druid/test/metabase/driver/druid/execute_test/select_results.json")]
- (json/decode+kw r))]
- (is (= {:projections [:a :b :c]
- :results [{:timestamp #t "2013-01-01T00:00Z[UTC]", :robot "1", :namespace "article", :page "11._korpus_(NOVJ)"}
- {:timestamp #t "2013-01-01T00:00Z[UTC]", :robot "0", :namespace "article", :page "112_U.S._580"}
- {:timestamp #t "2013-01-01T00:00Z[UTC]", :robot "0", :namespace "article", :page "113_U.S._243"}
- {:timestamp #t "2013-01-01T00:00Z[UTC]", :robot "0", :namespace "article", :page "113_U.S._73"}
- {:timestamp #t "2013-01-01T00:00Z[UTC]", :robot "0", :namespace "article", :page "113_U.S._756"}]}
- (#'druid.execute/post-process :metabase.driver.druid.query-processor/select [:a :b :c] nil results))))))
diff --git a/modules/drivers/druid/test/metabase/driver/druid/execute_test/select_results.json b/modules/drivers/druid/test/metabase/driver/druid/execute_test/select_results.json
deleted file mode 100644
index f1e9a2e43c0..00000000000
--- a/modules/drivers/druid/test/metabase/driver/druid/execute_test/select_results.json
+++ /dev/null
@@ -1,54 +0,0 @@
-[{
- "timestamp" : "2013-01-01T00:00:00.000Z",
- "result" : {
- "pagingIdentifiers" : {
- "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9" : 4
- },
- "events" : [ {
- "segmentId" : "wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9",
- "offset" : 0,
- "event" : {
- "timestamp" : "2013-01-01T00:00:00.000Z",
- "robot" : "1",
- "namespace" : "article",
- "page" : "11._korpus_(NOVJ)"
- }
- }, {
- "segmentId" : "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9",
- "offset" : 1,
- "event" : {
- "timestamp" : "2013-01-01T00:00:00.000Z",
- "robot" : "0",
- "namespace" : "article",
- "page" : "112_U.S._580"
- }
- }, {
- "segmentId" : "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9",
- "offset" : 2,
- "event" : {
- "timestamp" : "2013-01-01T00:00:00.000Z",
- "robot" : "0",
- "namespace" : "article",
- "page" : "113_U.S._243"
- }
- }, {
- "segmentId" : "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9",
- "offset" : 3,
- "event" : {
- "timestamp" : "2013-01-01T00:00:00.000Z",
- "robot" : "0",
- "namespace" : "article",
- "page" : "113_U.S._73"
- }
- }, {
- "segmentId" : "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9",
- "offset" : 4,
- "event" : {
- "timestamp" : "2013-01-01T00:00:00.000Z",
- "robot" : "0",
- "namespace" : "article",
- "page" : "113_U.S._756"
- }
- } ]
- }
-} ]
diff --git a/modules/drivers/druid/test/metabase/driver/druid/query_processor_test.clj b/modules/drivers/druid/test/metabase/driver/druid/query_processor_test.clj
deleted file mode 100644
index 9984ce4ec4b..00000000000
--- a/modules/drivers/druid/test/metabase/driver/druid/query_processor_test.clj
+++ /dev/null
@@ -1,675 +0,0 @@
-(ns ^:mb/driver-tests metabase.driver.druid.query-processor-test
- "Some tests to make sure the Druid Query Processor is generating sane Druid queries when compiling MBQL."
- {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.driver.druid.query-processor-test]}
- metabase.test.data/run-mbql-query {:namespaces [metabase.driver.druid.query-processor-test]}}}}}}
- (:require
- [clojure.test :refer :all]
- [clojure.tools.macro :as tools.macro]
- [java-time.api :as t]
- [medley.core :as m]
- [metabase.driver :as driver]
- [metabase.driver.common.table-rows-sample :as table-rows-sample]
- [metabase.driver.druid.query-processor :as druid.qp]
- [metabase.lib.test-metadata :as meta]
- [metabase.lib.test-util.macros :as lib.tu.macros]
- [metabase.query-processor.compile :as qp.compile]
- [metabase.query-processor.test :as qp]
- [metabase.query-processor.timeseries-test.util :as tqpt]
- [metabase.test :as mt]
- [metabase.util.date-2 :as u.date]
- [metabase.util.json :as json]
- [toucan2.core :as t2]))
-
-(defn- str->absolute-dt [s]
- [:absolute-datetime (u.date/parse s "UTC") :default])
-
-(deftest ^:parallel filter-intervals-test
- (let [dt-field [:field 1 {:temporal-unit :default}]
- filter-clause->intervals (comp @#'druid.qp/compile-intervals @#'druid.qp/filter-clause->intervals)]
- (testing :=
- (is (= ["2015-10-04T00:00:00Z/2015-10-04T00:00:00.001Z"]
- (filter-clause->intervals [:= dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]))
- ":= filters should get converted to intervals like `v/v+1`")
- (is (= nil
- (filter-clause->intervals [:= [:field 1 nil] "toucan"]))
- "Non-temporal filter clauses should return `nil` intervals"))
- (testing :<
- (is (= ["-5000/2015-10-11T00:00:00Z"]
- (filter-clause->intervals [:< dt-field (str->absolute-dt "2015-10-11T00:00:00Z")]))
- ":<, :<=, :>, and :>= should return an interval with -5000 or 5000 as min or max"))
- (testing :between
- (is (= ["2015-10-04T00:00:00Z/2015-10-20T00:00:00.001Z"]
- (filter-clause->intervals
- [:between dt-field (str->absolute-dt "2015-10-04T00:00:00Z") (str->absolute-dt "2015-10-20T00:00:00Z")]))))
- (testing :and
- (is (= ["2015-10-04T00:00:00Z/2015-10-11T00:00:00Z"]
- (filter-clause->intervals
- [:and
- [:>= dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]
- [:< dt-field (str->absolute-dt "2015-10-11T00:00:00Z")]]))
- "The Druid QP should be able to combine compound `:and` filter clauses into a single datetime interval.")
- (is (= ["2015-10-06T00:00:00Z/2015-10-20T00:00:00.001Z"]
- (filter-clause->intervals
- [:and
- [:between dt-field (str->absolute-dt "2015-10-04T00:00:00Z") (str->absolute-dt "2015-10-20T00:00:00Z")]
- [:between dt-field (str->absolute-dt "2015-10-06T00:00:00Z") (str->absolute-dt "2015-10-21T00:00:00Z")]]))
- "When two filters have overlapping intervals it should generate a single logically equivalent interval")
- (is (= nil
- (filter-clause->intervals
- [:and [:= [:field 1 nil] "toucan"] [:= [:field 2 nil] "threecan"]]))
- ":and clause should ignore non-temporal filters")
- (is (= ["2015-10-04T00:00:00Z/2015-10-04T00:00:00.001Z"]
- (filter-clause->intervals
- [:and
- [:= [:field 1 nil] "toucan"] [:= dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]]))
- ":and clause with no temporal filters should be compiled to `nil` interval")
- (is (= ["2015-10-04T00:00:00Z/2015-10-04T00:00:00.001Z"]
- (filter-clause->intervals
- [:and
- [:= dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]
- [:or
- [:>= dt-field (str->absolute-dt "2015-10-03T00:00:00Z")]
- [:< dt-field (str->absolute-dt "2015-10-11T00:00:00Z")]]]))
- ":and clause should ignore nested `:or` filters, since they can't be combined into a single filter"))
- (testing :or
- (is (= ["2015-10-04T00:00:00Z/5000" "-5000/2015-10-11T00:00:00Z"]
- (filter-clause->intervals
- [:or
- [:>= dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]
- [:< dt-field (str->absolute-dt "2015-10-11T00:00:00Z")]]))
- ":or filters should be combined into multiple intervals")
- (is (= ["2015-10-04T00:00:00Z/5000"]
- (filter-clause->intervals
- [:or
- [:>= dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]
- [:= [:field 1 nil] "toucan"]]))
- ":or clauses should ignore non-temporal filters")
- (is (= nil
- (filter-clause->intervals
- [:or
- [:= [:field 1 nil] "toucan"]
- [:= [:field 2 nil] "threecan"]]))
- ":or filters with no temporal filters should return nil"))
- (testing :not
- (is (= ["2015-10-04T00:00:00Z/2015-10-11T00:00:00Z"]
- (filter-clause->intervals
- [:not
- [:or
- [:< dt-field (str->absolute-dt "2015-10-04T00:00:00Z")]
- [:>= dt-field (str->absolute-dt "2015-10-11T00:00:00Z")]]]))))))
-
-(defn- do-query->native [query]
- (driver/with-driver :druid
- (tqpt/with-flattened-dbdef
- (binding [druid.qp/*random-query-id* (constantly "<Query ID>")]
- (qp.compile/compile query)))))
-
-(defmacro ^:private query->native [query]
- `(do-query->native
- (mt/mbql-query ~'checkins
- ~query)))
-
-(deftest ^:parallel compile-topN-test
- (mt/test-driver :druid
- (tqpt/with-flattened-dbdef
- (is (= {:projections [:venue_price :__count :expression]
- :query {:queryType :topN
- :threshold 1000
- :granularity :all
- :dataSource "checkins"
- :dimension "venue_price"
- :context {:queryId "<Query ID>"}
- :postAggregations [{:type :arithmetic
- :name "expression"
- :fn :*
- :fields [{:type :fieldAccess, :fieldName "__count"}
- {:type :constant, :name "10", :value 10}]}]
- :intervals ["1900-01-01/2100-01-01"]
- :metric {:type :alphaNumeric}
- :aggregations
- [{:type :filtered
- :filter {:type :not
- :field {:type :selector, :dimension "id", :value nil}}
- :aggregator {:type :count, :name "__count"}}]}
- :query-type ::druid.qp/topN
- :mbql? true}
- (query->native
- #_:clj-kondo/ignore
- {:aggregation [[:* [:count $id] 10]]
- :breakout [$venue_price]}))))))
-
-(deftest ^:parallel compile-topN-with-order-by-test
- (mt/test-driver :druid
- (tqpt/with-flattened-dbdef
- (is (= {:projections [:venue_category_name :__count_0]
- :query {:queryType :topN
- :threshold 1000
- :granularity :all
- :dataSource "checkins"
- :dimension "venue_category_name"
- :context {:queryId "<Query ID>"}
- :intervals ["1900-01-01/2100-01-01"]
- :metric "__count_0"
- :aggregations [{:type :cardinality
- :name "__count_0"
- :fieldNames ["venue_name"]
- :byRow true
- :round true}]}
- :query-type ::druid.qp/topN
- :mbql? true}
- (query->native
- #_:clj-kondo/ignore
- {:aggregation [[:aggregation-options [:distinct $checkins.venue_name] {:name "__count_0"}]]
- :breakout [$venue_category_name]
- :order-by [[:desc [:aggregation 0]] [:asc $checkins.venue_category_name]]}))))))
-
-(deftest ^:parallel compile-groupBy-test
- (mt/test-driver :druid
- (tqpt/with-flattened-dbdef
- (is (= {:projections [:venue_category_name :user_name :__count_0]
- :query {:queryType :groupBy
- :granularity :all
- :dataSource "checkins"
- :dimensions ["venue_category_name", "user_name"]
- :context {:queryId "<Query ID>"}
- :intervals ["1900-01-01/2100-01-01"]
- :aggregations [{:type :cardinality
- :name "__count_0"
- :fieldNames ["venue_name"]
- :byRow true
- :round true}]
- :limitSpec {:type :default
- :columns [{:dimension "__count_0", :direction :descending}
- {:dimension "venue_category_name", :direction :ascending}
- {:dimension "user_name", :direction :ascending}]}}
- :query-type ::druid.qp/groupBy
- :mbql? true}
- (query->native
- #_:clj-kondo/ignore
- {:aggregation [[:aggregation-options [:distinct $checkins.venue_name] {:name "__count_0"}]]
- :breakout [$venue_category_name $user_name]
- :order-by [[:desc [:aggregation 0]] [:asc $checkins.venue_category_name]]}))))))
-
-(deftest ^:parallel compile-groupBy-with-limit-test
- (mt/test-driver :druid
- (tqpt/with-flattened-dbdef
- (is (= {:projections [:venue_category_name :user_name :__count_0]
- :query {:queryType :groupBy
- :granularity :all
- :dataSource "checkins"
- :dimensions ["venue_category_name", "user_name"]
- :context {:queryId "<Query ID>"}
- :intervals ["1900-01-01/2100-01-01"]
- :aggregations [{:type :cardinality
- :name "__count_0"
- :fieldNames ["venue_name"]
- :byRow true
- :round true}]
- :limitSpec {:type :default
- :columns [{:dimension "__count_0", :direction :descending}
- {:dimension "venue_category_name", :direction :ascending}
- {:dimension "user_name", :direction :ascending}]
- :limit 5}}
- :query-type ::druid.qp/groupBy
- :mbql? true}
- (query->native
- {:aggregation [[:aggregation-options [:distinct $checkins.venue_name] {:name "__count_0"}]]
- :breakout [$venue_category_name $user_name]
- :order-by [[:desc [:aggregation 0]] [:asc $checkins.venue_category_name]]
- :limit 5}))))))
-
-(deftest ^:parallel finalizing-field-access-test
- (mt/test-driver :druid
- (tqpt/with-flattened-dbdef
- (testing "`distinct` when used in post aggregations should have type `:finalizingFieldAccess`"
- (is (= {:projections [:__distinct_0 :expression]
- :query {:queryType :timeseries
- :granularity :all
- :dataSource "checkins"
- :context {:queryId "<Query ID>"}
- :intervals ["1900-01-01/2100-01-01"]
- :aggregations [{:type :cardinality
- :name "__distinct_0"
- :fieldNames ["venue_name"]
- :byRow true
- :round true}]
- :postAggregations [{:type :arithmetic,
- :name "expression",
- :fn :+,
- :fields
- [{:type :constant, :name "1", :value 1}
- {:type :finalizingFieldAccess, :fieldName "__distinct_0"}]}]}
- :query-type ::druid.qp/total
- :mbql? true}
- (query->native
- {:aggregation [[:+ 1 [:aggregation-options [:distinct $checkins.venue_name] {:name "__distinct_0"}]]]})))))))
-
-(defn- table-rows-sample []
- (->> (table-rows-sample/table-rows-sample (t2/select-one :model/Table :id (mt/id :checkins))
- [(t2/select-one :model/Field :id (mt/id :checkins :id))
- (t2/select-one :model/Field :id (mt/id :checkins :venue_name))
- (t2/select-one :model/Field :id (mt/id :checkins :timestamp))]
- (constantly conj))
- (sort-by first)
- (take 5)))
-
-(deftest table-rows-sample-test
- (mt/test-driver :druid
- (tqpt/with-flattened-dbdef
- (testing "Druid driver doesn't need to convert results to the expected timezone for us. QP middleware can handle that."
- (let [expected [[1 "The Misfit Restaurant + Bar" (t/instant "2014-04-07T00:00:00Z")]
- [2 "Bludso's BBQ" (t/instant "2014-09-18T00:00:00Z")]
- [3 "Philippe the Original" (t/instant "2014-09-15T00:00:00Z")]
- [4 "Wurstküche" (t/instant "2014-03-11T00:00:00Z")]
- [5 "Hotel Biron" (t/instant "2013-05-05T00:00:00Z")]]]
- (testing "UTC timezone"
- (is (= expected
- (table-rows-sample))))
- (mt/with-temporary-setting-values [report-timezone "America/Los_Angeles"]
- (is (= expected
- (table-rows-sample))))
- (mt/with-system-timezone-id! "America/Chicago"
- (is (= expected
- (table-rows-sample)))))))))
-
-(def ^:private native-query-1
- (json/encode
- {:queryType :scan
- :dataSource :checkins
- :intervals ["1900-01-01/2100-01-01"]
- :granularity :all
- :limit 2
- :columns [:id
- :user_name
- :venue_price
- :venue_name
- :count]}))
-
-(defn- process-native-query [query]
- (driver/with-driver :druid
- (tqpt/with-flattened-dbdef
- (-> (qp/process-query {:native {:query query}
- :type :native
- :database (mt/id)})
- (m/dissoc-in [:data :results_metadata])))))
-
-(deftest ^:parallel native-query-test
- (mt/test-driver :druid
- (is (partial=
- {:row_count 2
- :status :completed
- :data {:rows [[931 "Simcha Yan" 1 "Kinaree Thai Bistro" 1]
- [285 "Kfir Caj" 2 "Ruen Pair Thai Restaurant" 1]]
- :cols [{:name "id"
- :source :native
- :display_name "id"
- :field_ref [:field "id" {:base-type :type/Integer}]
- :base_type :type/Integer
- :effective_type :type/Integer}
- {:name "user_name"
- :source :native
- :display_name "user_name"
- :base_type :type/Text
- :effective_type :type/Text
- :field_ref [:field "user_name" {:base-type :type/Text}]}
- {:name "venue_price"
- :source :native
- :display_name "venue_price"
- :base_type :type/Integer
- :effective_type :type/Integer
- :field_ref [:field "venue_price" {:base-type :type/Integer}]}
- {:name "venue_name"
- :source :native
- :display_name "venue_name"
- :base_type :type/Text
- :effective_type :type/Text
- :field_ref [:field "venue_name" {:base-type :type/Text}]}
- {:name "count"
- :source :native
- :display_name "count"
- :base_type :type/Integer
- :effective_type :type/Integer
- :field_ref [:field "count" {:base-type :type/Integer}]}]
- :native_form {:query native-query-1}
- :results_timezone "UTC"}}
- (-> (process-native-query native-query-1)
- (m/dissoc-in [:data :insights]))))))
-
-(def ^:private native-query-2
- (json/encode
- {:intervals ["1900-01-01/2100-01-01"]
- :granularity {:type :period
- :period :P1M
- :timeZone :UTC}
- :queryType :timeseries
- :dataSource :checkins
- :aggregations [{:type :count
- :name :count}]}))
-
-(deftest ^:parallel native-query-test-2
- (testing "make sure we can run a native :timeseries query. This was throwing an Exception -- see #3409"
- (mt/test-driver :druid
- (is (= :completed
- (:status (process-native-query native-query-2)))))))
-
-(defmacro ^:private druid-query {:style/indent 0} [& body]
- `(tqpt/with-flattened-dbdef
- (qp/process-query
- (mt/mbql-query ~'checkins
- ~@body))))
-
-(defmacro ^:private druid-query-returning-rows {:style/indent 0} [& body]
- `(mt/rows (druid-query ~@body)))
-
-(deftest ^:parallel start-of-week-test
- (mt/test-driver :druid
- (testing "Count the number of events in the given week."
- (is (= [["2015-10-04" 9]]
- (druid-query-returning-rows
- {:filter [:between !day.timestamp "2015-10-04" "2015-10-10"]
- :aggregation [[:count $id]]
- :breakout [!week.timestamp]}))))))
-
-(deftest ^:parallel sum-aggregation-test
- (mt/test-driver :druid
- (testing "sum, *"
- (is (= [["1" 110688.0]
- ["2" 616708.0]
- ["3" 179661.0]
- ["4" 86284.0]]
- (druid-query-returning-rows
- {:aggregation [[:sum [:* $id $venue_price]]]
- :breakout [$venue_price]}))))))
-
-(deftest ^:parallel min-aggregation-test
- (mt/test-driver :druid
- (testing "min, +"
- (is (= [["1" 4.0]
- ["2" 3.0]
- ["3" 8.0]
- ["4" 12.0]]
- (druid-query-returning-rows
- {:aggregation [[:min [:+ $id $venue_price]]]
- :breakout [$venue_price]}))))))
-
-(deftest ^:parallel max-aggregation-test
- (mt/test-driver :druid
- (testing "max, /"
- (is (= [["1" 1000.0]
- ["2" 499.5]
- ["3" 332.0]
- ["4" 248.25]]
- (druid-query-returning-rows
- {:aggregation [[:max [:/ $id $venue_price]]]
- :breakout [$venue_price]}))))))
-
-(deftest ^:parallel avg-aggregation-test
- (mt/test-driver :druid
- (testing "avg, -"
- (is (= [["1" 500.85067873303166]
- ["2" 1002.7772357723577]
- ["3" 1562.2695652173913]
- ["4" 1760.8979591836735]]
- (druid-query-returning-rows
- {:aggregation [[:avg [:* $id $venue_price]]]
- :breakout [$venue_price]}))))))
-
-(deftest ^:parallel share-aggregation-test
- (mt/test-driver :druid
- (testing "share"
- (is (= [[0.951]]
- (druid-query-returning-rows
- {:aggregation [[:share [:< $venue_price 4]]]}))))))
-
-(deftest ^:parallel count-where-aggregation-test
- (mt/test-driver :druid
- (testing "count-where"
- (is (= [[951]]
- (druid-query-returning-rows
- {:aggregation [[:count-where [:< $venue_price 4]]]}))))))
-
-(deftest ^:parallel sum-where-aggregation-test
- (mt/test-driver :druid
- (testing "sum-where"
- (is (= [[1796.0]]
- (druid-query-returning-rows
- {:aggregation [[:sum-where $venue_price [:< $venue_price 4]]]}))))))
-
-(deftest ^:parallel count-aggregation-test
- (mt/test-driver :druid
- (testing "aggregation w/o field"
- (is (= [["1" 222.0]
- ["2" 616.0]
- ["3" 116.0]
- ["4" 50.0]]
- (druid-query-returning-rows
- {:aggregation [[:+ 1 [:count]]]
- :breakout [$venue_price]}))))))
-
-(deftest ^:parallel expression-aggregations-test
- (mt/test-driver :druid
- (testing "post-aggregation math w/ 2 args: count + sum"
- (is (= [["1" 442.0]
- ["2" 1845.0]
- ["3" 460.0]
- ["4" 245.0]]
- (druid-query-returning-rows
- {:aggregation [[:+ [:count $id] [:sum $venue_price]]]
- :breakout [$venue_price]}))))
- (testing "post-aggregation math w/ 3 args: count + sum + count"
- (is (= [["1" 663.0]
- ["2" 2460.0]
- ["3" 575.0]
- ["4" 294.0]]
- (druid-query-returning-rows
- {:aggregation [[:+
- [:count $id]
- [:sum $venue_price]
- [:count $venue_price]]]
- :breakout [$venue_price]}))))
- (testing "post-aggregation math w/ a constant: count * 10"
- (is (= [["1" 2210.0]
- ["2" 6150.0]
- ["3" 1150.0]
- ["4" 490.0]]
- (druid-query-returning-rows
- {:aggregation [[:* [:count $id] 10]]
- :breakout [$venue_price]}))))
- (testing "nested post-aggregation math: count + (count * sum)"
- (is (= [["1" 49062.0]
- ["2" 757065.0]
- ["3" 39790.0]
- ["4" 9653.0]]
- (druid-query-returning-rows
- {:aggregation [[:+
- [:count $id]
- [:* [:count $id] [:sum $venue_price]]]]
- :breakout [$venue_price]}))))
- (testing "post-aggregation math w/ avg: count + avg"
- (is (= [["1" 721.8506787330316]
- ["2" 1116.388617886179]
- ["3" 635.7565217391304]
- ["4" 489.2244897959184]]
- (druid-query-returning-rows
- {:aggregation [[:+ [:count $id] [:avg $id]]]
- :breakout [$venue_price]}))))
- (testing "aggregation with math inside the aggregation :scream_cat:"
- (is (= [["1" 442.0]
- ["2" 1845.0]
- ["3" 460.0]
- ["4" 245.0]]
- (druid-query-returning-rows
- {:aggregation [[:sum [:+ $venue_price 1]]]
- :breakout [$venue_price]}))))
- (testing "post aggregation math + math inside aggregations: max(venue_price) + min(venue_price - id)"
- (is (= [["1" -998.0]
- ["2" -995.0]
- ["3" -990.0]
- ["4" -985.0]]
- (druid-query-returning-rows
- {:aggregation [[:+
- [:max $venue_price]
- [:min [:- $venue_price $id]]]]
- :breakout [$venue_price]}))))))
-
-(deftest ^:parallel named-top-level-aggregation-test
- (mt/test-driver :druid
- (testing "check that we can name an expression aggregation w/ aggregation at top-level"
- (is (= [["1" 442.0]
- ["2" 1845.0]
- ["3" 460.0]
- ["4" 245.0]]
- (mt/rows
- (druid-query
- {:aggregation [[:aggregation-options [:sum [:+ $venue_price 1]] {:name "New Price"}]]
- :breakout [$venue_price]})))))))
-
-(deftest ^:parallel named-expression-aggregations-test
- (mt/test-driver :druid
- (testing "check that we can name an expression aggregation w/ expression at top-level"
- (is (= {:rows [["1" 180.0]
- ["2" 1189.0]
- ["3" 304.0]
- ["4" 155.0]]
- :columns ["venue_price" "Sum-41"]}
- (mt/rows+column-names
- (druid-query
- {:aggregation [[:aggregation-options [:- [:sum $venue_price] 41] {:name "Sum-41"}]]
- :breakout [$venue_price]})))))))
-
-(deftest ^:parallel distinct-count-of-two-dimensions-test
- (mt/test-driver :druid
- (is (= {:rows [[979]]
- :columns ["count"]}
- (mt/rows+column-names
- (druid-query
- {:aggregation [[:distinct [:+ $id $checkins.venue_price]]]}))))))
-
-(deftest ^:parallel order-by-aggregation-test
- (mt/test-driver :druid
- (doseq [[direction expected-rows] {:desc [["Bar" "Felipinho Asklepios" 8]
- ["Bar" "Spiros Teofil" 8]
- ["Japanese" "Felipinho Asklepios" 7]
- ["Japanese" "Frans Hevel" 7]
- ["Mexican" "Shad Ferdynand" 7]]
- :asc [["American" "Rüstem Hebel" 1]
- ["Artisan" "Broen Olujimi" 1]
- ["Artisan" "Conchúr Tihomir" 1]
- ["Artisan" "Dwight Gresham" 1]
- ["Artisan" "Plato Yeshua" 1]]}]
- (testing direction
- (is (= expected-rows
- (druid-query-returning-rows
- {:aggregation [[:aggregation-options [:distinct $checkins.venue_name] {:name "__count_0"}]]
- :breakout [$venue_category_name $user_name]
- :order-by [[direction [:aggregation 0]] [:asc $checkins.venue_category_name]]
- :limit 5})))))))
-
-(deftest ^:parallel hll-count-test
- (mt/test-driver :druid
- (testing "Do we generate the correct count clause for HLL fields?"
- (is (= [["Bar" "Szymon Theutrich" 13]
- ["Mexican" "Dwight Gresham" 12]
- ["American" "Spiros Teofil" 10]
- ["Bar" "Felipinho Asklepios" 10]
- ["Bar" "Kaneonuskatew Eiran" 10]]
- (druid-query-returning-rows
- {:aggregation [[:aggregation-options [:count $checkins.user_name] {:name "unique_users"}]]
- :breakout [$venue_category_name $user_name]
- :order-by [[:desc [:aggregation 0]] [:asc $checkins.venue_category_name]]
- :limit 5}))))))
-
-(deftest ^:parallel numeric-filter-test
- (mt/test-driver :druid
- (tqpt/with-flattened-dbdef
- (letfn [(compiled [query]
- (-> (qp.compile/compile query) :query (select-keys [:filter :queryType])))]
- (doseq [[message field] {"Make sure we can filter by numeric columns (#10935)" :venue_price
- "We should be able to filter by Metrics (#11823)" :count}
- :let [field-clause [:field (mt/id :checkins field) nil]
- field-name (name field)]]
- (testing message
- (testing "scan query"
- (let [query (mt/mbql-query checkins
- {:fields [$id $venue_price $venue_name]
- :filter [:= field-clause 1]
- :order-by [[:desc $id]]
- :limit 5})]
- (is (= {:filter {:type :selector, :dimension field-name, :value 1}
- :queryType :scan}
- (compiled query)))
- (is (= [931 1 "Kinaree Thai Bistro"]
- (mt/first-row (qp/process-query query))))))
- (testing "topN query"
- (let [query (mt/mbql-query checkins
- {:aggregation [[:count]]
- :breakout [$venue_price]
- :filter [:= field-clause 1]})]
- (is (= {:filter {:type :selector, :dimension field-name, :value 1}
- :queryType :topN}
- (compiled query)))
- (is (= ["1" 221]
- (mt/first-row (qp/process-query query))))))
- (testing "groupBy query"
- (let [query (mt/mbql-query checkins
- {:aggregation [[:aggregation-options [:distinct $checkins.venue_name] {:name "__count_0"}]]
- :breakout [$venue_category_name $user_name]
- :order-by [[:desc [:aggregation 0]] [:asc $checkins.venue_category_name]]
- :filter [:= field-clause 1]})]
- (is (= {:filter {:type :selector, :dimension field-name, :value 1}
- :queryType :groupBy}
- (compiled query)))
- (is (= (case field
- :count ["Bar" "Felipinho Asklepios" 8]
- :venue_price ["Mexican" "Conchúr Tihomir" 4])
- (mt/first-row (qp/process-query query))))))
- (testing "timeseries query"
- (let [query (mt/mbql-query checkins
- {:aggregation [[:count]]
- :filter [:= field-clause 1]})]
- (is (= {:queryType :timeseries
- :filter {:type :selector, :dimension field-name, :value 1}}
- (compiled query)))
- (is (= (case field
- :count [1000]
- :venue_price [221])
- (mt/first-row (qp/process-query query))))))))))))
-
-(deftest ^:parallel parse-filter-test
- (testing "parse-filter should generate the correct filter clauses"
- (mt/with-metadata-provider meta/metadata-provider
- (tools.macro/macrolet [(parse-filter [filter-clause]
- `(#'druid.qp/parse-filter (lib.tu.macros/$ids :checkins ~filter-clause)))]
- (testing "normal non-compound filters should work as expected"
- (is (= {:type :selector, :dimension "PRICE", :value 2}
- (parse-filter [:= $venues.price [:value 2 {:base_type :type/Integer}]]))))
- (testing "temporal filters should get stripped out"
- (is (= nil
- (parse-filter [:>= !checkins.date [:absolute-datetime #t "2015-09-01T00:00Z[UTC]" :default]])))
- (is (= {:type :selector, :dimension "NAME", :value "Mexican"}
- (parse-filter
- [:and
- [:= $categories.name [:value "Mexican" {:base_type :type/Text}]]
- [:< !checkins.date [:absolute-datetime #t "2015-10-01T00:00Z[UTC]" :default]]]))))))))
-
-(deftest ^:parallel multiple-filters-test
- (mt/test-driver :druid
- (testing "Should be able to filter by both a temporal and a non-temporal filter (#15903)"
- (tqpt/with-flattened-dbdef
- (is (= [4]
- (mt/first-row
- (mt/run-mbql-query checkins
- {:aggregation [[:count]]
- :filter [:and
- [:= $venue_category_name "Mexican"]
- [:= !month.timestamp "2015-09"]]}))))))))
-
-(deftest ^:parallel open-ended-temporal-filter-test
- (mt/test-driver :druid
- (testing "Should be able to filter by an open-ended absolute temporal moment (#15902)"
- (tqpt/with-flattened-dbdef
- (is (= [58]
- (mt/first-row
- (mt/run-mbql-query checkins
- {:aggregation [[:count]]
- :filter [:> $timestamp "2015-10-01T00:00:00Z"]}))))))))
diff --git a/modules/drivers/druid/test/metabase/driver/druid/sync_test.clj b/modules/drivers/druid/test/metabase/driver/druid/sync_test.clj
deleted file mode 100644
index 6dd16223db3..00000000000
--- a/modules/drivers/druid/test/metabase/driver/druid/sync_test.clj
+++ /dev/null
@@ -1,82 +0,0 @@
-(ns ^:mb/driver-tests metabase.driver.druid.sync-test
- (:require
- [clojure.test :refer :all]
- [malli.error :as me]
- [metabase.driver :as driver]
- [metabase.driver.druid.client :as druid.client]
- [metabase.driver.druid.sync :as druid.sync]
- [metabase.query-processor.timeseries-test.util :as tqpt]
- [metabase.secrets.models.secret :as secret]
- [metabase.sync.sync-metadata.dbms-version :as sync-dbms-ver]
- [metabase.test :as mt]
- [metabase.util :as u]
- [metabase.util.malli.registry :as mr]
- [toucan2.core :as t2]))
-
-(deftest ^:parallel sync-test
- (mt/test-driver :druid
- (tqpt/with-flattened-dbdef
- (testing "describe-database"
- (is (= {:tables #{{:schema nil, :name "checkins"}}}
- (driver/describe-database :druid (mt/db)))))
- (testing "describe-table"
- (is (= {:schema nil
- :name "checkins"
- :fields [{:name "timestamp", :base-type :type/Instant, :database-type "timestamp", :database-position 0, :pk? false}
- {:name "venue_name", :base-type :type/Text, :database-type "STRING", :database-position 1}
- {:name "user_password", :base-type :type/Text, :database-type "STRING", :database-position 2}
- {:name "venue_longitude", :base-type :type/Float, :database-type "DOUBLE", :database-position 3}
- {:name "venue_latitude", :base-type :type/Float, :database-type "DOUBLE", :database-position 4}
- {:name "venue_price", :base-type :type/Integer, :database-type "LONG", :database-position 5}
- {:name "venue_category_name", :base-type :type/Text, :database-type "STRING", :database-position 6}
- {:name "id", :base-type :type/Integer, :database-type "LONG", :database-position 7}
- {:name "count", :base-type :type/Integer, :database-type "LONG [metric]", :database-position 8}
- {:name "unique_users", :base-type :type/DruidHyperUnique, :database-type "hyperUnique [metric]", :database-position 9}
- {:name "user_name", :base-type :type/Text, :database-type "STRING", :database-position 10}
- {:name "user_last_login", :base-type :type/Text, :database-type "STRING", :database-position 11}]}
- (-> (driver/describe-table :druid (mt/db) {:name "checkins"})
- (update :fields (partial sort-by :database-position)))))))))
-
-(defn- db-dbms-version [db-or-id]
- (t2/select-one-fn :dbms_version :model/Database :id (u/the-id db-or-id)))
-
-(defn- check-dbms-version [dbms-version]
- (me/humanize (mr/explain sync-dbms-ver/DBMSVersion dbms-version)))
-
-(deftest dbms-version-test
- (mt/test-driver :druid
- (testing (str "This tests populating the dbms_version field for a given database."
- " The sync happens automatically, so this test removes it first"
- " to ensure that it gets set when missing.")
- (tqpt/with-flattened-dbdef
- (let [db (mt/db)
- version-on-load (db-dbms-version db)
- _ (t2/update! :model/Database (u/the-id db) {:dbms_version nil})
- db (t2/select-one :model/Database :id (u/the-id db))
- version-after-update (db-dbms-version db)
- _ (sync-dbms-ver/sync-dbms-version! db)]
- (testing "On startup is the dbms-version specified?"
- (is (nil? (check-dbms-version version-on-load))))
- (testing "Check to make sure the test removed the timezone"
- (is (nil? version-after-update)))
- (testing "Check that the value was set again after sync"
- (is (nil? (check-dbms-version (db-dbms-version db))))))))))
-
-(deftest ^:synchronized auth-dbms-version-test
- (mt/test-driver
- :druid
- (testing "`dbms-version` uses auth parameters (#41579)"
- (tqpt/with-flattened-dbdef
- (let [get-args (volatile! nil)
- db-with-auth-details (update (mt/db) :details assoc
- :auth-enabled true
- :auth-username "admin"
- :auth-token-value "password1")]
- ;; Following ensures that auth parameters are passed to GET during version sync if present.
- (with-redefs [secret/value-as-string (constantly "password1")
- druid.client/GET (fn [_url & params] (vreset! get-args (apply hash-map params)) nil)]
- ;; Just fill in the params with internally modified `dbms-version`.
- (druid.sync/dbms-version db-with-auth-details)
- (is (true? (:auth-enabled @get-args)))
- (is (= "admin" (:auth-username @get-args)))
- (is (= "password1" (:auth-token-value @get-args)))))))))
diff --git a/modules/drivers/druid/test/metabase/test/data/druid.clj b/modules/drivers/druid/test/metabase/test/data/druid.clj
deleted file mode 100644
index 8e9988a4465..00000000000
--- a/modules/drivers/druid/test/metabase/test/data/druid.clj
+++ /dev/null
@@ -1,60 +0,0 @@
-(ns metabase.test.data.druid
- (:require
- [clojure.string :as str]
- [metabase.driver.druid.client :as druid.client]
- [metabase.test.data.impl :as data.impl]
- [metabase.test.data.interface :as tx]
- [metabase.util :as u]))
-
-(set! *warn-on-reflection* true)
-
-(tx/add-test-extensions! :druid)
-
-(defn- host []
- ;; force Druid to use localhost for now -- this is until we remove the env var in CircleCI so it doesn't override
- ;; this value.
- "http://localhost"
- #_(let [host (tx/db-test-env-var-or-throw :druid :host "localhost")]
- (cond->> host
- (not (str/starts-with? host "http")) (str "http://"))))
-
-(defn- broker-port []
- (Integer/parseUnsignedInt (tx/db-test-env-var-or-throw :druid :port "8082")))
-
-(defn- broker-port-up? []
- (u/host-port-up? (str/replace (host) #"^https?://" "") (broker-port)))
-
-(defmethod tx/dbdef->connection-details :druid
- [& _]
- {:host (host)
- :port (broker-port)})
-
-(defn- datasources-endpoint []
- (format "%s:%d/druid/v2/datasources" (host) (broker-port)))
-
-(defn- datasources []
- {:pre [(broker-port-up?)]}
- (druid.client/GET (datasources-endpoint)))
-
-(defn- already-loaded []
- (set (datasources)))
-
-(defmethod tx/create-db! :druid
- [_ dbdef & _]
- (let [{:keys [database-name], :as _dbdef} (tx/get-dataset-definition dbdef)]
- (assert (= database-name "checkins")
- "Druid tests currently only support the flattened test-data dataset.")
- (assert (contains? (already-loaded) "checkins")
- "Expected 'checkins' dataset to be present in Druid datasources. (This should be loaded as part of building Docker image)")
- nil))
-
-;; NO-OP
-(defmethod tx/destroy-db! :druid
- [& _]
- nil)
-
-;; no-op -- because the names of the columns actually loaded by Druid differ from ones in the database definition, the
-;; default impl will fail. TODO -- we should write an implementation that works for Druid
-(defmethod data.impl/verify-data-loaded-correctly :druid
- [_ _ _]
- nil)
diff --git a/resources/migrations/063/20260602_migrate_legacy_druid_driver.yaml b/resources/migrations/063/20260602_migrate_legacy_druid_driver.yaml
new file mode 100644
index 00000000000..2cd50c83fd8
--- /dev/null
+++ b/resources/migrations/063/20260602_migrate_legacy_druid_driver.yaml
@@ -0,0 +1,14 @@
+## TODO: quick description of what feature/bug this migration is for
+
+databaseChangeLog:
+ - changeSet:
+ id: v63.9lrcl8
+ author: camsaul
+ comment: Migrate databases using legacy druid driver to druid-jdbc
+ changes:
+ - sql:
+ sql: >-
+ UPDATE metabase_database
+ SET engine = 'druid-jdbc'
+ WHERE engine = 'druid';
+ rollback: # nothing to do
diff --git a/src/metabase/analyze/classifiers/name.clj b/src/metabase/analyze/classifiers/name.clj
index 8f74c9752bf..f101ec5958b 100644
--- a/src/metabase/analyze/classifiers/name.clj
+++ b/src/metabase/analyze/classifiers/name.clj
@@ -4,7 +4,6 @@
[clojure.string :as str]
[metabase.analyze.schema :as analyze.schema]
[metabase.config.core :as config]
- [metabase.driver.util :as driver.u]
[metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint]
[metabase.sync.util :as sync-util]
[metabase.util :as u]
@@ -189,7 +188,4 @@
(when (re-find pattern table-name)
type))
entity-types-patterns)
- (case (some-> (:db_id table) driver.u/database->driver)
- :druid :entity/EventTable
- nil)
:entity/GenericTable))))
diff --git a/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj b/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj
index e96bf366b9f..12ce3db1ef5 100644
--- a/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj
+++ b/test/metabase/driver/sql_jdbc/sync/describe_table_test.clj
@@ -780,7 +780,6 @@
(doseq [driver [:presto-jdbc
:h2 ;; TODO
- :druid
:druid-jdbc]]
(defmethod driver/database-supports? [driver ::describe-view-fields]
[_driver _feature _database]
@@ -790,7 +789,6 @@
:h2 ;; TODO
:snowflake ;; Requires enterprise account
:presto-jdbc
- :druid
:druid-jdbc
:mysql
:sqlserver
diff --git a/test/metabase/query_processor/alternative_date_test.clj b/test/metabase/query_processor/alternative_date_test.clj
index ca657a18ac5..efa98ac9c23 100644
--- a/test/metabase/query_processor/alternative_date_test.clj
+++ b/test/metabase/query_processor/alternative_date_test.clj
@@ -425,7 +425,6 @@
:bigquery-cloud-sdk
:clickhouse
:databricks
- :druid
:snowflake
:sparksql
:sqlserver
diff --git a/test/metabase/query_processor/timeseries_test/util.clj b/test/metabase/query_processor/timeseries_test/util.clj
index 65283ecabc6..8deeb9a9f6d 100644
--- a/test/metabase/query_processor/timeseries_test/util.clj
+++ b/test/metabase/query_processor/timeseries_test/util.clj
@@ -2,11 +2,12 @@
"Utility functions and macros for testing timeseries database drivers, such as Druid."
(:require
[metabase.test :as mt]
+ [metabase.test.data :as test.data]
[metabase.test.data.dataset-definitions :as defs]
[metabase.test.data.interface :as tx]))
(defn timeseries-drivers []
- #{:druid :druid-jdbc})
+ test.data/timeseries-drivers)
(def flattened-db-def
"The normal test-data DB definition as a flattened, single-table DB definition."
diff --git a/test/metabase/test/data.clj b/test/metabase/test/data.clj
index 13a0d2c9358..22062e95f28 100644
--- a/test/metabase/test/data.clj
+++ b/test/metabase/test/data.clj
@@ -322,7 +322,7 @@
;; Non-"normal" timeseries drivers are tested in [[metabase.query-processor.timeseries-test]] and elsewhere
(def timeseries-drivers
"Drivers that are so weird that we can't use the standard dataset loading against them."
- #{:druid :druid-jdbc})
+ #{:druid-jdbc})
(mr/def ::driver-selector
[:map {:closed true}