Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Transforms DB Connection (#68883)

Timothy S. Dean4ab6f9df02e8
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 47798754d77..1eaf8981e41 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -547,6 +547,8 @@
metabase.driver.common
metabase.driver.common.parameters
metabase.driver.common.parameters.parse
+ metabase.driver.connection
+ metabase.driver.connection.workspaces
metabase.driver.ddl.interface
metabase.driver.h2
metabase.driver.impl
@@ -569,7 +571,8 @@
metabase.driver.sql.util
metabase.driver.sync
metabase.driver.util}
- :uses #{app-db
+ :uses #{analytics
+ app-db
appearance
auth-provider
classloader
diff --git a/e2e/support/helpers/e2e-request-helpers.js b/e2e/support/helpers/e2e-request-helpers.ts
similarity index 52%
rename from e2e/support/helpers/e2e-request-helpers.js
rename to e2e/support/helpers/e2e-request-helpers.ts
index d19330a80eb..0b114742ac7 100644
--- a/e2e/support/helpers/e2e-request-helpers.js
+++ b/e2e/support/helpers/e2e-request-helpers.ts
@@ -24,7 +24,27 @@
export function spyRequestFinished(name = "requestFinishedSpy") {
const spy = cy.spy().as(name);
return {
- interceptor: (req) => req.continue((res) => spy(req, res)),
+ interceptor: (req: any) => req.continue((res: any) => spy(req, res)),
spy,
};
}
+
+const WAIT_TIMEOUT = 10000;
+const WAIT_INTERVAL = 100;
+
+export function retryRequest<T>(
+ callback: () => Cypress.Chainable<T>,
+ condition: (result: T) => boolean,
+ timeout = WAIT_TIMEOUT,
+): Cypress.Chainable<T> {
+ return callback().then((result) => {
+ if (condition(result)) {
+ return cy.wrap(result);
+ } else if (timeout > 0) {
+ cy.wait(WAIT_INTERVAL);
+ return retryRequest(callback, condition, timeout - WAIT_INTERVAL);
+ } else {
+ throw new Error("Retry timeout");
+ }
+ });
+}
diff --git a/e2e/support/helpers/e2e-transform-helpers.ts b/e2e/support/helpers/e2e-transform-helpers.ts
index f6ba93fcec7..8d7ca36d99d 100644
--- a/e2e/support/helpers/e2e-transform-helpers.ts
+++ b/e2e/support/helpers/e2e-transform-helpers.ts
@@ -2,16 +2,17 @@ import { WRITABLE_DB_ID } from "e2e/support/cypress_data";
import type {
Collection,
CollectionId,
- ListTransformRunsResponse,
PythonTransformTableAliases,
TransformId,
TransformRun,
+ TransformRunStatus,
TransformSourceCheckpointStrategy,
TransformTagId,
} from "metabase-types/api";
import { createTransform } from "./api";
import { getTableId } from "./e2e-qa-databases-helpers";
+import { retryRequest } from "./e2e-request-helpers";
export function createTransformCollection({
name,
@@ -33,34 +34,41 @@ export function visitTransform(transformId: TransformId) {
}
export function runTransform(transformId: TransformId) {
- cy.request("POST", `/api/transform/${transformId}/run`);
+ return cy.request("POST", `/api/transform/${transformId}/run`);
}
-const WAIT_TIMEOUT = 10000;
-const WAIT_INTERVAL = 100;
+export function runTransformAndWaitForStatus(
+ transformId: TransformId,
+ status: TransformRunStatus,
+) {
+ return runTransform(transformId).then(({ body: run }) => {
+ return retryRequest(
+ () => cy.request("GET", `/api/transform/run/${run.run_id}`),
+ (response) => response.status === 200 && response.body.status === status,
+ );
+ });
+}
+
+export function runTransformAndWaitForSuccess(transformId: TransformId) {
+ return runTransformAndWaitForStatus(transformId, "succeeded");
+}
+
+export function runTransformAndWaitForFailure(transformId: TransformId) {
+ return runTransformAndWaitForStatus(transformId, "failed");
+}
export function waitForTransformRuns(
filter: (runs: TransformRun[]) => boolean,
- timeout = WAIT_TIMEOUT,
-): Cypress.Chainable {
- return cy
- .request<ListTransformRunsResponse>("GET", "/api/transform/run")
- .then((response) => {
- if (filter(response.body.data)) {
- return cy.wrap(response);
- } else if (timeout > 0) {
- cy.wait(WAIT_INTERVAL);
- return waitForTransformRuns(filter, timeout - WAIT_INTERVAL);
- } else {
- throw new Error("Run retry timeout");
- }
- });
+) {
+ return retryRequest(
+ () => cy.request("GET", "/api/transform/run"),
+ ({ body }) => filter(body.data),
+ );
}
export function waitForSucceededTransformRuns() {
- waitForTransformRuns(
- (runs) =>
- runs.length > 0 && runs.every((run) => run.status === "succeeded"),
+ return waitForTransformRuns((runs) =>
+ runs.some((run) => run.status === "succeeded"),
);
}
@@ -222,9 +230,7 @@ export function createAndRunMbqlTransform({
visitTransform: false,
}).then(({ body: transform }) => {
// Run the transform
- cy.request("POST", `/api/transform/${transform.id}/run`);
- // Wait for it to complete successfully
- waitForSucceededTransformRuns();
+ runTransformAndWaitForSuccess(transform.id);
return cy.wrap({
transformId: transform.id,
diff --git a/e2e/test/scenarios/admin/database-routing/database-routing-admin.cy.spec.ts b/e2e/test/scenarios/admin/database-routing/database-routing-admin.cy.spec.ts
index d159235a80c..9df7f6b373a 100644
--- a/e2e/test/scenarios/admin/database-routing/database-routing-admin.cy.spec.ts
+++ b/e2e/test/scenarios/admin/database-routing/database-routing-admin.cy.spec.ts
@@ -90,7 +90,7 @@ describe("admin > database > database routing", () => {
});
dbRoutingSection().within(() => {
cy.findByText("Destination DB 1");
- cy.findByTestId("destination-db-health-info").realHover();
+ cy.findByTestId("database-connection-health-info").realHover();
});
H.tooltip().should("contain.text", "Connected");
diff --git a/e2e/test/scenarios/admin/databases/database-writable-connection.cy.spec.ts b/e2e/test/scenarios/admin/databases/database-writable-connection.cy.spec.ts
new file mode 100644
index 00000000000..9bbba3604c9
--- /dev/null
+++ b/e2e/test/scenarios/admin/databases/database-writable-connection.cy.spec.ts
@@ -0,0 +1,209 @@
+import { WRITABLE_DB_CONFIG, WRITABLE_DB_ID } from "e2e/support/cypress_data";
+import type { DatabaseId, TransformTagId } from "metabase-types/api";
+
+const { H } = cy;
+
+type DatabaseCredentials = {
+ username: string;
+ password: string;
+};
+
+const DEFAULT_USER: DatabaseCredentials = {
+ username: WRITABLE_DB_CONFIG.mysql.connection.user,
+ password: WRITABLE_DB_CONFIG.mysql.connection.password,
+};
+
+const READ_ONLY_USER: DatabaseCredentials = {
+ username: "readonly_user",
+ password: "readonly_user",
+};
+
+const DATABASE_NAME = WRITABLE_DB_CONFIG.mysql.connection.database;
+const TRANSFORM_TABLE_NAME = "transform_table";
+
+describe("scenarios > admin > databases > writable connection", () => {
+ beforeEach(() => {
+ H.restore("mysql-writable");
+ cy.signInAsAdmin();
+ H.activateToken("bleeding-edge");
+ createUser(READ_ONLY_USER);
+ });
+
+ afterEach(() => {
+ dropUser(READ_ONLY_USER);
+ dropTable(TRANSFORM_TABLE_NAME);
+ });
+
+ it("should be able to create, edit, and remove a writable connection", () => {
+ visitDatabase(WRITABLE_DB_ID);
+ createWritableConnection(DEFAULT_USER);
+ updateWritableConnection(READ_ONLY_USER);
+ removeWritableConnection();
+ getWritableConnectionInfoSection()
+ .findByText("Add writable connection")
+ .should("exist");
+ });
+
+ it("should validate writable connection details on save", () => {
+ visitDatabase(WRITABLE_DB_ID);
+ getWritableConnectionInfoSection()
+ .findByText("Add writable connection")
+ .click();
+ fillInCredentials({
+ username: "invalid",
+ password: "invalid",
+ });
+
+ cy.button("Save").click();
+ cy.findByRole("alert").should(
+ "contain.text",
+ "Metabase tried, but couldn't connect",
+ );
+ });
+
+ it("should show up-to-date connection health status", () => {
+ visitDatabase(WRITABLE_DB_ID);
+ createWritableConnection(READ_ONLY_USER);
+ getMainConnectionInfoSection().within(() => {
+ getDatabaseConnectionHealthInfo().should("have.text", "Connected");
+ });
+ getWritableConnectionInfoSection().within(() => {
+ getDatabaseConnectionHealthInfo().should("have.text", "Connected");
+ });
+
+ dropUser(READ_ONLY_USER);
+ visitDatabase(WRITABLE_DB_ID);
+ getMainConnectionInfoSection().within(() => {
+ getDatabaseConnectionHealthInfo().should("have.text", "Connected");
+ });
+ getWritableConnectionInfoSection().within(() => {
+ getDatabaseConnectionHealthInfo().should(
+ "contain.text",
+ "Could not connect",
+ );
+ });
+ });
+
+ it("should be able to run transforms with a writable connection", () => {
+ visitDatabase(WRITABLE_DB_ID);
+
+ createTransform().then(({ body: transform }) => {
+ updateMainConnection(READ_ONLY_USER);
+ H.runTransformAndWaitForFailure(transform.id);
+
+ createWritableConnection(DEFAULT_USER);
+ H.runTransformAndWaitForSuccess(transform.id);
+ });
+ });
+
+ it("should be able to run transforms via a job with a writable connection", () => {
+ visitDatabase(WRITABLE_DB_ID);
+ updateMainConnection(READ_ONLY_USER);
+ createWritableConnection(DEFAULT_USER);
+
+ H.createTransformTag({ name: "New tag" }).then(({ body: tag }) => {
+ createTransform({ tagIds: [tag.id] });
+ H.createTransformJob({
+ schedule: "* * * * * ? *", // every second
+ tag_ids: [tag.id],
+ });
+ });
+ H.waitForSucceededTransformRuns();
+ });
+});
+
+function createTransform({ tagIds = [] }: { tagIds?: TransformTagId[] } = {}) {
+ return H.createTransform({
+ name: "Test transform",
+ source: {
+ type: "query",
+ query: {
+ database: WRITABLE_DB_ID,
+ type: "native",
+ native: {
+ query: "SELECT 1 as num",
+ },
+ },
+ },
+ target: {
+ type: "table",
+ database: WRITABLE_DB_ID,
+ name: TRANSFORM_TABLE_NAME,
+ schema: null,
+ },
+ tag_ids: tagIds,
+ });
+}
+
+function queryDB(query: string) {
+ H.queryWritableDB(query, "mysql");
+}
+
+function createUser(credentials: DatabaseCredentials) {
+ queryDB(
+ `CREATE USER IF NOT EXISTS ${credentials.username} IDENTIFIED BY '${credentials.password}';
+ GRANT SELECT ON ${DATABASE_NAME}.* TO '${credentials.username}'; `,
+ );
+}
+
+function dropUser(credentials: DatabaseCredentials) {
+ queryDB(`DROP USER IF EXISTS '${credentials.username}';`);
+}
+
+function dropTable(tableName: string) {
+ queryDB(`DROP TABLE IF EXISTS ${tableName};`);
+}
+
+function visitDatabase(databaseId: DatabaseId) {
+ cy.visit(`/admin/databases/${databaseId}`);
+}
+
+function getMainConnectionInfoSection() {
+ return cy.findByTestId("database-connection-info-section");
+}
+
+function getWritableConnectionInfoSection() {
+ return cy.findByTestId("writable-connection-info-section");
+}
+
+function getDatabaseConnectionHealthInfo() {
+ return cy.findByTestId("database-connection-health-info");
+}
+
+function fillInCredentials(credentials: DatabaseCredentials) {
+ cy.findByLabelText("Username").clear().type(credentials.username);
+ cy.findByLabelText("Password").clear().type(credentials.password);
+}
+
+function createWritableConnection(credentials: DatabaseCredentials) {
+ getWritableConnectionInfoSection()
+ .findByText("Add writable connection")
+ .click();
+ fillInCredentials(credentials);
+ cy.button("Save").click();
+ getWritableConnectionInfoSection().should("be.visible");
+}
+
+function updateMainConnection(credentials: DatabaseCredentials) {
+ getMainConnectionInfoSection().findByText("Edit connection details").click();
+ fillInCredentials(credentials);
+ cy.button("Save changes").click();
+ getMainConnectionInfoSection().should("be.visible");
+}
+
+function updateWritableConnection(credentials: DatabaseCredentials) {
+ getWritableConnectionInfoSection()
+ .findByText("Edit connection details")
+ .click();
+ fillInCredentials(credentials);
+ cy.button("Save changes").click();
+ getWritableConnectionInfoSection().should("be.visible");
+}
+
+function removeWritableConnection() {
+ getWritableConnectionInfoSection()
+ .button("Remove writable connection")
+ .click();
+ H.modal().button("Remove").click();
+ getWritableConnectionInfoSection().should("be.visible");
+}
diff --git a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
index eeb69565fb3..48f1b9ed41a 100644
--- a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
@@ -1618,8 +1618,7 @@ LIMIT
cy.get<TransformId>("@transformId").then((transformId) => {
cy.log("run the transform to create the output table");
- cy.request("POST", `/api/transform/${transformId}/run`);
- H.waitForSucceededTransformRuns();
+ H.runTransformAndWaitForSuccess(transformId);
H.resyncDatabase({
dbId: WRITABLE_DB_ID,
tableName: transformTableName,
diff --git a/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts b/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts
index e9b5f57bb30..c39a31321a6 100644
--- a/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts
+++ b/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts
@@ -364,8 +364,7 @@ function createSqlTransformWithDependentMbqlQuestions() {
},
}).then(({ body: transform }) => {
cy.wrap(transform.id).as("transformId");
- cy.request("POST", `/api/transform/${transform.id}/run`);
- H.waitForSucceededTransformRuns();
+ H.runTransformAndWaitForSuccess(transform.id);
H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: transformTableName });
H.getTableId({ databaseId: WRITABLE_DB_ID, name: transformTableName }).then(
@@ -420,9 +419,8 @@ function createMbqlTransformWithDependentMbqlTransforms() {
{ wrapId: true },
);
cy.get<number>("@transformId").then((transformId) =>
- H.runTransform(transformId),
+ H.runTransformAndWaitForSuccess(transformId),
);
- H.waitForSucceededTransformRuns();
H.getTableId({ databaseId: WRITABLE_DB_ID, name: "base_transform" }).then(
(tableId) => {
diff --git a/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts b/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts
index 981972195cb..617db43985f 100644
--- a/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts
+++ b/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts
@@ -18,7 +18,6 @@ import type {
NativeQuerySnippetId,
SegmentId,
TableId,
- TransformId,
} from "metabase-types/api";
import { createMockCard } from "metabase-types/api/mocks";
@@ -493,7 +492,7 @@ describe("scenarios > dependencies > dependency graph", () => {
it("should display dependencies for a transform and navigate to them", () => {
createTableBasedTransform({ tableName: TABLE_NAME }).then(
({ body: transform }) => {
- runTransformAndWaitForSuccess(transform.id);
+ H.runTransformAndWaitForSuccess(transform.id);
visitGraphForEntity(transform.id, "transform");
},
);
@@ -1072,11 +1071,6 @@ function createSnippetBasedTransform({
});
}
-function runTransformAndWaitForSuccess(transformId: TransformId) {
- cy.request("POST", `/api/transform/${transformId}/run`);
- H.waitForSucceededTransformRuns();
-}
-
function createEmptySnippet() {
return H.createSnippet({
name: EMPTY_SNIPPET_NAME,
diff --git a/enterprise/backend/src/metabase_enterprise/database_routing/api.clj b/enterprise/backend/src/metabase_enterprise/database_routing/api.clj
index d6018ad7426..0d5e08afead 100644
--- a/enterprise/backend/src/metabase_enterprise/database_routing/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/database_routing/api.clj
@@ -109,6 +109,7 @@
(api/check-404 db)
(api/check-400 (not (:router_database_id db)) "Cannot make a destination database a router database")
(api/check-400 (not (:uploads_enabled db)) "Cannot enable database routing for a database with uploads enabled")
+ (api/check-400 (not (:write_data_details db)) "Cannot enable database routing for a database with a write connection configured")
(setting/with-database db
(api/check-400 (not (setting/get :persist-models-enabled)) "Cannot enable database routing for a database with model persistence enabled")
(api/check-400 (not (setting/get :database-enable-actions)) "Cannot enable database routing for a database with actions enabled")))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index a32696eff53..cc134fe2db4 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -8,6 +8,7 @@
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
[metabase.app-db.core :as app-db]
[metabase.driver :as driver]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.util :as driver.u]
[metabase.transforms.core :as transforms]
[metabase.transforms.instrumentation :as transforms.instrumentation]
@@ -332,8 +333,8 @@
(log/error e "Failed to to create resulting table")
(throw (ex-info "Failed to create the resulting table"
{:transform-message (or (:transform-message (ex-data e))
- ;; TODO keeping messaging the same at this level
- ;; should be more specific in underlying calls
+ ;; TODO keeping messaging the same at this level
+ ;; should be more specific in underlying calls
(i18n/tru "Failed to create the resulting table"))}
e)))))))))
@@ -351,37 +352,41 @@
[transform {:keys [run-method start-promise user-id]}]
(assert (transforms.util/python-transform? transform) "Transform must be a python transform")
(try
- (let [message-log (empty-message-log)
+ (let [message-log (empty-message-log)
{:keys [target owner_user_id creator_id] transform-id :id} transform
- {driver :engine :as db} (t2/select-one :model/Database (transforms.i/target-db-id transform))
+ {driver :engine :as db} (t2/select-one :model/Database (transforms.i/target-db-id transform))
;; For manual runs, use the triggering user; for cron, use owner/creator
- run-user-id (if (and (= run-method :manual) user-id)
- user-id
- (or owner_user_id creator_id))
- {run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method run-user-id)]
+ run-user-id (if (and (= run-method :manual) user-id)
+ user-id
+ (or owner_user_id creator_id))
+ {run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method run-user-id)]
(some-> start-promise (deliver [:started run-id]))
(log! message-log (i18n/tru "Executing Python transform"))
- (log/info "Executing Python transform" transform-id "with target" (pr-str target))
- (let [start-ms (u/start-timer)
- transform-details {:db-id (:id db)
- :transform-id transform-id
- :transform-type (keyword (:type target))
- :conn-spec (driver/connection-spec driver db)
- :output-schema (:schema target)
- :output-table (transforms.util/qualified-table-name driver target)}
- run-fn (fn [cancel-chan]
- (run-python-transform! transform db run-id cancel-chan message-log)
- (log! message-log (i18n/tru "Python execution finished successfully in {0}" (u.format/format-milliseconds (u/since-ms start-ms))))
- (save-log-to-transform-run-message! run-id message-log))
- ex-message-fn #(exceptional-run-message message-log %)
- result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
- (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
- (transforms.util/handle-transform-complete!
- :run-id run-id
- :transform transform
- :db db)
- {:run_id run-id
- :result result}))
+ (driver.conn/with-write-connection
+ (log/info "Executing Python transform" transform-id "with target" (pr-str target)
+ (when (driver.conn/write-connection-requested?)
+ " using write connection"))
+ (let [start-ms (u/start-timer)
+ conn-spec (driver/connection-spec driver db)
+ transform-details {:db-id (:id db)
+ :transform-id transform-id
+ :transform-type (keyword (:type target))
+ :conn-spec conn-spec
+ :output-schema (:schema target)
+ :output-table (transforms.util/qualified-table-name driver target)}
+ run-fn (fn [cancel-chan]
+ (run-python-transform! transform db run-id cancel-chan message-log)
+ (log! message-log (i18n/tru "Python execution finished successfully in {0}" (u.format/format-milliseconds (u/since-ms start-ms))))
+ (save-log-to-transform-run-message! run-id message-log))
+ ex-message-fn #(exceptional-run-message message-log %)
+ result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
+ (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
+ (transforms.util/handle-transform-complete!
+ :run-id run-id
+ :transform transform
+ :db db)
+ {:run_id run-id
+ :result result})))
(catch Throwable t
(log/error t "Error executing Python transform")
(throw t))))
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/isolation.clj b/enterprise/backend/src/metabase_enterprise/workspaces/isolation.clj
index 4754cf04984..295415d2345 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/isolation.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/isolation.clj
@@ -13,6 +13,7 @@
This namespace provides the public API for workspace isolation."
(:require
[metabase.driver :as driver]
+ [metabase.driver.connection.workspaces :as driver.w]
[metabase.driver.util :as driver.u]))
;;;; Delegation to driver methods
@@ -43,8 +44,8 @@
(defn do-with-workspace-isolation
"Impl of* with-workspace-isolation*."
[workspace thunk]
- (driver/with-swapped-connection-details (:database_id workspace)
- (:database_details workspace)
+ (driver.w/with-swapped-connection-details (:database_id workspace)
+ (:database_details workspace)
(thunk)))
(defmacro with-workspace-isolation
diff --git a/enterprise/backend/test/metabase_enterprise/database_routing/api_test.clj b/enterprise/backend/test/metabase_enterprise/database_routing/api_test.clj
index 7b789676b68..90df56052db 100644
--- a/enterprise/backend/test/metabase_enterprise/database_routing/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/database_routing/api_test.clj
@@ -125,6 +125,10 @@
(is (= "Cannot enable database routing for a database with uploads enabled"
(mt/user-http-request :crowberto :put 400 (str "ee/database-routing/router-database/" db-id)
{:user_attribute "db_name"}))))
+ (mt/with-temp [:model/Database {db-id :id} {:write_data_details {:host "other-host"}}]
+ (is (= "Cannot enable database routing for a database with a write connection configured"
+ (mt/user-http-request :crowberto :put 400 (str "ee/database-routing/router-database/" db-id)
+ {:user_attribute "db_name"}))))
(mt/with-temp [:model/Database {router-db-id :id} {}
:model/Database {db-id :id} {:router_database_id router-db-id}]
(is (= "Cannot make a destination database a router database"
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/drivers_test.clj
index fafef33b444..ff44280a708 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/drivers_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/drivers_test.clj
@@ -1,7 +1,7 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.drivers-test
(:require
[clojure.test :refer :all]
- [metabase.driver :as driver]
+ [metabase.driver.connection.workspaces :as driver.w]
[metabase.test :as mt]))
(deftest drivers-support-workspaces-has-to-support-db-swapping-test
@@ -13,7 +13,7 @@
(catch Exception _e
true)))]
(testing "db is swapped if being executed inside a macros"
- (driver/with-swapped-connection-details (mt/id) {:user "unicorn"}
+ (driver.w/with-swapped-connection-details (mt/id) {:user "unicorn"}
(is (db-swapped?))))
(testing "sanity check that it's not swapped outside of the macros"
diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/WritableConnectionInfoSection.tsx b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/WritableConnectionInfoSection.tsx
new file mode 100644
index 00000000000..5523d927df9
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/WritableConnectionInfoSection.tsx
@@ -0,0 +1,96 @@
+import { Link } from "react-router";
+import { t } from "ttag";
+
+import { DatabaseConnectionHealthInfo } from "metabase/admin/databases/components/DatabaseConnectionHealthInfo";
+import {
+ DatabaseInfoSection,
+ DatabaseInfoSectionDivider,
+} from "metabase/admin/databases/components/DatabaseInfoSection";
+import {
+ hasDbRoutingEnabled,
+ isDbModifiable,
+} from "metabase/admin/databases/utils";
+import { useUpdateDatabaseMutation } from "metabase/api";
+import { useConfirmation } from "metabase/common/hooks/use-confirmation";
+import * as Urls from "metabase/lib/urls";
+import type { WritableConnectionInfoSectionProps } from "metabase/plugins/oss/permissions";
+import { Alert, Button, Group, Icon } from "metabase/ui";
+
+export function WritableConnectionInfoSection({
+ database,
+}: WritableConnectionInfoSectionProps) {
+ const hasWritableConnection = database.write_data_details !== null;
+ const [updateDatabase] = useUpdateDatabaseMutation();
+ const { modalContent, show: showConfirmation } = useConfirmation();
+ const isDbRoutingEnabled = hasDbRoutingEnabled(database);
+
+ if (!isDbModifiable(database)) {
+ return null;
+ }
+
+ const handleRemove = () => {
+ showConfirmation({
+ title: t`Remove writable connection?`,
+ message: t`This will remove the writable connection for this database. Any actions that depend on it will stop working.`,
+ confirmButtonText: t`Remove`,
+ onConfirm: async () => {
+ await updateDatabase({
+ id: database.id,
+ write_data_details: null,
+ }).unwrap();
+ },
+ });
+ };
+
+ return (
+ <DatabaseInfoSection
+ name={t`Writable connection`}
+ description={t`You can add a separate writable connection to use with features like transforms and editable tables.`}
+ data-testid="writable-connection-info-section"
+ >
+ <Group justify="space-between" gap="lg">
+ {hasWritableConnection && (
+ <DatabaseConnectionHealthInfo
+ databaseId={database.id}
+ connectionType="write-data"
+ />
+ )}
+ <Button
+ component={isDbRoutingEnabled ? undefined : Link}
+ to={Urls.editDatabaseWritableConnection(database.id)}
+ disabled={isDbRoutingEnabled}
+ >
+ {hasWritableConnection
+ ? t`Edit connection details`
+ : t`Add writable connection`}
+ </Button>
+ </Group>
+
+ {isDbRoutingEnabled && (
+ <>
+ <DatabaseInfoSectionDivider />
+ <Alert
+ variant="light"
+ color="info"
+ icon={<Icon name="info" />}
+ mb="md"
+ >
+ {t`Writable connection can't be enabled when Database Routing is enabled.`}
+ </Alert>
+ </>
+ )}
+
+ {hasWritableConnection && (
+ <>
+ <DatabaseInfoSectionDivider condensed />
+ <Group>
+ <Button variant="filled" color="error" onClick={handleRemove}>
+ {t`Remove writable connection`}
+ </Button>
+ </Group>
+ </>
+ )}
+ {modalContent}
+ </DatabaseInfoSection>
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/WritableConnectionInfoSection.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/WritableConnectionInfoSection.unit.spec.tsx
new file mode 100644
index 00000000000..1c256dc02e5
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/WritableConnectionInfoSection.unit.spec.tsx
@@ -0,0 +1,59 @@
+import { setupDatabaseEndpoints } from "__support__/server-mocks/database";
+import { renderWithProviders, screen } from "__support__/ui";
+import type { Database } from "metabase-types/api";
+import { createMockDatabase } from "metabase-types/api/mocks";
+
+import { WritableConnectionInfoSection } from "./WritableConnectionInfoSection";
+
+interface SetupOpts {
+ database?: Database;
+}
+
+function setup({ database = createMockDatabase() }: SetupOpts = {}) {
+ setupDatabaseEndpoints(database);
+ renderWithProviders(<WritableConnectionInfoSection database={database} />);
+}
+
+describe("WritableConnectionInfoSection", () => {
+ it("should not render for a non-modifiable database (attached DWH)", () => {
+ setup({ database: createMockDatabase({ is_attached_dwh: true }) });
+
+ expect(
+ screen.queryByTestId("writable-connection-info-section"),
+ ).not.toBeInTheDocument();
+ });
+
+ it("should render for a modifiable database", () => {
+ setup({ database: createMockDatabase({ is_attached_dwh: false }) });
+
+ expect(
+ screen.getByTestId("writable-connection-info-section"),
+ ).toBeInTheDocument();
+ });
+
+ it("should show a warning when database routing is enabled", () => {
+ setup({
+ database: createMockDatabase({
+ router_user_attribute: "some_attribute",
+ }),
+ });
+
+ expect(
+ screen.getByText(
+ "Writable connection can't be enabled when Database Routing is enabled.",
+ ),
+ ).toBeInTheDocument();
+ });
+
+ it("should not show a warning when database routing is not enabled", () => {
+ setup({
+ database: createMockDatabase({ router_user_attribute: null }),
+ });
+
+ expect(
+ screen.queryByText(
+ "Writable connection can't be enabled when Database Routing is enabled.",
+ ),
+ ).not.toBeInTheDocument();
+ });
+});
diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/index.ts b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/index.ts
new file mode 100644
index 00000000000..22c5fb40b9f
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/index.ts
@@ -0,0 +1 @@
+export * from "./WritableConnectionInfoSection";
diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/index.js b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/index.js
index cab3d9bbd54..bc7218fd797 100644
--- a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/index.js
+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/index.js
@@ -21,11 +21,13 @@ import {
import { hasPremiumFeature } from "metabase-enterprise/settings";
import { ImpersonationModal } from "./components/ImpersonationModal";
+import { WritableConnectionInfoSection } from "./components/WritableConnectionInfoSection";
import {
shouldRestrictNativeQueryPermissions,
upgradeViewPermissionsIfNeeded,
} from "./graph";
import { advancedPermissionsSlice, getImpersonatedPostAction } from "./reducer";
+import { getWritableConnectionInfoRoutes } from "./routes";
import { getImpersonations } from "./selectors";
const IMPERSONATED_PERMISSION_OPTION = {
@@ -157,6 +159,11 @@ export function initializePlugin() {
upgradeViewPermissionsIfNeeded;
PLUGIN_DATA_PERMISSIONS.shouldRestrictNativeQueryPermissions =
shouldRestrictNativeQueryPermissions;
+
+ PLUGIN_ADVANCED_PERMISSIONS.getWritableConnectionInfoRoutes =
+ getWritableConnectionInfoRoutes;
+ PLUGIN_ADVANCED_PERMISSIONS.WritableConnectionInfoSection =
+ WritableConnectionInfoSection;
}
}
diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/pages/WritableConnectionInfoPage/WritableConnectionInfoPage.tsx b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/pages/WritableConnectionInfoPage/WritableConnectionInfoPage.tsx
new file mode 100644
index 00000000000..d20fc632ef1
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/pages/WritableConnectionInfoPage/WritableConnectionInfoPage.tsx
@@ -0,0 +1,138 @@
+import { useMemo, useState } from "react";
+import type { Route } from "react-router";
+import { push } from "react-router-redux";
+import { t } from "ttag";
+
+import { SettingsSection } from "metabase/admin/components/SettingsSection";
+import {
+ skipToken,
+ useGetDatabaseQuery,
+ useUpdateDatabaseMutation,
+} from "metabase/api";
+import { LeaveRouteConfirmModal } from "metabase/common/components/LeaveConfirmModal";
+import { DelayedLoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper/DelayedLoadingAndErrorWrapper";
+import { DatabaseForm } from "metabase/databases/components/DatabaseForm";
+import type { DatabaseFormConfig } from "metabase/databases/types";
+import { useDispatch } from "metabase/lib/redux";
+import * as Urls from "metabase/lib/urls";
+import { Box, Flex, ScrollArea, Title } from "metabase/ui";
+import type { Database, DatabaseData } from "metabase-types/api";
+
+const FORM_CONFIG: DatabaseFormConfig = {
+ engine: {
+ fieldState: "disabled",
+ },
+ name: {
+ fieldState: "hidden",
+ },
+ isAdvanced: true,
+};
+
+type WritableConnectionInfoPageParams = {
+ databaseId: string;
+};
+
+type WritableConnectionInfoPageProps = {
+ params: WritableConnectionInfoPageParams;
+ route: Route;
+};
+
+export function WritableConnectionInfoPage({
+ params,
+ route,
+}: WritableConnectionInfoPageProps) {
+ const databaseId = Urls.extractEntityId(params.databaseId);
+ const {
+ data: database,
+ isLoading,
+ error,
+ } = useGetDatabaseQuery(databaseId != null ? { id: databaseId } : skipToken);
+
+ if (isLoading || error != null || database == null) {
+ return <DelayedLoadingAndErrorWrapper loading={isLoading} error={error} />;
+ }
+
+ return <WritableConnectionInfoPageBody database={database} route={route} />;
+}
+
+type WritableConnectionInfoPageBodyProps = {
+ database: Database;
+ route: Route;
+};
+
+function WritableConnectionInfoPageBody({
+ database,
+ route,
+}: WritableConnectionInfoPageBodyProps) {
+ const title = getTitle(database);
+ const initialValues = useMemo(() => getInitialValues(database), [database]);
+ const [isDirty, setIsDirty] = useState(false);
+ const [updateDatabase, { isLoading: isSaving }] = useUpdateDatabaseMutation();
+ const dispatch = useDispatch();
+
+ const handleSubmit = async (newValues: DatabaseData) => {
+ await updateDatabase({
+ id: database.id,
+ write_data_details: getSubmitDetails(newValues),
+ }).unwrap();
+ dispatch(push(Urls.viewDatabase(database.id)));
+ };
+
+ const handleCancel = () => {
+ dispatch(push(Urls.viewDatabase(database.id)));
+ };
+
+ return (
+ <Flex
+ direction="row"
+ h="100%"
+ bg="background-secondary"
+ data-testid="writable-connection-info-page"
+ >
+ <Box h="100%" w="100%" component={ScrollArea}>
+ <Box w="100%" maw="54rem" mx="auto" p={{ base: "md", sm: "xl" }}>
+ <Flex mb="lg" align="center">
+ <Title order={1} fz="h2">
+ {title}
+ </Title>
+ </Flex>
+ <SettingsSection>
+ <DatabaseForm
+ initialValues={initialValues}
+ config={FORM_CONFIG}
+ location="full-page"
+ onSubmit={handleSubmit}
+ onCancel={handleCancel}
+ onDirtyStateChange={setIsDirty}
+ />
+ </SettingsSection>
+ </Box>
+ </Box>
+ <LeaveRouteConfirmModal isEnabled={isDirty && !isSaving} route={route} />
+ </Flex>
+ );
+}
+
+function getTitle(database: Database): string {
+ return database.write_data_details == null
+ ? t`Add writable connection`
+ : t`Edit writable connection details`;
+}
+
+function getInitialValues(database: Database): DatabaseData {
+ const { password, ...details } = database.details ?? {};
+
+ return {
+ ...database,
+ id: database.write_data_details ? database.id : undefined,
+ details: {
+ ...details,
+ ...database.write_data_details,
+ "write-data-connection": true,
+ },
+ };
+}
+
+function getSubmitDetails(values: DatabaseData) {
+ return values.details ?? {};
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/pages/WritableConnectionInfoPage/index.ts b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/pages/WritableConnectionInfoPage/index.ts
new file mode 100644
index 00000000000..051c3cd9b34
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/pages/WritableConnectionInfoPage/index.ts
@@ -0,0 +1 @@
+export * from "./WritableConnectionInfoPage";
diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/routes.tsx b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/routes.tsx
new file mode 100644
index 00000000000..c4e789db0bd
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/routes.tsx
@@ -0,0 +1,12 @@
+import { Route } from "react-router";
+
+import { WritableConnectionInfoPage } from "./pages/WritableConnectionInfoPage";
+
+export function getWritableConnectionInfoRoutes() {
+ return (
+ <Route
+ path=":databaseId/write-data"
+ component={WritableConnectionInfoPage}
+ />
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/auth/components/AuthSettingsPage.tsx b/enterprise/frontend/src/metabase-enterprise/auth/components/AuthSettingsPage.tsx
index 7e58758f446..735e65c4794 100644
--- a/enterprise/frontend/src/metabase-enterprise/auth/components/AuthSettingsPage.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/auth/components/AuthSettingsPage.tsx
@@ -9,7 +9,10 @@ import { LdapAuthCard } from "metabase/admin/settings/auth/containers/LdapAuthCa
import { ManageApiKeys } from "metabase/admin/settings/components/ApiKeys/ManageApiKeys";
import { AdminSettingInput } from "metabase/admin/settings/components/widgets/AdminSettingInput";
import { useHasTokenFeature } from "metabase/common/hooks";
-import { PLUGIN_AUTH_PROVIDERS } from "metabase/plugins";
+import {
+ type AuthSettingsPageProps,
+ PLUGIN_AUTH_PROVIDERS,
+} from "metabase/plugins";
import { JwtAuthCard } from "../containers/JwtAuthCard";
import { SamlAuthCard } from "../containers/SamlAuthCard";
@@ -17,9 +20,9 @@ import { useHasSsoEnabled } from "../utils";
import { SessionTimeoutSetting } from "./SessionTimeoutSetting";
-type Tabs = "authentication" | "user-provisioning" | "api-keys";
-
-export function AuthSettingsPage({ tab = "authentication" }: { tab?: Tabs }) {
+export function AuthSettingsPage({
+ tab = "authentication",
+}: AuthSettingsPageProps) {
if (tab === "api-keys") {
return <ManageApiKeys />;
}
diff --git a/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.tsx b/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.tsx
index 417a231d2fd..b675fa65e24 100644
--- a/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.tsx
@@ -143,12 +143,28 @@ export const DatabaseRoutingSection = ({
/>
</Box>
</Tooltip>
- <UnstyledButton onClick={() => setIsExpanded(!isExpanded)} px="xs">
- <Icon name={isExpanded ? "chevronup" : "chevrondown"} />
- </UnstyledButton>
+ {disabledFeatMsg == null && (
+ <UnstyledButton onClick={() => setIsExpanded(!isExpanded)} px="xs">
+ <Icon name={isExpanded ? "chevronup" : "chevrondown"} />
+ </UnstyledButton>
+ )}
</Flex>
</Flex>
+ {disabledFeatMsg != null && (
+ <>
+ <DatabaseInfoSectionDivider />
+ <Alert
+ variant="light"
+ color="info"
+ icon={<Icon name="info" />}
+ mb="md"
+ >
+ {disabledFeatMsg}
+ </Alert>
+ </>
+ )}
+
{isExpanded && (
<>
<DatabaseInfoSectionDivider />
diff --git a/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.unit.spec.tsx
index da76d608f03..66f3110b7ff 100644
--- a/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.unit.spec.tsx
@@ -15,13 +15,15 @@ import { createMockSettingsState } from "metabase-types/store/mocks";
import { DatabaseRoutingSection } from "./DatabaseRoutingSection";
-const setup = (database: Partial<Database>) => {
- const db = createMockDatabase(database);
+interface SetupOpts {
+ database?: Database;
+}
+const setup = ({ database = createMockDatabase() }: SetupOpts = {}) => {
setupUserAttributesEndpoint(["cool_guy", "boss_gal"]);
- setupDatabasesEndpoints([db]);
+ setupDatabasesEndpoints([database]);
- renderWithProviders(<DatabaseRoutingSection database={db} />, {
+ renderWithProviders(<DatabaseRoutingSection database={database} />, {
storeInitialState: {
currentUser: createMockUser({ is_superuser: true }),
settings: createMockSettingsState(
@@ -44,7 +46,12 @@ const setup = (database: Partial<Database>) => {
describe("DatabaseRoutingSection", () => {
it("should render DatabaseRoutingSection", () => {
- setup({ engine: "postgres", features: ["database-routing"] });
+ setup({
+ database: createMockDatabase({
+ engine: "postgres",
+ features: ["database-routing"],
+ }),
+ });
expect(screen.getByText("Database routing")).toBeInTheDocument();
expect(
@@ -58,8 +65,10 @@ describe("DatabaseRoutingSection", () => {
it("should render DatabaseRoutingSection with custom db_routing_info", () => {
setup({
- engine: "bigquery-cloud-sdk",
- features: ["database-routing"],
+ database: createMockDatabase({
+ engine: "bigquery-cloud-sdk",
+ features: ["database-routing"],
+ }),
});
expect(screen.getByText("Database routing")).toBeInTheDocument();
@@ -70,9 +79,11 @@ describe("DatabaseRoutingSection", () => {
it("should hide section if database is attached DWH", () => {
setup({
- engine: "postgres",
- is_attached_dwh: true,
- features: ["database-routing"],
+ database: createMockDatabase({
+ engine: "postgres",
+ is_attached_dwh: true,
+ features: ["database-routing"],
+ }),
});
expect(screen.queryByText("Database routing")).not.toBeInTheDocument();
@@ -80,16 +91,36 @@ describe("DatabaseRoutingSection", () => {
it("should hide section if database is sample", () => {
setup({
- engine: "postgres",
- is_sample: true,
- features: ["database-routing"],
+ database: createMockDatabase({
+ engine: "postgres",
+ is_sample: true,
+ features: ["database-routing"],
+ }),
});
expect(screen.queryByText("Database routing")).not.toBeInTheDocument();
});
it("should hide section if database routing is not supported by the db engine", async () => {
- setup({ engine: "clickhouse", features: [] });
+ setup({
+ database: createMockDatabase({ engine: "clickhouse", features: [] }),
+ });
expect(screen.queryByText("Database routing")).not.toBeInTheDocument();
});
+
+ it("should show a warning when writable connection is enabled", () => {
+ setup({
+ database: createMockDatabase({
+ engine: "postgres",
+ features: ["database-routing"],
+ write_data_details: { host: "localhost" },
+ }),
+ });
+
+ expect(
+ screen.getByText(
+ "Database routing can't be enabled when a Writable Connection is enabled.",
+ ),
+ ).toBeInTheDocument();
+ });
});
diff --git a/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/utils.tsx b/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/utils.tsx
index 00f1efdcefb..79d200c5f32 100644
--- a/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/utils.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/utils.tsx
@@ -2,7 +2,11 @@ import { Link } from "react-router";
import { match } from "ts-pattern";
import { t } from "ttag";
-import { hasActionsEnabled, hasFeature } from "metabase/admin/databases/utils";
+import {
+ hasActionsEnabled,
+ hasFeature,
+ hasWritableConnectionDetails,
+} from "metabase/admin/databases/utils";
import { Text } from "metabase/ui";
import type { Database } from "metabase-types/api";
@@ -12,6 +16,7 @@ export const getDisabledFeatureMessage = (database: Database) => {
isPersisted: hasFeature(database, "persist-models-enabled"),
isUploadDb: database.uploads_enabled,
supportsRouting: !!database.features?.includes("database-routing"),
+ hasWritableConnection: hasWritableConnectionDetails(database),
})
.with(
{ supportsRouting: false },
@@ -36,6 +41,11 @@ export const getDisabledFeatureMessage = (database: Database) => {
() =>
t`Database routing can't be enabled if uploads are enabled for this database.`,
)
+ .with(
+ { hasWritableConnection: true },
+ () =>
+ t`Database routing can't be enabled when a Writable Connection is enabled.`,
+ )
.otherwise(() => undefined);
};
diff --git a/enterprise/frontend/src/metabase-enterprise/database_routing/DestinationDatabasesList/DestinationDatabasesList.tsx b/enterprise/frontend/src/metabase-enterprise/database_routing/DestinationDatabasesList/DestinationDatabasesList.tsx
index 67f371a92e7..dc906a16748 100644
--- a/enterprise/frontend/src/metabase-enterprise/database_routing/DestinationDatabasesList/DestinationDatabasesList.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/database_routing/DestinationDatabasesList/DestinationDatabasesList.tsx
@@ -92,7 +92,6 @@ const DestinationDatabasesListItem = ({
<DatabaseConnectionHealthInfo
databaseId={database.id}
displayText="tooltip"
- data-testid="destination-db-health-info"
/>
<Text>{database.name}</Text>
</Flex>
diff --git a/enterprise/frontend/src/metabase-enterprise/semantic_search/SearchSettingsWidget.tsx b/enterprise/frontend/src/metabase-enterprise/semantic_search/SearchSettingsWidget.tsx
index 4bbc626519c..66cdeca4a3e 100644
--- a/enterprise/frontend/src/metabase-enterprise/semantic_search/SearchSettingsWidget.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/semantic_search/SearchSettingsWidget.tsx
@@ -8,6 +8,7 @@ import { UpsellSemanticSearchPill } from "metabase/admin/upsells/UpsellSemanticS
import { getErrorMessage, useAdminSetting } from "metabase/api/utils";
import { getPlan, isProPlan } from "metabase/common/utils/plan";
import { useSelector } from "metabase/lib/redux";
+import type { SearchSettingsWidgetProps } from "metabase/plugins";
import { getSetting } from "metabase/selectors/settings";
import { Box, Progress, Stack, Text, Tooltip } from "metabase/ui";
import { useGetSemanticSearchStatusQuery } from "metabase-enterprise/api/search";
@@ -26,9 +27,7 @@ function useLatch(bool: boolean) {
export function SearchSettingsWidget({
statusPollingInterval = 5000,
-}: {
- statusPollingInterval?: number;
-}) {
+}: SearchSettingsWidgetProps) {
const plan = useSelector((state) =>
getPlan(getSetting(state, "token-features")),
);
diff --git a/frontend/src/metabase-types/api/database.ts b/frontend/src/metabase-types/api/database.ts
index 9ba8ff96064..7fba227d4f1 100644
--- a/frontend/src/metabase-types/api/database.ts
+++ b/frontend/src/metabase-types/api/database.ts
@@ -96,6 +96,7 @@ export interface DatabaseData {
// missing in responses from the backend, cf. implementation of
// [[metabase.models.interface/to-json]] for `:model/Database`:
details?: Record<string, unknown>;
+ write_data_details?: Record<string, unknown> | null;
schedules: DatabaseSchedules;
auto_run_queries: boolean | null;
refingerprint: boolean | null;
@@ -138,6 +139,13 @@ export type DatabaseLocalSettingAvailability =
| { enabled: true }
| { enabled: false; reasons: DatabaseLocalSettingDisableReason[] };
+export type DatabaseConnectionType = "default" | "write-data";
+
+export type GetDatabaseHealthRequest = {
+ id: DatabaseId;
+ "connection-type"?: DatabaseConnectionType;
+};
+
export type GetDatabaseHealthResponse =
| { status: "ok" }
| { status: "error"; message: string; errors: unknown };
@@ -212,6 +220,7 @@ export interface UpdateDatabaseRequest {
engine?: string;
refingerprint?: boolean | null;
details?: Record<string, unknown>;
+ write_data_details?: Record<string, unknown> | null;
schedules?: DatabaseSchedules;
description?: string;
caveats?: string;
diff --git a/frontend/src/metabase/admin/databases/components/DatabaseConnectionHealthInfo.tsx b/frontend/src/metabase/admin/databases/components/DatabaseConnectionHealthInfo.tsx
index 2f54f80d7b1..6f1df04f5e2 100644
--- a/frontend/src/metabase/admin/databases/components/DatabaseConnectionHealthInfo.tsx
+++ b/frontend/src/metabase/admin/databases/components/DatabaseConnectionHealthInfo.tsx
@@ -4,19 +4,24 @@ import { t } from "ttag";
import { useGetDatabaseHealthQuery } from "metabase/api";
import { Badge, Flex, type FlexProps, Text, Tooltip } from "metabase/ui";
-import type { DatabaseId } from "metabase-types/api";
+import type { DatabaseConnectionType, DatabaseId } from "metabase-types/api";
interface DatabaseConnectionHealthInfoProps extends FlexProps {
databaseId: DatabaseId;
+ connectionType?: DatabaseConnectionType;
displayText?: "inline" | "tooltip";
}
export const DatabaseConnectionHealthInfo = ({
databaseId,
displayText = "inline",
+ connectionType = "default",
...props
}: DatabaseConnectionHealthInfoProps) => {
- const healthQuery = useGetDatabaseHealthQuery(databaseId);
+ const healthQuery = useGetDatabaseHealthQuery({
+ id: databaseId,
+ "connection-type": connectionType,
+ });
const health = useMemo(() => {
return match(healthQuery)
.with(
@@ -49,7 +54,12 @@ export const DatabaseConnectionHealthInfo = ({
}, [healthQuery]);
return (
- <Flex align="center" gap="sm" {...props}>
+ <Flex
+ align="center"
+ gap="sm"
+ {...props}
+ data-testid="database-connection-health-info"
+ >
<Tooltip label={health.message}>
<Badge size="12" circle bg={health.color} style={{ flexShrink: 0 }} />
</Tooltip>
diff --git a/frontend/src/metabase/admin/databases/containers/DatabaseEditApp.tsx b/frontend/src/metabase/admin/databases/containers/DatabaseEditApp.tsx
index db0d79481e7..ec7f2c060a9 100644
--- a/frontend/src/metabase/admin/databases/containers/DatabaseEditApp.tsx
+++ b/frontend/src/metabase/admin/databases/containers/DatabaseEditApp.tsx
@@ -16,6 +16,7 @@ import CS from "metabase/css/core/index.css";
import { usePageTitle } from "metabase/hooks/use-page-title";
import { connect, useSelector } from "metabase/lib/redux";
import {
+ PLUGIN_ADVANCED_PERMISSIONS,
PLUGIN_DATABASE_REPLICATION,
PLUGIN_DB_ROUTING,
PLUGIN_TABLE_EDITING,
@@ -103,6 +104,10 @@ function DatabaseEditAppInner({
>
<DatabaseConnectionInfoSection database={database} />
+ <PLUGIN_ADVANCED_PERMISSIONS.WritableConnectionInfoSection
+ database={database}
+ />
+
<DatabaseModelFeaturesSection
database={database}
isModelPersistenceEnabled={isModelPersistenceEnabled}
diff --git a/frontend/src/metabase/admin/databases/containers/DatabasePage.tsx b/frontend/src/metabase/admin/databases/containers/DatabasePage.tsx
index 47c3d506034..b0ef1e47a55 100644
--- a/frontend/src/metabase/admin/databases/containers/DatabasePage.tsx
+++ b/frontend/src/metabase/admin/databases/containers/DatabasePage.tsx
@@ -62,21 +62,9 @@ export function DatabasePage({ params, route }: DatabasePageProps) {
};
return (
- <Flex
- direction="row"
- h="100%"
- style={{ backgroundColor: "var(--mb-color-background-secondary)" }}
- >
+ <Flex direction="row" h="100%" bg="background-secondary">
<Box h="100%" w="100%" component={ScrollArea}>
- <Box
- w="100%"
- maw="54rem"
- mx="auto"
- p={{
- base: `md`,
- sm: `xl`,
- }}
- >
+ <Box w="100%" maw="54rem" mx="auto" p={{ base: "md", sm: "xl" }}>
<Flex
mb="lg"
align="center"
diff --git a/frontend/src/metabase/admin/databases/utils.ts b/frontend/src/metabase/admin/databases/utils.ts
index 9b91207d5ed..0be3cecfcf7 100644
--- a/frontend/src/metabase/admin/databases/utils.ts
+++ b/frontend/src/metabase/admin/databases/utils.ts
@@ -23,6 +23,12 @@ export const hasActionsEnabled = (database: Pick<Database, "settings">) => {
return Boolean(database.settings?.["database-enable-actions"]);
};
+export const hasWritableConnectionDetails = (
+ database: Pick<Database, "write_data_details">,
+) => {
+ return database.write_data_details != null;
+};
+
export const hasDbRoutingEnabled = (
database: Pick<Database, "router_user_attribute">,
) => {
diff --git a/frontend/src/metabase/admin/routes.jsx b/frontend/src/metabase/admin/routes.jsx
index 378272d6e70..9dda2dc7daa 100644
--- a/frontend/src/metabase/admin/routes.jsx
+++ b/frontend/src/metabase/admin/routes.jsx
@@ -42,6 +42,7 @@ import { DataModelV1 } from "metabase/metadata/pages/DataModelV1";
import {
PLUGIN_ADMIN_TOOLS,
PLUGIN_ADMIN_USER_MENU_ROUTES,
+ PLUGIN_ADVANCED_PERMISSIONS,
PLUGIN_CACHING,
PLUGIN_DB_ROUTING,
PLUGIN_DEPENDENCIES,
@@ -83,6 +84,7 @@ export const getRoutes = (store, CanAccessSettings, IsAdmin) => {
<Route path="create" component={DatabasePage} />
</Route>
<Route path=":databaseId/edit" component={DatabasePage} />
+ {PLUGIN_ADVANCED_PERMISSIONS.getWritableConnectionInfoRoutes(IsAdmin)}
<Route path=":databaseId" component={DatabaseEditApp}>
{PLUGIN_DB_ROUTING.getDestinationDatabaseRoutes(IsAdmin)}
</Route>
diff --git a/frontend/src/metabase/admin/settings/components/SettingsPages/AuthenticationSettingsPage.tsx b/frontend/src/metabase/admin/settings/components/SettingsPages/AuthenticationSettingsPage.tsx
index 2367b8a46cd..ccd296036f0 100644
--- a/frontend/src/metabase/admin/settings/components/SettingsPages/AuthenticationSettingsPage.tsx
+++ b/frontend/src/metabase/admin/settings/components/SettingsPages/AuthenticationSettingsPage.tsx
@@ -4,7 +4,10 @@ import { SettingsPageWrapper } from "metabase/admin/components/SettingsSection";
import { UpsellSSO } from "metabase/admin/upsells";
import { useGetSettingsQuery } from "metabase/api";
import { hasAnySsoFeature } from "metabase/common/utils/plan";
-import { PLUGIN_AUTH_PROVIDERS } from "metabase/plugins";
+import {
+ type AuthSettingsPageTab,
+ PLUGIN_AUTH_PROVIDERS,
+} from "metabase/plugins";
import { Box, Flex, Stack } from "metabase/ui";
import { ApiKeysAuthCard } from "../../auth/components/ApiKeysAuthCard";
@@ -12,7 +15,11 @@ import { GoogleAuthCard } from "../../auth/containers/GoogleAuthCard/GoogleAuthC
import { LdapAuthCard } from "../../auth/containers/LdapAuthCard";
import { ManageApiKeys } from "../ApiKeys/ManageApiKeys";
-export function AuthenticationSettingsPage({ tab }: { tab: string }) {
+export function AuthenticationSettingsPage({
+ tab,
+}: {
+ tab: AuthSettingsPageTab;
+}) {
const hasSSO = useHasSso();
if (hasSSO) {
diff --git a/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.enterprise.unit.spec.tsx b/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.enterprise.unit.spec.tsx
index a270b3f7cc5..8368ff591b7 100644
--- a/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.enterprise.unit.spec.tsx
+++ b/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.enterprise.unit.spec.tsx
@@ -1,12 +1,13 @@
import { setupEnterpriseOnlyPlugin } from "__support__/enterprise";
import { screen } from "__support__/ui";
+import type { AuthSettingsPageTab } from "metabase/plugins";
import type { EnterpriseSettings } from "metabase-types/api";
import { setup as OSSSetup } from "./AuthenticationSettingsPage.setup";
const setup = async (
extraSettings?: Partial<EnterpriseSettings>,
- tab = "authentication",
+ tab: AuthSettingsPageTab = "authentication",
) => {
setupEnterpriseOnlyPlugin("auth");
return OSSSetup(extraSettings, true, tab);
diff --git a/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.setup.tsx b/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.setup.tsx
index 256aaa04300..d2408b70e64 100644
--- a/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.setup.tsx
+++ b/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.setup.tsx
@@ -7,6 +7,7 @@ import {
} from "__support__/server-mocks";
import { renderWithProviders } from "__support__/ui";
import { UndoListing } from "metabase/common/components/UndoListing";
+import type { AuthSettingsPageTab } from "metabase/plugins";
import type { ApiKey, EnterpriseSettings } from "metabase-types/api";
import {
createMockSettings,
@@ -54,7 +55,7 @@ export const testApiKeys: ApiKey[] = [
export const setup = async (
extraSettings?: Partial<EnterpriseSettings>,
isEnterprise = false,
- tab = "authentication",
+ tab: AuthSettingsPageTab = "authentication",
) => {
const settings = createMockSettings({
"google-auth-enabled": false,
diff --git a/frontend/src/metabase/api/database.ts b/frontend/src/metabase/api/database.ts
index 7fae1024e9f..6b2fcb6c856 100644
--- a/frontend/src/metabase/api/database.ts
+++ b/frontend/src/metabase/api/database.ts
@@ -11,6 +11,7 @@ import type {
Database,
DatabaseId,
Field,
+ GetDatabaseHealthRequest,
GetDatabaseHealthResponse,
GetDatabaseMetadataRequest,
GetDatabaseRequest,
@@ -69,13 +70,17 @@ export const databaseApi = Api.injectEndpoints({
dispatch(updateMetadata(data, DatabaseSchema)),
),
}),
- getDatabaseHealth: builder.query<GetDatabaseHealthResponse, DatabaseId>({
- query: (id) => ({
+ getDatabaseHealth: builder.query<
+ GetDatabaseHealthResponse,
+ GetDatabaseHealthRequest
+ >({
+ query: ({ id, ...params }) => ({
method: "GET",
url: `/api/database/${id}/healthcheck`,
+ params,
}),
// invalidate health check in the case db connection info changes
- providesTags: (_, __, id) => [idTag("database", id)],
+ providesTags: (_, __, { id }) => [idTag("database", id)],
}),
getDatabaseMetadata: builder.query<Database, GetDatabaseMetadataRequest>({
query: ({ id, ...params }) => ({
diff --git a/frontend/src/metabase/databases/components/DatabaseForm/DatabaseForm.tsx b/frontend/src/metabase/databases/components/DatabaseForm/DatabaseForm.tsx
index 84062a1ff44..8b3c1bd6ae8 100644
--- a/frontend/src/metabase/databases/components/DatabaseForm/DatabaseForm.tsx
+++ b/frontend/src/metabase/databases/components/DatabaseForm/DatabaseForm.tsx
@@ -62,7 +62,6 @@ export const DatabaseForm = ({
config = {},
}: DatabaseFormProps): JSX.Element => {
const isAdvanced = config.isAdvanced || false;
- const engineFieldState = config.engine?.fieldState;
const engines = useSelector(getEngines);
const initialEngineKey = useMemo(() => {
@@ -119,7 +118,6 @@ export const DatabaseForm = ({
// casting won't be needed after migrating all usages of engineKey
engineKey={engineKey as EngineKey}
engines={engines}
- engineFieldState={engineFieldState}
autofocusFieldName={autofocusFieldName}
isAdvanced={isAdvanced}
onEngineChange={handleEngineChange}
diff --git a/frontend/src/metabase/databases/components/DatabaseForm/DatabaseFormBody.tsx b/frontend/src/metabase/databases/components/DatabaseForm/DatabaseFormBody.tsx
index 1db4bd13885..63b1973ed6d 100644
--- a/frontend/src/metabase/databases/components/DatabaseForm/DatabaseFormBody.tsx
+++ b/frontend/src/metabase/databases/components/DatabaseForm/DatabaseFormBody.tsx
@@ -22,7 +22,6 @@ interface DatabaseFormBodyProps {
engine: Engine | undefined;
engineKey: EngineKey | undefined;
engines: Record<string, Engine>;
- engineFieldState?: "default" | "hidden" | "disabled";
autofocusFieldName?: string;
isAdvanced: boolean;
onEngineChange: (engineKey: string | undefined) => void;
@@ -35,7 +34,6 @@ export const DatabaseFormBody = ({
engine,
engineKey,
engines,
- engineFieldState = "default",
autofocusFieldName,
isAdvanced,
onEngineChange,
@@ -45,6 +43,7 @@ export const DatabaseFormBody = ({
}: DatabaseFormBodyProps): JSX.Element => {
const { setValues } = useFormikContext<DatabaseData>();
const hasConnectionError = useHasConnectionError();
+ const { engine: engineFieldConfig, name: nameFieldConfig } = config;
const px = match(location)
.with("setup", () => "sm")
@@ -56,14 +55,14 @@ export const DatabaseFormBody = ({
return (
<Box mah={mah} mb="md" px={px} style={{ overflowY: "auto" }}>
- {engineFieldState !== "hidden" && (
+ {engineFieldConfig?.fieldState !== "hidden" && (
<>
<DatabaseEngineField
engineKey={engineKey}
engines={engines}
isAdvanced={isAdvanced}
onChange={onEngineChange}
- disabled={engineFieldState === "disabled"}
+ disabled={engineFieldConfig?.fieldState === "disabled"}
showSampleDatabase={showSampleDatabase}
/>
<DatabaseEngineWarning
@@ -78,7 +77,7 @@ export const DatabaseFormBody = ({
location={location}
setValues={setValues}
/>
- {engine && (
+ {engine && nameFieldConfig?.fieldState !== "hidden" && (
<DatabaseNameField
engine={engine}
config={config}
diff --git a/frontend/src/metabase/databases/types.ts b/frontend/src/metabase/databases/types.ts
index bf9f4d32b39..2cbefd96700 100644
--- a/frontend/src/metabase/databases/types.ts
+++ b/frontend/src/metabase/databases/types.ts
@@ -26,17 +26,16 @@ export interface EngineFieldProps {
export type FormLocation = "admin" | "setup" | "embedding_setup" | "full-page";
-export type EngineFieldState = "default" | "hidden" | "disabled";
+export type FieldState = "default" | "hidden" | "disabled";
export interface DatabaseFormConfig {
/** present the form with advanced configuration options */
isAdvanced?: boolean;
engine?: {
- /** present the engine field as normal, disabled, or hidden */
- fieldState?: EngineFieldState | undefined;
+ fieldState?: FieldState;
};
name?: {
- /** present the name field as a slug */
+ fieldState?: FieldState;
isSlug?: boolean;
};
}
diff --git a/frontend/src/metabase/databases/utils/schema.ts b/frontend/src/metabase/databases/utils/schema.ts
index d5abfeea58e..5bce2ad9006 100644
--- a/frontend/src/metabase/databases/utils/schema.ts
+++ b/frontend/src/metabase/databases/utils/schema.ts
@@ -29,7 +29,6 @@ export const getValidationSchema = (
const definedFields = filterFieldsInAdvancedMode(flattenedFields, isAdvanced);
const fields = definedFields.filter(isDetailField);
const entries = fields.map((field) => [field.name, getFieldSchema(field)]);
-
return Yup.object({
id: Yup.number(),
engine: Yup.string().default(engineKey).required(Errors.required),
diff --git a/frontend/src/metabase/lib/urls/admin.ts b/frontend/src/metabase/lib/urls/admin.ts
index 945941f61af..c923eb50c2c 100644
--- a/frontend/src/metabase/lib/urls/admin.ts
+++ b/frontend/src/metabase/lib/urls/admin.ts
@@ -64,6 +64,10 @@ export function editDatabase(databaseId: DatabaseId) {
return `/admin/databases/${databaseId}/edit`;
}
+export function editDatabaseWritableConnection(databaseId: DatabaseId) {
+ return `/admin/databases/${databaseId}/write-data`;
+}
+
type DataModelParams = {
databaseId?: DatabaseId;
schemaName?: SchemaName | null;
diff --git a/frontend/src/metabase/plugins/components/PluginPlaceholder/PluginPlaceholder.tsx b/frontend/src/metabase/plugins/components/PluginPlaceholder/PluginPlaceholder.tsx
index cbe7124ed27..7abb0ce3c67 100644
--- a/frontend/src/metabase/plugins/components/PluginPlaceholder/PluginPlaceholder.tsx
+++ b/frontend/src/metabase/plugins/components/PluginPlaceholder/PluginPlaceholder.tsx
@@ -6,11 +6,11 @@ interface Props {
[key: string]: any;
}
-export function PluginPlaceholder(_props: Props): JSX.Element | null {
+export function PluginPlaceholder<T = Props>(_props: T): JSX.Element | null {
return null;
}
-export function NotFoundPlaceholder(_props: Props): JSX.Element | null {
+export function NotFoundPlaceholder<T = Props>(_props: T): JSX.Element | null {
return (
<EmptyState
illustrationElement={<img src={NoResults} />}
diff --git a/frontend/src/metabase/plugins/index.ts b/frontend/src/metabase/plugins/index.ts
index 2384bccc96e..fb95eceec47 100644
--- a/frontend/src/metabase/plugins/index.ts
+++ b/frontend/src/metabase/plugins/index.ts
@@ -15,6 +15,8 @@ export {
PLUGIN_LDAP_FORM_FIELDS,
PLUGIN_IS_PASSWORD_USER,
PLUGIN_ADMIN_USER_FORM_FIELDS,
+ type AuthSettingsPageProps,
+ type AuthSettingsPageTab,
} from "./oss/auth";
export {
PLUGIN_CACHING,
@@ -109,7 +111,10 @@ export {
} from "./oss/permissions";
export { PLUGIN_REMOTE_SYNC } from "./oss/remote-sync";
export { PLUGIN_RESOURCE_DOWNLOADS } from "./oss/resource-downloads";
-export { PLUGIN_SEMANTIC_SEARCH } from "./oss/semantic-search";
+export {
+ PLUGIN_SEMANTIC_SEARCH,
+ type SearchSettingsWidgetProps,
+} from "./oss/semantic-search";
export { PLUGIN_ADMIN_SETTINGS } from "./oss/settings";
export { PLUGIN_SMTP_OVERRIDE } from "./oss/smtp-override";
export {
diff --git a/frontend/src/metabase/plugins/oss/auth.ts b/frontend/src/metabase/plugins/oss/auth.ts
index 621f13f9d51..71fc7249de9 100644
--- a/frontend/src/metabase/plugins/oss/auth.ts
+++ b/frontend/src/metabase/plugins/oss/auth.ts
@@ -6,9 +6,18 @@ import type { User } from "metabase-types/api";
import type { GetAuthProviders } from "../types";
+export type AuthSettingsPageTab =
+ | "authentication"
+ | "user-provisioning"
+ | "api-keys";
+
+export type AuthSettingsPageProps = {
+ tab?: AuthSettingsPageTab;
+};
+
const getDefaultPluginAuthProviders = () => ({
isEnabled: () => false,
- AuthSettingsPage: PluginPlaceholder,
+ AuthSettingsPage: PluginPlaceholder<AuthSettingsPageProps>,
UserProvisioningSettings: NotFoundPlaceholder,
SettingsSAMLForm: NotFoundPlaceholder,
SettingsJWTForm: NotFoundPlaceholder,
diff --git a/frontend/src/metabase/plugins/oss/permissions.ts b/frontend/src/metabase/plugins/oss/permissions.ts
index 91abbcc24bf..52d258d18ab 100644
--- a/frontend/src/metabase/plugins/oss/permissions.ts
+++ b/frontend/src/metabase/plugins/oss/permissions.ts
@@ -1,4 +1,4 @@
-import type { ReactNode } from "react";
+import type { ComponentType, ReactNode } from "react";
import {
type DataPermission,
@@ -11,6 +11,7 @@ import {
import { getUserIsAdmin } from "metabase/selectors/user";
import type Database from "metabase-lib/v1/metadata/Database";
import type {
+ Database as DatabaseType,
Dataset,
Group,
GroupPermissions,
@@ -19,6 +20,7 @@ import type {
} from "metabase-types/api";
import type { State } from "metabase-types/store";
+import { PluginPlaceholder } from "../components/PluginPlaceholder";
import type { PluginGroupManagersType } from "../types";
const getDefaultAdminPermissionsDatabaseRoutes = () => [];
@@ -112,6 +114,10 @@ const getDefaultAdminUserMenuRoutes = (): (() => React.ReactNode)[] => [];
export const PLUGIN_ADMIN_USER_MENU_ITEMS = getDefaultAdminUserMenuItems();
export const PLUGIN_ADMIN_USER_MENU_ROUTES = getDefaultAdminUserMenuRoutes();
+export type WritableConnectionInfoSectionProps = {
+ database: DatabaseType;
+};
+
const getDefaultAdvancedPermissions = () => ({
addDatabasePermissionOptions: (permissions: any[], _database: Database) =>
permissions,
@@ -127,6 +133,9 @@ const getDefaultAdvancedPermissions = () => ({
isRestrictivePermission: (_value: string) => false,
shouldShowViewDataColumn: false,
defaultViewDataPermission: DataPermissionValue.UNRESTRICTED,
+ getWritableConnectionInfoRoutes: (_IsAdmin: ComponentType): ReactNode => null,
+ WritableConnectionInfoSection:
+ PluginPlaceholder<WritableConnectionInfoSectionProps>,
});
export const PLUGIN_ADVANCED_PERMISSIONS = getDefaultAdvancedPermissions();
diff --git a/frontend/src/metabase/plugins/oss/semantic-search.ts b/frontend/src/metabase/plugins/oss/semantic-search.ts
index ad79a76dc0b..604b4765ad4 100644
--- a/frontend/src/metabase/plugins/oss/semantic-search.ts
+++ b/frontend/src/metabase/plugins/oss/semantic-search.ts
@@ -1,7 +1,11 @@
import { PluginPlaceholder } from "metabase/plugins/components/PluginPlaceholder";
+export type SearchSettingsWidgetProps = {
+ statusPollingInterval?: number;
+};
+
const getDefaultPluginSemanticSearch = () => ({
- SearchSettingsWidget: PluginPlaceholder,
+ SearchSettingsWidget: PluginPlaceholder<SearchSettingsWidgetProps>,
});
export const PLUGIN_SEMANTIC_SEARCH = getDefaultPluginSemanticSearch();
diff --git a/modules/drivers/athena/src/metabase/driver/athena.clj b/modules/drivers/athena/src/metabase/driver/athena.clj
index 8bcf0e4b600..ecc2246c2a0 100644
--- a/modules/drivers/athena/src/metabase/driver/athena.clj
+++ b/modules/drivers/athena/src/metabase/driver/athena.clj
@@ -10,6 +10,7 @@
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
[metabase.driver.athena.schema-parser :as athena.schema-parser]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.sql-jdbc.common :as sql-jdbc.common]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
@@ -47,7 +48,7 @@
(defmethod driver/database-supports? [:athena :schemas]
[_driver _feature db]
- (not (seq (:dbname (:details db)))))
+ (not (seq (:dbname (driver.conn/effective-details db)))))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | metabase.driver.sql-jdbc method impls |
@@ -459,18 +460,19 @@
;; Because describe-table-fields might fail, we catch the error here and return an empty set of columns
(defmethod driver/describe-table :athena
- [driver {{:keys [catalog dbname]} :details, :as database} table]
- (sql-jdbc.execute/do-with-connection-with-options
- driver
- database
- nil
- (fn [^Connection conn]
- (let [metadata (.getMetaData conn)]
- (assoc (select-keys table [:name :schema])
- :fields (try
- (describe-table-fields metadata database driver table catalog dbname)
- (catch Throwable _
- (set nil))))))))
+ [driver database table]
+ (let [{:keys [catalog dbname]} (driver.conn/effective-details database)]
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver
+ database
+ nil
+ (fn [^Connection conn]
+ (let [metadata (.getMetaData conn)]
+ (assoc (select-keys table [:name :schema])
+ :fields (try
+ (describe-table-fields metadata database driver table catalog dbname)
+ (catch Throwable _
+ (set nil)))))))))
(defn- get-tables
[^DatabaseMetaData metadata, ^String schema-or-nil, ^String db-name-or-nil]
@@ -526,14 +528,14 @@
remarks)}))))))
(defmethod driver/describe-database* :athena
- [driver {details :details, :as database}]
+ [driver database]
(sql-jdbc.execute/do-with-connection-with-options
driver
database
nil
(fn [^Connection conn]
(let [metadata (.getMetaData conn)]
- {:tables (fast-active-tables driver metadata details)}))))
+ {:tables (fast-active-tables driver metadata (driver.conn/effective-details database))}))))
(defmethod sql.qp/format-honeysql :athena
[driver honeysql-form]
diff --git a/modules/drivers/athena/test/metabase/driver/athena_test.clj b/modules/drivers/athena/test/metabase/driver/athena_test.clj
index ae1378678a4..81a9e3a9412 100644
--- a/modules/drivers/athena/test/metabase/driver/athena_test.clj
+++ b/modules/drivers/athena/test/metabase/driver/athena_test.clj
@@ -445,7 +445,8 @@
[true? {:dbname nil}]
[true? {:dbname ""}]
[false? {:dbname "db_name"}]]]
- (is (schemas-supported? (driver/database-supports? :athena :schemas {:details details})))))
+ (is (schemas-supported? (driver/database-supports? :athena :schemas {:lib/type :metadata/database
+ :details details})))))
(deftest ^:parallel athena-describe-database
(mt/test-driver :athena
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index 248f264b20a..7df652219ff 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -1,5 +1,5 @@
(ns metabase.driver.bigquery-cloud-sdk
- (:refer-clojure :exclude [mapv some empty? not-empty get-in])
+ (:refer-clojure :exclude [mapv some empty? not-empty])
(:require
[clojure.core.async :as a]
[clojure.set :as set]
@@ -11,6 +11,7 @@
[metabase.driver.bigquery-cloud-sdk.params :as bigquery.params]
[metabase.driver.bigquery-cloud-sdk.query-processor :as bigquery.qp]
[metabase.driver.common.table-rows-sample :as table-rows-sample]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.settings :as driver.settings]
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc :as driver.sql-jdbc]
@@ -25,7 +26,7 @@
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [mapv some empty? not-empty get-in]]
+ [metabase.util.performance :refer [mapv some empty? not-empty]]
^{:clj-kondo/ignore [:discouraged-namespace]}
[toucan2.core :as t2])
(:import
@@ -186,8 +187,10 @@
(.getTable client dataset-id table-id empty-table-options)))
(mu/defn- get-table :- (driver-api/instance-of-class Table)
- (^Table [{{:keys [project-id]} :details, :as database} dataset-id table-id]
- (get-table (database-details->client (:details database)) project-id dataset-id table-id))
+ (^Table [database dataset-id table-id]
+ (let [details (driver.conn/effective-details database)
+ project-id (:project-id details)]
+ (get-table (database-details->client details) project-id dataset-id table-id)))
(^Table [^BigQuery client :- (driver-api/instance-of-class BigQuery)
project-id :- [:maybe driver-api/schema.common.non-blank-string]
@@ -196,8 +199,9 @@
(get-table* client project-id dataset-id table-id)))
(defmethod driver/table-exists? :bigquery-cloud-sdk
- [_ {:keys [details] :as _database} {table-id :name, dataset-id :schema :as _table}]
- (let [client (database-details->client details)
+ [_ database {table-id :name, dataset-id :schema :as _table}]
+ (let [details (driver.conn/effective-details database)
+ client (database-details->client details)
project-id (get-project-id details)]
(boolean
(get-table* client project-id dataset-id table-id))))
@@ -225,7 +229,8 @@
(defn- describe-database-tables
[driver database]
- (let [project-id (get-project-id (:details database))
+ (let [details (driver.conn/effective-details database)
+ project-id (get-project-id details)
query-dataset (fn [dataset-id]
(query-honeysql
driver
@@ -249,7 +254,7 @@
;; Maybe this is something we can do once we can parse sql
(= "BASE TABLE" table-type)
require-partition-filter))})]
- (->> (list-datasets (:details database) :logging-schema-exclusions? true)
+ (->> (list-datasets details :logging-schema-exclusions? true)
(eduction (mapcat (fn [dataset-id] (eduction (map #(table-info dataset-id %)) (query-dataset dataset-id))))))))
(defmethod driver/describe-database* :bigquery-cloud-sdk
@@ -475,8 +480,9 @@
(defmethod driver/describe-fields :bigquery-cloud-sdk
[driver database & {:keys [schema-names table-names]}]
- (let [project-id (get-project-id (:details database))
- dataset-ids (or schema-names (list-datasets (:details database)))]
+ (let [details (driver.conn/effective-details database)
+ project-id (get-project-id details)
+ dataset-ids (or schema-names (list-datasets details))]
;; The contract of [[driver/describe-fields]] requires results ordered by:
;; `table-schema`, `table-name`, `database-position`
@@ -636,7 +642,7 @@
(throw-invalid-query t sql parameters)))
(defn- effective-query-timezone-id [database]
- (if (get-in database [:details :use-jvm-timezone])
+ (if (:use-jvm-timezone (driver.conn/effective-details database))
(driver-api/system-timezone-id)
"UTC"))
@@ -732,22 +738,23 @@
;; - Waiting for the cancel-chan to get either a cancel message or to be closed.
;; - Running the BigQuery execution in another thread, since it's blocking.
(let [^BigQuery client (database-details->client database-details)
- result-promise (promise)
- request (build-bigquery-request sql parameters)
- job (.create client (JobInfo/of request) (u/varargs BigQuery$JobOption))
- job-id (.getJobId job)
- query-future (future
- ;; ensure the classloader is available within the future.
- (driver-api/the-classloader)
- (try
- (*page-callback*)
- (let [result-options (if *page-size* [(BigQuery$QueryResultsOption/pageSize *page-size*)] [])
- result (.getQueryResults job (u/varargs BigQuery$QueryResultsOption result-options))]
- (if result
- (deliver result-promise [:ready result])
- (throw (ex-info "Null response from query" {}))))
- (catch Throwable t
- (deliver result-promise [:error t]))))]
+ result-promise (promise)
+ request (build-bigquery-request sql parameters)
+ _ (driver.conn/track-connection-acquisition! database-details)
+ job (.create client (JobInfo/of request) (u/varargs BigQuery$JobOption))
+ job-id (.getJobId job)
+ query-future (future
+ ;; ensure the classloader is available within the future.
+ (driver-api/the-classloader)
+ (try
+ (*page-callback*)
+ (let [result-options (if *page-size* [(BigQuery$QueryResultsOption/pageSize *page-size*)] [])
+ result (.getQueryResults job (u/varargs BigQuery$QueryResultsOption result-options))]
+ (if result
+ (deliver result-promise [:ready result])
+ (throw (ex-info "Null response from query" {}))))
+ (catch Throwable t
+ (deliver result-promise [:error t]))))]
;; This `go` is responsible for cancelling the *initial* job execution.
;; Future pages may still not be fetched and so the reducer needs to check `cancel-chan` as well.
@@ -779,8 +786,7 @@
{:pre [(map? database) (map? (:details database))]}
;; automatically retry the query if it times out or otherwise fails. This is on top of the auto-retry added by
;; `execute`
- ;; Apply any swapped connection details (e.g., for workspace isolation)
- (let [details (driver/maybe-swap-details (:id database) (:details database))
+ (let [details (driver.conn/effective-details database)
thunk (fn []
(execute-bigquery
respond
@@ -801,7 +807,7 @@
(let [database (driver-api/database (driver-api/metadata-provider))]
(binding [bigquery.common/*bigquery-timezone-id* (effective-query-timezone-id database)]
(log/tracef "Running BigQuery query in %s timezone" bigquery.common/*bigquery-timezone-id*)
- (let [sql (if (get-in database [:details :include-user-id-and-hash] true)
+ (let [sql (if (:include-user-id-and-hash (driver.conn/effective-details database) true)
(str "-- " (driver-api/query->remark :bigquery-cloud-sdk outer-query) "\n" sql)
sql)]
(*process-native* respond database sql params (driver-api/canceled-chan))))))
@@ -880,27 +886,33 @@
;; duplicated tables with nil schema. Happily only in the "dataset-id" schema and not all schemas. But just
;; leave them with nil schemas and they will get deactivated in sync.
(catch Exception _e))
- (let [updated-db (-> (assoc-in database [:details :dataset-filters-type] "inclusion")
- (assoc-in [:details :dataset-filters-patterns] dataset-id)
- (m/dissoc-in [:details :dataset-id]))]
- (t2/update! :model/Database db-id {:details (:details updated-db)})
- updated-db)))
+ (let [updated-details (-> (driver.conn/default-details database)
+ (assoc :dataset-filters-type "inclusion")
+ (assoc :dataset-filters-patterns dataset-id)
+ (dissoc :dataset-id))]
+ (t2/update! :model/Database db-id {:details updated-details})
+ (assoc database :details updated-details))))
;; TODO: THIS METHOD SHOULD NOT BE UPDATING THE APP-DB (which it does in [convert-dataset-id-to-filters!])
;; Issue: https://github.com/metabase/metabase/issues/39392
+;; NOTE: This normalize is a legacy migration (OAuth -> service-account, dataset-id -> filters).
+;; write-data-details is a new feature and will never contain these legacy fields, so we only
+;; normalize the primary details here. The convert-dataset-id-to-filters! function has DB write
+;; side effects that are specific to the primary details.
(defmethod driver/normalize-db-details :bigquery-cloud-sdk
- [_driver {:keys [details] :as database}]
- (when-not (empty? (filter some? ((juxt :auth-code :client-id :client-secret) details)))
- (log/errorf (str "Database ID %d, which was migrated from the legacy :bigquery driver to :bigquery-cloud-sdk, has"
- " one or more OAuth style authentication scheme parameters saved to db-details, which cannot"
- " be automatically migrated to the newer driver (since it *requires* service-account-json instead);"
- " this database must therefore be updated by an administrator (by adding a service-account-json)"
- " before sync and queries will work again")
- (u/the-id database)))
- (if-let [dataset-id (get details :dataset-id)]
- (when-not (str/blank? dataset-id)
- (convert-dataset-id-to-filters! database dataset-id))
- database))
+ [_driver database]
+ (let [details (driver.conn/default-details database)]
+ (when-not (empty? (filter some? ((juxt :auth-code :client-id :client-secret) details)))
+ (log/errorf (str "Database ID %d, which was migrated from the legacy :bigquery driver to :bigquery-cloud-sdk, has"
+ " one or more OAuth style authentication scheme parameters saved to db-details, which cannot"
+ " be automatically migrated to the newer driver (since it *requires* service-account-json instead);"
+ " this database must therefore be updated by an administrator (by adding a service-account-json)"
+ " before sync and queries will work again")
+ (u/the-id database)))
+ (if-let [dataset-id (get details :dataset-id)]
+ (when-not (str/blank? dataset-id)
+ (convert-dataset-id-to-filters! database dataset-id))
+ database)))
(defmethod driver/prettify-native-form :bigquery-cloud-sdk
[_ native-form]
@@ -937,13 +949,17 @@
(defmethod driver/create-table! :bigquery-cloud-sdk
[driver database-id table-name column-definitions & {:keys [primary-key]}]
- (let [sql (#'driver.sql-jdbc/create-table!-sql driver table-name column-definitions :primary-key primary-key)]
- (driver/execute-raw-queries! driver (t2/select-one :model/Database database-id) [sql])))
+ (let [sql (#'driver.sql-jdbc/create-table!-sql driver table-name column-definitions :primary-key primary-key)
+ database (t2/select-one :model/Database database-id)
+ conn-spec (driver/connection-spec driver database)]
+ (driver/execute-raw-queries! driver conn-spec [sql])))
(defmethod driver/drop-table! :bigquery-cloud-sdk
[driver database-id table-name]
- (let [sql (driver/compile-drop-table driver table-name)]
- (driver/execute-raw-queries! driver (t2/select-one :model/Database database-id) [sql])))
+ (let [sql (driver/compile-drop-table driver table-name)
+ database (t2/select-one :model/Database database-id)
+ conn-spec (driver/connection-spec driver database)]
+ (driver/execute-raw-queries! driver conn-spec [sql])))
(defn- convert-value-for-insertion
[base-type value]
@@ -992,9 +1008,10 @@
(defmethod driver/insert-from-source! [:bigquery-cloud-sdk :rows]
[_driver db-id {table-name :name :keys [columns]} {:keys [data]}]
(let [col-names (map :name columns)
- {:keys [details]} (t2/select-one :model/Database db-id)
+ database (t2/select-one :model/Database db-id)
+ details (driver.conn/effective-details database)
- client (database-details->client details)
+ client (database-details->client details)
project-id (get-project-id details)
dataset-id (namespace table-name)
@@ -1008,18 +1025,22 @@
(doseq [^java.util.Map row chunk]
(.addRow insert-request-builder (InsertAllRequest$RowToInsert/of row)))
(let [insert-request (.build insert-request-builder)
- response (.insertAll client insert-request)]
+ response (.insertAll client insert-request)]
(when (.hasErrors response)
(let [errors (.getInsertErrors response)]
(throw (ex-info "BigQuery insert failed"
{:errors (into [] (map str errors))})))))))))
(defmethod driver/execute-raw-queries! :bigquery-cloud-sdk
- [_driver connection-details queries]
- ;; connection-details is either database details directly (from transforms)
- ;; or a database map with :details key (from other contexts)
- (let [details (get connection-details :details connection-details)
- client (database-details->client details)]
+ [driver conn-spec queries]
+ ;; conn-spec should be flat details (from driver/connection-spec)
+ ;; Defensive: multiple callsites used to pass in a database map. If the map has
+ ;; :details key, we will call driver/connection-spec on it.
+ (let [details (if (:details conn-spec)
+ (driver/connection-spec driver conn-spec)
+ conn-spec)
+ client (database-details->client details)]
+ (driver.conn/track-connection-acquisition! details)
(try
(doall
(for [query queries]
@@ -1041,14 +1062,14 @@
(let [qualified-name (if schema
(keyword schema name)
(keyword name))
- drop-sql (first (driver/compile-drop-table driver qualified-name))]
- (driver/execute-raw-queries! driver database [drop-sql])
+ drop-sql (first (driver/compile-drop-table driver qualified-name))
+ conn-spec (driver/connection-spec driver database)]
+ (driver/execute-raw-queries! driver conn-spec [drop-sql])
nil))
(defmethod driver/connection-spec :bigquery-cloud-sdk
[_driver database]
- ;; Return the database details directly since we don't use a JDBC spec for bigquery
- (driver/maybe-swap-details (:id database) (:details database)))
+ (driver.conn/effective-details database))
(defmethod driver.sql/default-schema :bigquery-cloud-sdk
[_]
@@ -1059,9 +1080,8 @@
;; Check if dataset exists using the BigQuery API before trying to create.
;; This is important for workspace isolation where the impersonated SA has
;; access to an existing isolated dataset but cannot create new datasets.
- (let [details (get conn-spec :details conn-spec)
- client (database-details->client details)
- project-id (get-project-id details)
+ (let [client (database-details->client conn-spec) ;; for bigquery, connection spec *is* the details
+ project-id (get-project-id conn-spec)
dataset-id (DatasetId/of project-id schema)]
(when-not (.getDataset client dataset-id (u/varargs BigQuery$DatasetOption))
;; Dataset doesn't exist, try to create it
@@ -1073,7 +1093,7 @@
(driver-api/with-metadata-provider db-id
(->> (driver-api/metadata-provider)
driver-api/database
- :details
+ driver.conn/effective-details
list-datasets
(some #{schema}))))
@@ -1366,7 +1386,7 @@
(defmethod driver/init-workspace-isolation! :bigquery-cloud-sdk
[_driver database workspace]
- (let [details (:details database)
+ (let [details (driver.conn/effective-details database)
client (ws-database-details->client details)
iam-client (ws-database-details->iam-client details)
project-id (get-project-id details)
@@ -1420,7 +1440,7 @@
[_driver database workspace tables]
;; For BigQuery, the workspace contains the service account email in database_details
(let [ws-sa-email (-> workspace :database_details :impersonate-service-account)
- details (:details database)
+ details (driver.conn/effective-details database)
client (ws-database-details->client details)
project-id (get-project-id details)]
@@ -1468,7 +1488,7 @@
(defmethod driver/destroy-workspace-isolation! :bigquery-cloud-sdk
[_driver database workspace]
- (let [details (:details database)
+ (let [details (driver.conn/effective-details database)
client (ws-database-details->client details)
iam-client (ws-database-details->iam-client details)
project-id (get-project-id details)
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj
index 346f1800948..748a3968c1e 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj
@@ -1,7 +1,9 @@
(ns metabase.driver.bigquery-cloud-sdk.common
"Common utility functions and utilities for the bigquery-cloud-sdk driver and related namespaces."
(:require
+ [metabase.driver.connection :as driver.conn]
[metabase.util :as u]
+ [metabase.util.log :as log]
[metabase.util.malli :as mu]
^{:clj-kondo/ignore [:discouraged-namespace]}
[toucan2.core :as t2])
@@ -57,8 +59,21 @@
Returns the calculated project-id (see [[database-details->credential-project-id]]) String from the credentials."
{:added "0.42.0"}
- ^String [{:keys [details] :as database} :- [:map [:details RequiredDetails]]]
- (let [creds-proj-id (database-details->credential-project-id details)]
+ ^String [database :- [:map [:details RequiredDetails]]]
+ ;; :project-id-from-credentials is a database-level cache managed by this driver. We store and read it from
+ ;; `:details` regardless of connection type. This is valid so long as read and write service accounts share a
+ ;; project ID. See also: [[metabase.driver.bigquery-cloud-sdk.query-processor/project-id-for-current-query]]
+ (let [details (driver.conn/default-details database)
+ creds-proj-id (database-details->credential-project-id details)]
+ (when (driver.conn/details-for-exact-type database :write-data)
+ (let [write-proj-id (driver.conn/with-write-connection
+ (database-details->credential-project-id
+ (driver.conn/effective-details database)))]
+ (when (not= creds-proj-id write-proj-id)
+ (log/warnf (str "Database %d: read and write service accounts belong to different GCP projects "
+ "(%s vs %s). The cached project-id-from-credentials uses the read SA's project; "
+ "query qualification may be incorrect for write connections.")
+ (u/the-id database) creds-proj-id write-proj-id))))
(t2/update! :model/Database
(u/the-id database)
{:details (assoc details :project-id-from-credentials creds-proj-id)})
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
index 1a9105e6f7e..6880b320750 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
@@ -10,6 +10,7 @@
[metabase.driver.bigquery-cloud-sdk.common :as bigquery.common]
[metabase.driver.common :as driver.common]
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.sql.parameters.substitution :as sql.params.substitution]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor.util :as sql.qp.u]
@@ -59,12 +60,16 @@
project ID associated with the service account credentials."
[]
(when (driver-api/initialized?)
- (when-let [{:keys [details], driver :engine, :as database} (driver-api/database (driver-api/metadata-provider))]
+ (when-let [{driver :engine, :as database} (driver-api/database (driver-api/metadata-provider))]
;; this is mostly here to catch tests that do something dumb like try to run a BigQuery tests with a MBQL query
;; targeting the H2 test database
(when driver
(assert (isa? driver/hierarchy driver :bigquery-cloud-sdk) "Sanity check: Database is not a BigQuery database"))
- (let [project-id-override (:project-id details)
+ ;; :project-id-from-credentials is a database-level cache managed by this driver. We store and read it from
+ ;; `:details` regardless of connection type. This is valid so long as read and write service accounts share a
+ ;; project ID. If they don't, [[bigquery.common/populate-project-id-from-credentials!]] will log a warning.
+ (let [details (driver.conn/default-details database)
+ project-id-override (:project-id details)
project-id-creds (:project-id-from-credentials details)
ret-fn (fn [proj-id-1 proj-id-2]
(when (and (some? proj-id-1) (not= proj-id-1 proj-id-2))
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
index 96ae85203a9..ecc5fe9cb87 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
@@ -1379,3 +1379,4 @@
(is (= ["INSERT INTO `PRODUCTS_COPY` SELECT * FROM products" nil]
(driver/compile-insert :bigquery-cloud-sdk {:query {:query "SELECT * FROM products"}
:output-table :PRODUCTS_COPY}))))))
+
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_introspection.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_introspection.clj
index 5cfad28a909..b825540f05a 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_introspection.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_introspection.clj
@@ -1,16 +1,17 @@
(ns metabase.driver.clickhouse-introspection
- (:refer-clojure :exclude [empty? get-in])
+ (:refer-clojure :exclude [empty?])
(:require
[clojure.java.jdbc :as jdbc]
[clojure.string :as str]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.ddl.interface :as ddl.i]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
[metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table]
[metabase.util :as u]
- [metabase.util.performance :refer [empty? get-in]])
+ [metabase.util.performance :refer [empty?]])
(:import
(java.sql Connection DatabaseMetaData)))
@@ -117,9 +118,10 @@
(defn- get-all-tables-in-all-dbs
[driver db]
- (let [db-filters-patterns (set (map (comp #(ddl.i/format-name driver %) str/trim)
- (remove empty? (str/split (or (get-in db [:details :db-filters-patterns]) "") #","))))
- db-filters-type (get-in db [:details :db-filters-type])]
+ (let [details (driver.conn/effective-details db)
+ db-filters-patterns (set (map (comp #(ddl.i/format-name driver %) str/trim)
+ (remove empty? (str/split (or (:db-filters-patterns details) "") #","))))
+ db-filters-type (:db-filters-type details)]
(sql-jdbc.execute/do-with-connection-with-options
driver db nil
(fn [^Connection conn]
@@ -144,8 +146,9 @@
;; but the actual sync from the UI uses :dbname
(defn- get-db-name
[db]
- (or (get-in db [:details :dbname])
- (get-in db [:details :db])))
+ (let [details (driver.conn/effective-details db)]
+ (or (:dbname details)
+ (:db details))))
(defn- get-tables-in-db
[driver db]
@@ -161,12 +164,12 @@
(set)))
(defmethod driver/describe-database* :clickhouse
- [driver {{:keys [enable-multiple-db]}
- :details :as db}]
- {:tables
- (if (boolean enable-multiple-db)
- (get-all-tables-in-all-dbs driver db)
- (get-tables-in-db driver db))})
+ [driver db]
+ (let [{:keys [enable-multiple-db]} (driver.conn/effective-details db)
+ tables (if (boolean enable-multiple-db)
+ (get-all-tables-in-all-dbs driver db)
+ (get-tables-in-db driver db))]
+ {:tables tables}))
(defn- ^:private is-db-required?
[field]
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj
index 4550a256944..1677b19ceaa 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj
@@ -1,10 +1,11 @@
-"Provides the info about the ClickHouse version. Extracted from the main clickhouse.clj file,
- as both Driver and QP overrides require access to it, avoiding circular dependencies."
(ns metabase.driver.clickhouse-version
+ "Provides the info about the ClickHouse version. Extracted from the main clickhouse.clj file,
+ as both Driver and QP overrides require access to it, avoiding circular dependencies."
(:require
[clojure.core.memoize :as memoize]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.util :as driver.u]))
@@ -21,15 +22,15 @@
(def ^:private ^{:arglists '([database])} get-clickhouse-version
(memoize/ttl
- ^{:clojure.core.memoize/args-fn (fn [[database]] (:details database))}
+ ^{:clojure.core.memoize/args-fn (fn [[database]] (driver.conn/effective-details database))}
(fn [database]
(sql-jdbc.execute/do-with-connection-with-options
:clickhouse
(sql-jdbc.conn/db->pooled-connection-spec database)
nil
(fn [^java.sql.Connection conn]
- (with-open [ver-stmt (.createStatement conn)
- ver-rset (.executeQuery ver-stmt clickhouse-version-query)
+ (with-open [ver-stmt (.createStatement conn)
+ ver-rset (.executeQuery ver-stmt clickhouse-version-query)
cloud-stmt (.createStatement conn)
cloud-rset (.executeQuery cloud-stmt "SELECT value='1' FROM system.settings WHERE name='cloud_mode'")]
(cond-> nil
diff --git a/modules/drivers/databricks/src/metabase/driver/databricks.clj b/modules/drivers/databricks/src/metabase/driver/databricks.clj
index 45f9a6f9c40..8916bc62c25 100644
--- a/modules/drivers/databricks/src/metabase/driver/databricks.clj
+++ b/modules/drivers/databricks/src/metabase/driver/databricks.clj
@@ -1,5 +1,5 @@
(ns metabase.driver.databricks
- (:refer-clojure :exclude [not-empty get-in])
+ (:refer-clojure :exclude [not-empty])
(:require
[clojure.java.jdbc :as jdbc]
[clojure.string :as str]
@@ -7,6 +7,7 @@
[java-time.api :as t]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.hive-like :as driver.hive-like]
[metabase.driver.sql-jdbc :as sql-jdbc]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
@@ -19,7 +20,7 @@
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.log :as log]
- [metabase.util.performance :refer [not-empty get-in]]
+ [metabase.util.performance :refer [not-empty]]
[ring.util.codec :as codec])
(:import
[java.sql
@@ -78,9 +79,10 @@
(defmethod driver/adjust-schema-qualification :databricks
[_driver database schema]
- (let [multi-level? (get-in database [:details :multi-level-schema])
- catalog (get-in database [:details :catalog])
- prefix (str catalog ".")]
+ (let [details (driver.conn/effective-details database)
+ multi-level? (:multi-level-schema details)
+ catalog (:catalog details)
+ prefix (str catalog ".")]
(cond
(and multi-level? (not (str/includes? schema ".")))
(str prefix schema)
@@ -134,7 +136,7 @@
(into
#{}
(filter (comp included? :schema))
- (sql-jdbc.execute/reducible-query database (get-tables-sql driver (:details database)))))}
+ (sql-jdbc.execute/reducible-query database (get-tables-sql driver (driver.conn/effective-details database)))))}
(catch Throwable e
(throw (ex-info (format "Error in %s describe-database: %s" driver (ex-message e))
{}
diff --git a/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj b/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj
index ad8b346ec8b..42c6519e14b 100644
--- a/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj
+++ b/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj
@@ -7,6 +7,7 @@
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
[metabase.driver.common :as driver.common]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
@@ -188,7 +189,7 @@
(defmethod driver/dbms-version :druid-jdbc
[_driver database]
- (let [{:keys [host port]} (:details database)]
+ (let [{:keys [host port]} (driver.conn/effective-details database)]
(try (let [version (-> (http/get (format "%s:%s/status" host port))
:body
json/decode
diff --git a/modules/drivers/druid/src/metabase/driver/druid/execute.clj b/modules/drivers/druid/src/metabase/driver/druid/execute.clj
index 9c2a13587be..ecc551aa201 100644
--- a/modules/drivers/druid/src/metabase/driver/druid/execute.clj
+++ b/modules/drivers/druid/src/metabase/driver/druid/execute.clj
@@ -4,6 +4,7 @@
[java-time.api :as t]
[medley.core :as m]
[metabase.driver-api.core :as driver-api]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.druid.query-processor :as druid.qp]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
@@ -150,12 +151,14 @@
:as mbql-query}
respond]
{:pre [query]}
- (let [details (:details (driver-api/database (driver-api/metadata-provider)))
+ (let [database (-> (driver-api/metadata-provider) driver-api/database)
+ details (driver.conn/effective-details database)
query (if (string? query)
(json/decode+kw query)
query)
query-type (or query-type
(keyword (namespace ::druid.qp/query) (name (:queryType query))))
+ _ (driver.conn/track-connection-acquisition! details)
results (try
(execute* details query)
(catch Throwable e
diff --git a/modules/drivers/druid/src/metabase/driver/druid/sync.clj b/modules/drivers/druid/src/metabase/driver/druid/sync.clj
index c48338a36c2..8d6297a8525 100644
--- a/modules/drivers/druid/src/metabase/driver/druid/sync.clj
+++ b/modules/drivers/druid/src/metabase/driver/druid/sync.clj
@@ -3,6 +3,7 @@
(:require
[medley.core :as m]
[metabase.driver-api.core :as driver-api]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.druid.client :as druid.client]
[metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh]
[metabase.util.performance :refer [select-keys]]))
@@ -28,7 +29,7 @@
"Impl of `driver/describe-table` for Druid."
[database table]
{:pre [(map? database) (map? table)]}
- (ssh/with-ssh-tunnel [details-with-tunnel (:details database)]
+ (ssh/with-ssh-tunnel [details-with-tunnel (driver.conn/effective-details database)]
(let [{:keys [columns aggregators]} (first (do-segment-metadata-query details-with-tunnel (:name table)))
metric-column-names (set (keys aggregators))]
{:schema nil
@@ -53,21 +54,23 @@
"Impl of `driver/describe-database` for Druid."
[database]
{:pre [(map? (:details database))]}
- (ssh/with-ssh-tunnel [details-with-tunnel (:details database)]
- (let [druid-datasources (druid.client/GET (druid.client/details->url details-with-tunnel "/druid/v2/datasources")
- :auth-enabled (-> database :details :auth-enabled)
- :auth-username (-> database :details :auth-username)
- :auth-token-value (driver-api/secret-value-as-string :druid (:details database) "auth-token"))]
- {:tables (set (for [table-name druid-datasources]
- {:schema nil, :name table-name}))})))
+ (let [details (driver.conn/effective-details database)]
+ (ssh/with-ssh-tunnel [details-with-tunnel details]
+ (let [druid-datasources (druid.client/GET (druid.client/details->url details-with-tunnel "/druid/v2/datasources")
+ :auth-enabled (:auth-enabled details)
+ :auth-username (:auth-username details)
+ :auth-token-value (driver-api/secret-value-as-string :druid details "auth-token"))]
+ {:tables (set (for [table-name druid-datasources]
+ {:schema nil, :name table-name}))}))))
(defn dbms-version
"Impl of `driver/dbms-version` for Druid."
[database]
{:pre [(map? (:details database))]}
- (ssh/with-ssh-tunnel [details-with-tunnel (:details database)]
- (-> (druid.client/GET (druid.client/details->url details-with-tunnel "/status")
- :auth-enabled (-> database :details :auth-enabled)
- :auth-username (-> database :details :auth-username)
- :auth-token-value (driver-api/secret-value-as-string :druid (:details database) "auth-token"))
- (select-keys [:version]))))
+ (let [details (driver.conn/effective-details database)]
+ (ssh/with-ssh-tunnel [details-with-tunnel details]
+ (-> (druid.client/GET (druid.client/details->url details-with-tunnel "/status")
+ :auth-enabled (:auth-enabled details)
+ :auth-username (:auth-username details)
+ :auth-token-value (driver-api/secret-value-as-string :druid details "auth-token"))
+ (select-keys [:version])))))
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.clj
index 14feb25369b..a03f17c8ab8 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj
@@ -7,6 +7,7 @@
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
[metabase.driver.common.table-rows-sample :as table-rows-sample]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.mongo.connection :as mongo.connection]
[metabase.driver.mongo.conversion :as mongo.conversion]
[metabase.driver.mongo.database :as mongo.db]
@@ -599,7 +600,7 @@
(defmethod driver/connection-spec :mongo
[_driver database]
- (driver/maybe-swap-details (:id database) (:details database)))
+ (driver.conn/effective-details database))
(defmulti ^:private type->database-type
"Internal type->database-type multimethod for MongoDB that dispatches on type."
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/connection.clj b/modules/drivers/mongo/src/metabase/driver/mongo/connection.clj
index 8923fcd5bed..01d044ea04f 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/connection.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/connection.clj
@@ -4,6 +4,7 @@
(:require
[clojure.string :as str]
[metabase.driver-api.core :as driver-api]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.mongo.database :as mongo.db]
[metabase.driver.mongo.util :as mongo.util]
[metabase.driver.settings :as driver.settings]
@@ -95,6 +96,7 @@
(let [db-details (mongo.db/details-normalized database)]
(ssh/with-ssh-tunnel [details-with-tunnel db-details]
(let [client (mongo.util/mongo-client (db-details->mongo-client-settings details-with-tunnel))]
+ (driver.conn/track-connection-acquisition! db-details)
(log/debug (u/format-color 'cyan "Opened new MongoClient."))
(try
(binding [*mongo-client* client]
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/database.clj b/modules/drivers/mongo/src/metabase/driver/mongo/database.clj
index b28ac309fd6..d0327dc8c51 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/database.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/database.clj
@@ -3,6 +3,7 @@
(:refer-clojure :exclude [empty? not-empty])
(:require
[metabase.driver-api.core :as driver-api]
+ [metabase.driver.connection :as driver.conn]
[metabase.util.i18n :refer [tru]]
[metabase.util.performance :refer [empty? not-empty]])
(:import
@@ -38,13 +39,26 @@
[database]
(let [db-details
(cond
- (integer? database) (driver-api/with-metadata-provider database
- (:details (driver-api/database (driver-api/metadata-provider))))
- (string? database) {:dbname database}
- (:dbname (:details database)) (:details database) ; entire Database obj
- (:dbname database) database ; connection details map only
- (:conn-uri database) database ; connection URI has all the parameters
- (:conn-uri (:details database)) (:details database)
+ (integer? database)
+ (driver-api/with-metadata-provider database
+ (driver.conn/effective-details (driver-api/database (driver-api/metadata-provider))))
+
+ (string? database)
+ {:dbname database}
+
+ ;; Full Database object - use effective-details to respect connection type
+ (:dbname (:details database))
+ (driver.conn/effective-details database)
+
+ (:dbname database)
+ database ; connection details map only
+
+ (:conn-uri database)
+ database ; connection URI has all the parameters
+
+ (:conn-uri (:details database))
+ (driver.conn/effective-details database)
+
:else
(throw (ex-info (tru "Unable to to get database details.")
{:database database})))]
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/database_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/database_test.clj
index 1b3a2cfb3e7..06f38d040ea 100644
--- a/modules/drivers/mongo/test/metabase/driver/mongo/database_test.clj
+++ b/modules/drivers/mongo/test/metabase/driver/mongo/database_test.clj
@@ -1,6 +1,7 @@
(ns metabase.driver.mongo.database-test
(:require
[clojure.test :refer :all]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.mongo.database :as mongo.db]))
(deftest ^:parallel fqdn?-test
@@ -17,3 +18,37 @@
(testing "`dbname` is encoded in conn-uri"
(is (= "some_db"
(mongo.db/db-name {:conn-uri "mongodb://localhost/some_db"})))))
+
+(deftest ^:parallel details-normalized-respects-connection-type-test
+ (testing "Mongo's details-normalized uses effective-details for full database objects"
+ (let [read-details {:dbname "test-db" :host "read-host"}
+ write-details {:dbname "test-db" :host "write-host"}
+ database {:details read-details
+ :write_data_details write-details}]
+ (testing "default connection type uses :details"
+ (let [result (mongo.db/details-normalized database)]
+ (is (= "read-host" (:host result)))))
+ (testing "write connection type uses :write_data_details"
+ (driver.conn/with-write-connection
+ (let [result (mongo.db/details-normalized database)]
+ (is (= "write-host" (:host result))))))))
+ (testing "Mongo's details-normalized falls back to :details when no write details"
+ (let [read-details {:dbname "test-db" :host "read-host"}
+ database {:details read-details}]
+ (driver.conn/with-write-connection
+ (let [result (mongo.db/details-normalized database)]
+ (is (= "read-host" (:host result))))))))
+
+(deftest ^:parallel details-normalized-conn-uri-respects-connection-type-test
+ (testing "Mongo's details-normalized uses effective-details for conn-uri databases"
+ (let [read-details {:conn-uri "mongodb://read-host:27017/test-db"}
+ write-details {:conn-uri "mongodb://write-host:27017/test-db"}
+ database {:details read-details
+ :write_data_details write-details}]
+ (testing "default connection type uses :details"
+ (let [result (mongo.db/details-normalized database)]
+ (is (= "mongodb://read-host:27017/test-db" (:conn-uri result)))))
+ (testing "write connection type uses :write_data_details"
+ (driver.conn/with-write-connection
+ (let [result (mongo.db/details-normalized database)]
+ (is (= "mongodb://write-host:27017/test-db" (:conn-uri result)))))))))
diff --git a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj b/modules/drivers/oracle/test/metabase/driver/oracle_test.clj
index cc0136168fd..1020c7f8e61 100644
--- a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj
+++ b/modules/drivers/oracle/test/metabase/driver/oracle_test.clj
@@ -158,6 +158,7 @@
{:name "tunnel-private-key-passphrase"}
{:name "advanced-options"}
{:name "destination-database"}
+ {:name "write-data-connection"}
{:name "auto_run_queries"}
{:name "let-user-control-scheduling"}
{:name "schedules.metadata_sync"}
diff --git a/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj b/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj
index fa341ba7926..0cd48617ca0 100644
--- a/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj
+++ b/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj
@@ -11,6 +11,7 @@
[java-time.api :as t]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.sql-jdbc.common :as sql-jdbc.common]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
@@ -599,35 +600,37 @@
(jdbc/reducible-query {:connection conn} sql))))
(defmethod driver/describe-database* :presto-jdbc
- [driver {{:keys [catalog schema] :as _details} :details :as database}]
- (sql-jdbc.execute/do-with-connection-with-options
- driver
- database
- nil
- (fn [^Connection conn]
- (let [schemas (if schema #{(describe-schema driver conn catalog schema)}
- (all-schemas driver conn catalog))]
- {:tables (reduce set/union #{} schemas)}))))
+ [driver database]
+ (let [{:keys [catalog schema]} (driver.conn/effective-details database)]
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver
+ database
+ nil
+ (fn [^Connection conn]
+ (let [schemas (if schema #{(describe-schema driver conn catalog schema)}
+ (all-schemas driver conn catalog))]
+ {:tables (reduce set/union #{} schemas)})))))
(defmethod driver/describe-table :presto-jdbc
- [driver {{:keys [catalog] :as _details} :details :as database} {schema :schema, table-name :name}]
- (sql-jdbc.execute/do-with-connection-with-options
- driver
- database
- nil
- (fn [^Connection conn]
- (let [sql (describe-table-sql driver catalog schema table-name)]
- (log/tracef "Running statement in describe-table: %s" sql)
- {:schema schema
- :name table-name
- :fields (into
- #{}
- (map-indexed (fn [idx {:keys [column type] :as _col}]
- {:name column
- :database-type type
- :base-type (presto-type->base-type type)
- :database-position idx}))
- (jdbc/reducible-query {:connection conn} sql))}))))
+ [driver database {schema :schema, table-name :name}]
+ (let [{:keys [catalog]} (driver.conn/effective-details database)]
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver
+ database
+ nil
+ (fn [^Connection conn]
+ (let [sql (describe-table-sql driver catalog schema table-name)]
+ (log/tracef "Running statement in describe-table: %s" sql)
+ {:schema schema
+ :name table-name
+ :fields (into
+ #{}
+ (map-indexed (fn [idx {:keys [column type] :as _col}]
+ {:name column
+ :database-type type
+ :base-type (presto-type->base-type type)
+ :database-position idx}))
+ (jdbc/reducible-query {:connection conn} sql))})))))
;;; The Presto JDBC driver DOES NOT support the `.getImportedKeys` method so just return `nil` here so the `:sql-jdbc`
;;; implementation doesn't try to use it.
diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
index 344a126a022..23494fd0db9 100644
--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
@@ -1,6 +1,6 @@
(ns metabase.driver.snowflake
"Snowflake Driver."
- (:refer-clojure :exclude [select-keys not-empty get-in mapv])
+ (:refer-clojure :exclude [select-keys not-empty mapv])
(:require
[buddy.core.codecs :as codecs]
[clojure.java.jdbc :as jdbc]
@@ -12,6 +12,7 @@
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
[metabase.driver.common :as driver.common]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc :as sql-jdbc]
[metabase.driver.sql-jdbc.common :as sql-jdbc.common]
@@ -34,7 +35,7 @@
[metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [get-in mapv not-empty select-keys]]
+ [metabase.util.performance :refer [mapv not-empty select-keys]]
[ring.util.codec :as codec])
(:import
(java.io File)
@@ -549,11 +550,11 @@
tests (and Snowflake itself) expected `details.db`. This has since been fixed, but for legacy support we'll still
accept either. Throw an Exception if neither key can be found."
{:arglists '([database])}
- [{details :details}]
- ;; ignore any blank keys
- (or (m/find-first (every-pred string? (complement str/blank?))
- ((juxt :db :dbname) details))
- (throw (Exception. (tru "Invalid Snowflake connection details: missing DB name.")))))
+ [database]
+ (let [details (driver.conn/effective-details database)]
+ (or (m/find-first (every-pred string? (complement str/blank?))
+ ((juxt :db :dbname) details))
+ (throw (Exception. (tru "Invalid Snowflake connection details: missing DB name."))))))
(defn- query-db-name []
;; the store is always initialized when running QP queries; for some stuff like the test extensions DDL statements
@@ -856,20 +857,26 @@
(jdbc/query spec (format "SHOW SCHEMAS IN DATABASE \"%s\";" db))
true))))
+(defn- normalize-details
+ "Normalize a Snowflake details map: merge regionid into account, infer use-password. Given nil, returns nil."
+ [details]
+ (cond-> details
+ (not (str/blank? (:regionid details)))
+ (-> (update :account #(str/join "." [% (:regionid details)]))
+ (dissoc :regionid))
+
+ (and (not (contains? details :use-password))
+ (:password details)
+ (nil? (:private-key-id details))
+ (nil? (:private-key-path details))
+ (nil? (:private-key-value details)))
+ (assoc :use-password true)))
+
(defmethod driver/normalize-db-details :snowflake
[_ database]
- (cond-> database
- (not (str/blank? (-> database :details :regionid)))
- (-> (update-in [:details :account] #(str/join "." [% (-> database :details :regionid)]))
- (m/dissoc-in [:details :regionid]))
-
- (and
- (not (contains? (:details database) :use-password))
- (get-in database [:details :password])
- (nil? (get-in database [:details :private-key-id]))
- (nil? (get-in database [:details :private-key-path]))
- (nil? (get-in database [:details :private-key-value])))
- (assoc-in [:details :use-password] true)))
+ (-> database
+ (m/update-existing :details normalize-details)
+ (m/update-existing :write_data_details normalize-details)))
;;; If you try to read a Snowflake `timestamptz` as a String with `.getString` it always comes back in
;;; `America/Los_Angeles` for some reason I cannot figure out. Let's just read them out as UTC, which is what they're
@@ -919,7 +926,7 @@
(defmethod driver.sql/default-database-role :snowflake
[_ database]
- (-> database :details :role))
+ (:role (driver.conn/effective-details database)))
(defmethod sql-jdbc/impl-query-canceled? :snowflake [_ e]
(= (sql-jdbc/get-sql-state e) "57014"))
@@ -1002,9 +1009,10 @@
(defmethod driver/init-workspace-isolation! :snowflake
[_driver database workspace]
- (let [schema-name (driver.u/workspace-isolation-namespace-name workspace)
- db-name (-> database :details :db)
- warehouse (-> database :details :warehouse)
+ (let [details (driver.conn/effective-details database)
+ schema-name (driver.u/workspace-isolation-namespace-name workspace)
+ db-name (:db details)
+ warehouse (:warehouse details)
role-name (isolation-role-name workspace)
read-user {:user (driver.u/workspace-isolation-user-name workspace)
:password (driver.u/random-workspace-password)}
@@ -1032,8 +1040,9 @@
(defmethod driver/destroy-workspace-isolation! :snowflake
[_driver database workspace]
- (let [schema-name (driver.u/workspace-isolation-namespace-name workspace)
- db-name (-> database :details :db)
+ (let [details (driver.conn/effective-details database)
+ schema-name (driver.u/workspace-isolation-namespace-name workspace)
+ db-name (:db details)
role-name (isolation-role-name workspace)
username (driver.u/workspace-isolation-user-name workspace)
conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))]
@@ -1049,7 +1058,7 @@
(defmethod driver/grant-workspace-read-access! :snowflake
[_driver database workspace tables]
(let [conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))
- db-name (-> database :details :db)
+ db-name (:db (driver.conn/effective-details database))
role-name (-> workspace :database_details :role)]
(when-not db-name
(throw (ex-info "Snowflake database configuration is missing required 'db' (database name) setting"
diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
index 6fb3ba3f8b2..58473e656e2 100644
--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
@@ -931,6 +931,25 @@
(is (=? {:password "abc" :private-key-id int? :use-password complement} (:details db2)))
(is (= {:password "abc" :use-password false} (:details db3)))))))
+(deftest ^:parallel normalize-write-data-details-test
+ (mt/test-driver :snowflake
+ (testing "write-data-details should be normalized the same way as details"
+ (testing "regionid normalization"
+ (mt/with-temp [:model/Database db {:name "Snowflake DB with write details"
+ :engine :snowflake
+ :details {:account "my-instance"}
+ :write_data_details {:account "write-instance"
+ :regionid "eu-west-2"}}]
+ (is (= {:account "write-instance.eu-west-2"}
+ (:write_data_details db)))))
+ (testing "use-password inference"
+ (mt/with-temp [:model/Database db {:name "Snowflake DB with write pw"
+ :engine :snowflake
+ :details {:account "my-instance"}
+ :write_data_details {:password "secret"}}]
+ (is (= {:password "secret" :use-password true}
+ (:write_data_details db))))))))
+
(deftest ^:parallel set-role-statement-test
(testing "set-role-statement should return a USE ROLE command, with the role quoted if it contains special characters"
;; No special characters
diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
index e4fbc104555..ec8c4734ea0 100644
--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
@@ -11,6 +11,7 @@
[java-time.api :as t]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc :as sql-jdbc]
[metabase.driver.sql-jdbc.common :as sql-jdbc.common]
@@ -1010,8 +1011,7 @@
;; Use a "role" (sqlserver user) if it exists. Do not fall back to the user
;; field automatically, as it represents the login user which may not be a
;; valid database user for impersonation (see issue #60665).
- (let [{:keys [role]} (:details database)]
- role))
+ (:role (driver.conn/effective-details database)))
(defmethod driver.sql/set-role-statement :sqlserver
[_driver role]
diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
index 6f5ae8bd5b5..0c80ef82185 100644
--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
@@ -709,19 +709,23 @@
(deftest ^:parallel default-database-role-test
(testing "SQL Server default database role handling"
(testing "returns role when explicitly configured"
- (let [database {:details {:user "login_user" :role "db_user"}}]
+ (let [database {:lib/type :metadata/database
+ :details {:user "login_user" :role "db_user"}}]
(is (= "db_user" (driver.sql/default-database-role :sqlserver database)))))
(testing "returns nil when no role is configured"
- (let [database {:details {:user "login_user"}}]
+ (let [database {:lib/type :metadata/database
+ :details {:user "login_user"}}]
(is (nil? (driver.sql/default-database-role :sqlserver database)))))
(testing "returns nil even when user is 'sa'"
- (let [database {:details {:user "sa"}}]
+ (let [database {:lib/type :metadata/database
+ :details {:user "sa"}}]
(is (nil? (driver.sql/default-database-role :sqlserver database)))))
(testing "ignores user field and only uses role field"
- (let [database {:details {:user "login_user" :role "impersonation_user"}}]
+ (let [database {:lib/type :metadata/database
+ :details {:user "login_user" :role "impersonation_user"}}]
(is (= "impersonation_user" (driver.sql/default-database-role :sqlserver database)))))))
(deftest ^:parallel wtf-test
diff --git a/modules/drivers/starburst/src/metabase/driver/starburst.clj b/modules/drivers/starburst/src/metabase/driver/starburst.clj
index adbb9258a64..33eabbfcdad 100644
--- a/modules/drivers/starburst/src/metabase/driver/starburst.clj
+++ b/modules/drivers/starburst/src/metabase/driver/starburst.clj
@@ -1,6 +1,6 @@
(ns metabase.driver.starburst
"starburst driver."
- (:refer-clojure :exclude [select-keys get-in])
+ (:refer-clojure :exclude [select-keys])
(:require
[clojure.java.jdbc :as jdbc]
[clojure.set :as set]
@@ -10,6 +10,7 @@
[java-time.api :as t]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc.common :as sql-jdbc.common]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
@@ -26,7 +27,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :refer [trs]]
[metabase.util.log :as log]
- [metabase.util.performance :refer [select-keys get-in]])
+ [metabase.util.performance :refer [select-keys]])
(:import
(com.mchange.v2.c3p0 C3P0ProxyConnection)
(io.trino.jdbc TrinoConnection)
@@ -489,37 +490,39 @@
(jdbc/reducible-result-set rs {})))))
(defmethod driver/describe-database* :starburst
- [driver {{:keys [catalog schema] :as _details} :details :as database}]
- (sql-jdbc.execute/do-with-connection-with-options
- driver
- database
- nil
- (fn [^Connection conn]
- (let [schemas (if schema
- #{(describe-schema driver conn catalog schema)}
- (all-schemas driver conn catalog))]
- {:tables (reduce set/union #{} schemas)}))))
+ [driver database]
+ (let [{:keys [catalog schema]} (driver.conn/effective-details database)]
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver
+ database
+ nil
+ (fn [^Connection conn]
+ (let [schemas (if schema
+ #{(describe-schema driver conn catalog schema)}
+ (all-schemas driver conn catalog))]
+ {:tables (reduce set/union #{} schemas)})))))
(defmethod driver/describe-table :starburst
- [driver {{:keys [catalog] :as _details} :details :as database} {schema :schema, table-name :name}]
- (sql-jdbc.execute/do-with-connection-with-options
- driver
- database
- nil
- (fn [^Connection conn]
- (with-open [stmt (.createStatement conn)]
- (let [sql (describe-table-sql driver catalog schema table-name)
- rs (sql-jdbc.execute/execute-statement! driver stmt sql)]
- {:schema schema
- :name table-name
- :fields (into
- #{}
- (map-indexed (fn [idx {:keys [column type] :as _col}]
- {:name column
- :database-type type
- :base-type (starburst-type->base-type type)
- :database-position idx}))
- (jdbc/reducible-result-set rs {}))})))))
+ [driver database {schema :schema, table-name :name}]
+ (let [{:keys [catalog]} (driver.conn/effective-details database)]
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver
+ database
+ nil
+ (fn [^Connection conn]
+ (with-open [stmt (.createStatement conn)]
+ (let [sql (describe-table-sql driver catalog schema table-name)
+ rs (sql-jdbc.execute/execute-statement! driver stmt sql)]
+ {:schema schema
+ :name table-name
+ :fields (into
+ #{}
+ (map-indexed (fn [idx {:keys [column type] :as _col}]
+ {:name column
+ :database-type type
+ :base-type (starburst-type->base-type type)
+ :database-position idx}))
+ (jdbc/reducible-result-set rs {}))}))))))
(defmethod driver/db-default-timezone :starburst
[driver database]
@@ -993,7 +996,7 @@
(defmethod driver.sql/default-database-role :starburst
[_driver database]
- (get-in database [:details :user]))
+ (:user (driver.conn/effective-details database)))
(defmethod driver/set-role! :starburst
[_driver ^Connection conn role]
diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yaml
index c743d48d805..95e1c9938d8 100644
--- a/resources/migrations/059_update_migrations.yaml
+++ b/resources/migrations/059_update_migrations.yaml
@@ -3467,6 +3467,30 @@ databaseChangeLog:
path: instance_analytics_views/tasks/v4/h2-tasks.sql
relativeToChangelogFile: true
+ - changeSet:
+ id: v59.2026-02-16T18:00:00
+ author: galdre
+ comment: Add write_data_details column to metabase_database for PRO-86 separate write connections
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - columnExists:
+ tableName: metabase_database
+ columnName: write_data_details
+ changes:
+ - addColumn:
+ tableName: metabase_database
+ columns:
+ - column:
+ name: write_data_details
+ type: ${text.type}
+ remarks: Encrypted JSON containing connection details for write operations (transforms, etc.)
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: metabase_database
+ columnName: write_data_details
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index de28d960b96..d21ea10788e 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -136,12 +136,12 @@
(defn- conn-pool-bean-diag-info [acc ^ObjectName jmx-bean]
;; We should not be using specific driver implementations
- (let [pool-var (requiring-resolve 'metabase.driver.sql-jdbc.connection/database-id->connection-pool)]
+ (let [pool-var (requiring-resolve 'metabase.driver.sql-jdbc.connection/pool-cache-key->connection-pool)]
;; Using this `locking` is non-obvious but absolutely required to avoid the deadlock inside c3p0 implementation. The
;; act of JMX attribute reading first locks a DynamicPooledDataSourceManagerMBean object, and then a
;; PoolBackedDataSource object. Conversely, the act of creating a pool (with
;; com.mchange.v2.c3p0.DataSources/pooledDataSource) first locks PoolBackedDataSource and then
- ;; DynamicPooledDataSourceManagerMBean. We have to lock a common monitor (which `database-id->connection-pool` is)
+ ;; DynamicPooledDataSourceManagerMBean. We have to lock a common monitor (which `pool-cache-key->connection-pool` is)
;; to prevent the deadlock. Hopefully.
;; Issue against c3p0: https://github.com/swaldman/c3p0/issues/95
(locking @pool-var
@@ -265,7 +265,7 @@
{:description "Number of times this instance converted a card's MBQL 4 to 5 and `clean` made a real change"})
(prometheus/gauge :metabase-database/status
{:description "Does a given database using driver pass a health check."
- :labels [:driver :healthy :reason]})
+ :labels [:driver :healthy :reason :connection-type]})
(prometheus/counter :metabase-query-processor/query
{:description "Did a query run by a specific driver succeed or fail"
:labels [:driver :status]})
@@ -501,6 +501,13 @@
(prometheus/counter :metabase-token-check/attempt
{:description "Total number of token checks. Includes a status label."
:labels [:status]})
+ ;; Write-connection telemetry
+ (prometheus/counter :metabase-db-connection/write-op
+ {:description "JDBC connection pool acquisitions by connection type (default or write-data)."
+ :labels [:connection-type]})
+ (prometheus/counter :metabase-db-connection/type-resolved
+ {:description "Write-data details resolved by effective-details (driver-agnostic). Only incremented when write-data-details are genuinely used, not on fallback or workspace swap."
+ :labels [:connection-type]})
;; SQL parsing metrics
(prometheus/counter :metabase-sql-parsing/context-timeouts
{:description "Number of Python/GraalVM SQL parsing execution timeouts."})
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index 155a90d7672..218d572c8ed 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -21,7 +21,6 @@
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.json :as json]
- [metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.performance :refer [mapv empty?]]
@@ -63,91 +62,6 @@
[driver & body]
`(do-with-driver ~driver (fn [] ~@body)))
-;;; +----------------------------------------------------------------------------------------------------------------+
-;;; | Connection Details Swapping |
-;;; +----------------------------------------------------------------------------------------------------------------+
-
-(def ^:private ^:dynamic *swapped-connection-details*
- "A dynamic var that holds a map of database-id -> swapped-details-map for temporarily swapping connection details.
- When a connection spec is created for a database, if its ID is present in this map, the swap map will be
- merged into the connection `:details` before they are used to create a connection.
-
- This provides a mechanism for temporarily using different connection details (e.g., using alternative credentials
- for workspaces) without mutating the database record.
-
- The swap map is merged into the database `:details` map. The swap is applied before any connection-specific
- processing (like hash calculation for connection pooling), so different swaps will result in different
- connection pools.
-
- Different drivers may apply this swap at different points in their connection lifecycle, but the semantics
- are consistent: swapped details are used for the duration of the dynamic scope.
-
- See [[with-swapped-connection-details]] for usage."
- nil)
-
-(defn- apply-detail-swaps
- "Merges the `swap-map` into `details`. Supports nested maps via deep merge."
- [details swap-map]
- (reduce-kv
- (fn [acc k v]
- (if (and (map? v) (map? (get acc k)))
- (assoc acc k (apply-detail-swaps (get acc k) v))
- (assoc acc k v)))
- details
- swap-map))
-
-(defn has-connection-swap?
- "Returns true if there is an active connection detail swap for `database-id`."
- [database-id]
- (contains? *swapped-connection-details* database-id))
-
-(defn maybe-swap-details
- "Returns the database details with any swaps applied from [[*swapped-connection-details*]].
- If no swap exists for `database-id`, returns `details` unchanged.
-
- Drivers should call this function when creating connections to apply any active swaps.
- For JDBC drivers, this is called in [[metabase.driver.sql-jdbc.connection/db->pooled-connection-spec]].
- For other drivers (e.g., MongoDB), this should be called in their connection creation logic."
- [database-id details]
- (if-let [swap-map (get *swapped-connection-details* database-id)]
- (do
- (log/debugf "Applying swapped connection details for database %d, swap keys: %s"
- database-id (keys swap-map))
- (apply-detail-swaps details swap-map))
- details))
-
-(defn do-with-swapped-connection-details
- "Implementation for [[with-swapped-connection-details]]."
- [database-id swap-map thunk]
- (when (contains? *swapped-connection-details* database-id)
- (throw (ex-info "Nested connection detail swaps are not supported for the same database"
- {:database-id database-id})))
- (log/debugf "Entering swapped connection details scope for database %d, swap keys: %s"
- database-id (keys swap-map))
- (binding [*swapped-connection-details* (assoc *swapped-connection-details* database-id swap-map)]
- (thunk)))
-
-(defmacro with-swapped-connection-details
- "Temporarily swap the connection details for a specific database within the dynamic scope of `body`.
-
- The `swap-map` is a map of detail keys to swap values. These will be merged into the database's
- connection `:details` map. Nested maps are deep-merged.
-
- Any code that creates a connection for `database-id` within this scope will use the modified details.
-
- **Important:** Nested swaps for the same database are not supported and will throw an exception.
- Different databases can have concurrent swaps.
-
- Example:
-
- ;; Swap connection to use alternate credentials
- (driver/with-swapped-connection-details 1 {:user \"workspace-user\" :password \"workspace-pass\"}
- ;; All connections created in this scope use the swapped credentials
- (qp/process-query query))"
- {:style/indent 2}
- [database-id swap-map & body]
- `(do-with-swapped-connection-details ~database-id ~swap-map (fn [] ~@body)))
-
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Driver Registration / Hierarchy / Multimethod Dispatch |
;;; +----------------------------------------------------------------------------------------------------------------+
@@ -1235,7 +1149,12 @@
regular Liquibase queries. This multimethod will be called from a `:post-select` handler within the database model.
The full `database` model object is passed as the 2nd parameter, and the multimethod implementation is expected to
update the value for `:details`. The default implementation is essentially `identity` (i.e returns `database`
- unchanged). This multimethod will only be called if `:details` is actually present in the `database` map."
+ unchanged). This multimethod will only be called if `:details` is actually present in the `database` map.
+
+ Implementations should normalize both `:details` and `:write-data-details` (if present), since
+ `:write-data-details` is merged on top of `:details` by [[metabase.driver.connection/effective-details]].
+ Un-normalized fields in `:write-data-details` can leak through to the merged result. See
+ [[metabase.driver.connection]] for more information."
{:added "0.41.0" :arglists '([driver database])}
dispatch-on-initialized-driver
:hierarchy #'hierarchy)
diff --git a/src/metabase/driver/common.clj b/src/metabase/driver/common.clj
index 0b6f4e226f7..12d82d09a91 100644
--- a/src/metabase/driver/common.clj
+++ b/src/metabase/driver/common.clj
@@ -1,16 +1,15 @@
(ns metabase.driver.common
"Shared definitions and helper functions for use across different drivers."
- (:refer-clojure :exclude [get-in])
#_{:clj-kondo/ignore [:metabase/modules]}
(:require
[clojure.string :as str]
[metabase.driver :as driver]
+ [metabase.driver.connection :as driver.conn]
[metabase.premium-features.core :as premium-features]
[metabase.settings.core :as setting]
[metabase.util.i18n :refer [deferred-tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [get-in]]
[metabase.warehouses.core :as warehouses]))
(set! *warn-on-reflection* true)
@@ -130,6 +129,12 @@
:type :hidden
:default false})
+(def write-data-connection-option
+ "Map representing the 'is this a writable connection' option"
+ {:name "write-data-connection"
+ :type :hidden
+ :default false})
+
(def advanced-options-start
"Map representing the start of the advanced option section in a DB connection form. Fields in this section should
have their visibility controlled using the `visible-if` property."
@@ -147,7 +152,8 @@
:description (deferred-tru
(str "We execute the underlying query when you explore data using Summarize or Filter. "
"This is on by default but you can turn it off if performance is slow."))
- :visible-if {"advanced-options" true}})
+ :visible-if {"advanced-options" true
+ "write-data-connection" false}})
(def let-user-control-scheduling
"Map representing the `let-user-control-scheduling` option in a DB connection form."
@@ -155,7 +161,8 @@
:type :boolean
:display-name (deferred-tru "Choose when syncs and scans happen")
:description (deferred-tru "By default, Metabase does a lightweight hourly sync and an intensive daily scan of field values. If you have a large database, turn this on to make changes.")
- :visible-if {"advanced-options" true}})
+ :visible-if {"advanced-options" true
+ "write-data-connection" false}})
(def metadata-sync-schedule
"Map representing the `schedules.metadata_sync` option in a DB connection form, which should be only visible if
@@ -184,7 +191,8 @@
{:name "json-unfolding"
:display-name (deferred-tru "Allow unfolding of JSON columns")
:type :boolean
- :visible-if {"advanced-options" true}
+ :visible-if {"advanced-options" true
+ "write-data-connection" false}
:description (deferred-tru
(str "This enables unfolding JSON columns into their component fields. "
"Disable unfolding if performance is slow. If enabled, you can still disable unfolding for "
@@ -199,7 +207,8 @@
:description (deferred-tru
(str "This enables Metabase to scan for additional field values during syncs allowing smarter "
"behavior, like improved auto-binning on your bar charts."))
- :visible-if {"advanced-options" true}})
+ :visible-if {"advanced-options" true
+ "write-data-connection" false}})
(def multi-level-schema
"Map representing the `multi-level-schema` option for databases. Stores schemas with multiple levels of hierarchy."
@@ -208,8 +217,14 @@
:default false})
(def default-advanced-options
- "Vector containing the three most common options present in the advanced option section of the DB connection form."
- [destination-database-option auto-run-queries let-user-control-scheduling metadata-sync-schedule cache-field-values-schedule refingerprint])
+ "Vector containing the most common options present in the advanced option section of the DB connection form."
+ [destination-database-option
+ write-data-connection-option
+ auto-run-queries
+ let-user-control-scheduling
+ metadata-sync-schedule
+ cache-field-values-schedule
+ refingerprint])
(def default-options
"Default options listed above, keyed by name. These keys can be listed in the plugin manifest to specify connection
@@ -406,7 +421,7 @@
;; This allows adding support for nested-field-columns for drivers in the future and
;; have json-unfolding enabled by default, without
;; needing a migration to add the `json-unfolding=true` key to the database details.
- (let [json-unfolding (get-in database [:details :json-unfolding])]
+ (let [json-unfolding (:json-unfolding (driver.conn/effective-details database))]
(if (nil? json-unfolding)
true
json-unfolding)))
diff --git a/src/metabase/driver/connection.clj b/src/metabase/driver/connection.clj
new file mode 100644
index 00000000000..022dccfef2e
--- /dev/null
+++ b/src/metabase/driver/connection.clj
@@ -0,0 +1,174 @@
+(ns metabase.driver.connection
+ "Centralized access to database connection details.
+
+ Direct access to `(:details database)` is an anti-pattern. It couples callers to
+ the raw data layout, which means any change to how details are resolved — connection
+ routing, write credentials, workspace isolation, security boundaries — requires
+ finding and updating every call site. This namespace provides the indirection that
+ makes those changes possible.
+
+ Primary API: [[effective-details]], [[with-write-connection]], [[default-details]]."
+ (:require
+ [metabase.analytics.prometheus :as prometheus]
+ [metabase.driver.connection.workspaces :as driver.w]
+ [metabase.driver.util :as driver.u]
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [metabase.util.malli.registry :as mr]))
+
+;; Database `:details` is a single encrypted JSON column that stores everything a driver
+;; needs to know about a connection. The problem is that "everything a driver needs to know"
+;; has grown to become a big blob:
+;;
+;; - Credentials: `password`, `secret_key`, `service-account-json`, `private-key`,
+;; `tunnel-private-key`, `access-token`, and friends. These are what you'd expect
+;; to find in an encrypted column.
+;;
+;; - Connection identifiers: `host`, `port`, `db`, `user`, `account`, `warehouse`.
+;; These identify where and as whom to connect. Not credentials per se, but together
+;; with the above they form a complete connection string.
+;;
+;; - Configuration and preferences: `auto_run_queries`, `let-user-control-scheduling`,
+;; `schedules.metadata_sync`, `refingerprint`, `json-unfolding`, `schema-filters`.
+;; These control Metabase behavior. They do not participate in authentication or
+;; connection establishment in any way.
+;;
+;; All three categories live in the same encrypted blob, which means `t2/select :model/Database`
+;; decrypts AWS secret keys just so the sync scheduler can check whether the user wants to
+;; control their own sync timing. Every code path that touches any detail — even purely
+;; cosmetic ones — carries the full credential payload through memory.
+;;
+;; Across 19 drivers it seems there are roughly 80 distinct field names that could live in
+;; `:details`, with inconsistent naming conventions (snake_case, kebab-case, etc. together).
+;; Some fields are ambiguous: `additional-options`, `kerberos-keytab-path`, etc. The existing
+;; `sensitive-fields` function in `driver.util` identifies credential-type fields well, but
+;; there's no corresponding classification for "this field doesn't need to be encrypted at all."
+;;
+;; This namespace centralizes all reads of `:details` (and `:write-data-details`) behind
+;; functions that can apply connection-type routing, workspace isolation, and — eventually —
+;; a proper separation of credentials from configuration. The immediate goal is to make
+;; direct `(:details database)` access a greppable code smell. The longer-term goal is to
+;; make it possible to store non-sensitive configuration outside the encrypted column
+;; without a codebase-wide refactor. The immediate need for this namespace was to make it
+;; possible to access and *use* configuration in a controllable and auditable way as we
+;; are going from one master-connection to a database to two + any number of dynamic
+;; workspaces connections.
+;;
+;; Please use/adapt/augment/improve this namespace and avoid all such patterns:
+;; - (:details database)
+;; - (let [{:keys [details]} database])
+;; - (let [{{:keys [user]} :details} database])
+
+(mr/def ::connection-type
+ [:enum :default :write-data])
+
+(def ^:dynamic *connection-type*
+ "Which connection details [[effective-details]] should resolve.
+
+ - `:default` — primary `:details`
+ - `:write-data` — `:write-data-details` merged over `:details` (if configured)
+
+ Bind via [[with-write-connection]], not directly."
+ :default)
+
+(defmacro with-write-connection
+ "Establishes a write-connection context for body.
+
+ [[effective-details]] calls within this scope resolve to take `:write-data-details`
+ into account (if configured) instead of only primary `:details`."
+ [& body]
+ `(binding [*connection-type* :write-data]
+ ~@body))
+
+(def ^:dynamic ^:private *suppress-resolution-telemetry*
+ false)
+
+(defmacro without-resolution-telemetry
+ "Suppresses [[effective-details]] from incrementing the `:metabase-db-connection/type-resolved`
+ Prometheus counter. Use for infrastructure calls (e.g., health checks) that resolve
+ write-data-details but should not inflate feature-usage metrics."
+ [& body]
+ `(binding [*suppress-resolution-telemetry* true]
+ ~@body))
+
+(defn effective-details
+ "Returns the connection details map appropriate for the current context.
+
+ Accepts a database (Toucan2 instance or lib/metadata). Returns nil for nil input.
+
+ By default, returns the primary `:details`. Within a [[with-write-connection]] scope,
+ takes `:write-data-details` into account (if configured). Within a
+ [[driver.w/with-swapped-connection-details]] scope, applies workspace isolation
+ overrides on top."
+ [database]
+ (when-let [database (some-> database driver.u/ensure-lib-database)]
+ (let [write? (= *connection-type* :write-data)
+ write-details (when write? (:write-data-details database))
+ base (merge (:details database) write-details)]
+ ;; Track when write-data-details are genuinely used (not fallback, not workspace-swapped).
+ ;; Default resolutions are not tracked here — see :metabase-db-connection/write-op for
+ ;; pool-level connection acquisition metrics.
+ (when (and write-details
+ (not *suppress-resolution-telemetry*)
+ (not (driver.w/has-connection-swap? (:id database))))
+ (try (prometheus/inc! :metabase-db-connection/type-resolved {:connection-type "write-data"})
+ (catch Exception _ nil)))
+ (-> (driver.w/maybe-swap-details (:id database) base)
+ (assoc ::effective-connection-type (if write-details :write-data :default))
+ (assoc ::database-id (u/id database))))))
+
+(defn details-for-exact-type
+ "Returns the details map for exactly the given connection-type, with no fallback or merging.
+
+ Unlike [[effective-details]], `:write-data` returns only `:write-data-details` (possibly nil),
+ not a merge with `:details`. Use when you need to inspect or update a specific details map
+ without resorting to raw key access."
+ [database connection-type]
+ (let [database (driver.u/ensure-lib-database database)]
+ (case connection-type
+ :default (:details database)
+ :write-data (:write-data-details database))))
+
+(defn write-connection-requested?
+ "True if currently executing within a [[with-write-connection]] scope."
+ []
+ (= *connection-type* :write-data))
+
+(defn effective-connection-type
+ "Returns the connection type actually in effect for the given database. Return value
+ matches malli schema [[::connection-type]].
+
+ Returns `:write-data` only when a write connection is both *requested*
+ ([[with-write-connection]]) and *configured* (the database has `:write-data-details`).
+ Otherwise returns `:default`, avoiding duplicate resource allocation (e.g. connection
+ pools) when the requested type would resolve identically to `:default`."
+ [database]
+ (if (and (= *connection-type* :write-data)
+ (some? (:write-data-details (driver.u/ensure-lib-database database))))
+ :write-data
+ :default))
+
+(defn track-connection-acquisition!
+ "Increments a Prometheus counter tracking connection acquisitions by connection type
+ and logs the connection type + database ID at DEBUG.
+
+ Accepts a map of connection details, expected to be the output of [[effective-details]]
+
+ Call at the point where a driver actually obtains a connection (e.g., pool checkout).
+ Non-JDBC drivers that manage their own connections should call this explicitly."
+ [connection-details]
+ (if-let [conn-type (::effective-connection-type connection-details)]
+ (do
+ (log/debugf "Acquiring %s connection for db %s" conn-type (::database-id connection-details))
+ (try (prometheus/inc! :metabase-db-connection/write-op {:connection-type (name conn-type)})
+ (catch Exception _ nil)))
+ (log/warnf "%s was unable to determine connection type" `track-connection-acquisition!)))
+
+(defn default-details
+ "Returns primary `:details`, ignoring [[*connection-type*]].
+
+ For operations that must always use the base configuration regardless of context —
+ configuration migrations, admin-initiated writes, credential caching. For normal
+ driver operations, prefer [[effective-details]]."
+ [database]
+ (:details database))
diff --git a/src/metabase/driver/connection/workspaces.clj b/src/metabase/driver/connection/workspaces.clj
new file mode 100644
index 00000000000..8f397050a3c
--- /dev/null
+++ b/src/metabase/driver/connection/workspaces.clj
@@ -0,0 +1,84 @@
+(ns metabase.driver.connection.workspaces
+ (:require
+ [metabase.util.log :as log]))
+
+(def ^:private ^:dynamic *swapped-connection-details*
+ "A dynamic var that holds a map of database-id -> swapped-details-map for temporarily swapping connection details.
+ When a connection spec is created for a database, if its ID is present in this map, the swap map will be
+ merged into the connection `:details` before they are used to create a connection.
+
+ This provides a mechanism for temporarily using different connection details (e.g., using alternative credentials
+ for workspaces) without mutating the database record.
+
+ The swap map is merged into the database `:details` map. The swap is applied before any connection-specific
+ processing (like hash calculation for connection pooling), so different swaps will result in different
+ connection pools.
+
+ Different drivers may apply this swap at different points in their connection lifecycle, but the semantics
+ are consistent: swapped details are used for the duration of the dynamic scope.
+
+ See [[with-swapped-connection-details]] for usage."
+ nil)
+
+(defn- apply-detail-swaps
+ "Merges the `swap-map` into `details`. Supports nested maps via deep merge."
+ [details swap-map]
+ (reduce-kv
+ (fn [acc k v]
+ (if (and (map? v) (map? (get acc k)))
+ (assoc acc k (apply-detail-swaps (get acc k) v))
+ (assoc acc k v)))
+ details
+ swap-map))
+
+(defn has-connection-swap?
+ "Returns true if there is an active connection detail swap for `database-id`."
+ [database-id]
+ (contains? *swapped-connection-details* database-id))
+
+(defn maybe-swap-details
+ "Returns the database details with any swaps applied from [[*swapped-connection-details*]].
+ If no swap exists for `database-id`, returns `details` unchanged.
+
+ Drivers should call this function when creating connections to apply any active swaps.
+ For JDBC drivers, this is called in [[metabase.driver.sql-jdbc.connection/db->pooled-connection-spec]].
+ For other drivers (e.g., MongoDB), this should be called in their connection creation logic."
+ [database-id details]
+ (if-let [swap-map (get *swapped-connection-details* database-id)]
+ (do
+ (log/debugf "Applying swapped connection details for database %d, swap keys: %s"
+ database-id (keys swap-map))
+ (apply-detail-swaps details swap-map))
+ details))
+
+(defn do-with-swapped-connection-details
+ "Implementation for [[with-swapped-connection-details]]."
+ [database-id swap-map thunk]
+ (when (contains? *swapped-connection-details* database-id)
+ (throw (ex-info "Nested connection detail swaps are not supported for the same database"
+ {:database-id database-id})))
+ (log/debugf "Entering swapped connection details scope for database %d, swap keys: %s"
+ database-id (keys swap-map))
+ (binding [*swapped-connection-details* (assoc *swapped-connection-details* database-id swap-map)]
+ (thunk)))
+
+(defmacro with-swapped-connection-details
+ "Temporarily swap the connection details for a specific database within the dynamic scope of `body`.
+
+ The `swap-map` is a map of detail keys to swap values. These will be merged into the database's
+ connection `:details` map. Nested maps are deep-merged.
+
+ Any code that creates a connection for `database-id` within this scope will use the modified details.
+
+ **Important:** Nested swaps for the same database are not supported and will throw an exception.
+ Different databases can have concurrent swaps.
+
+ Example:
+
+ ;; Swap connection to use alternate credentials
+ (driver/with-swapped-connection-details 1 {:user \"workspace-user\" :password \"workspace-pass\"}
+ ;; All connections created in this scope use the swapped credentials
+ (qp/process-query query))"
+ {:style/indent 2}
+ [database-id swap-map & body]
+ `(do-with-swapped-connection-details ~database-id ~swap-map (fn [] ~@body)))
diff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.clj
index 08ea620ca1f..3c1fba0c5c8 100644
--- a/src/metabase/driver/h2.clj
+++ b/src/metabase/driver/h2.clj
@@ -1,5 +1,5 @@
(ns metabase.driver.h2
- (:refer-clojure :exclude [some every? get-in])
+ (:refer-clojure :exclude [some every?])
(:require
[clojure.java.jdbc :as jdbc]
[clojure.math.combinatorics :as math.combo]
@@ -9,6 +9,7 @@
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
[metabase.driver.common :as driver.common]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.h2.actions :as h2.actions]
[metabase.driver.settings :as driver.settings]
[metabase.driver.sql :as sql]
@@ -28,7 +29,7 @@
[metabase.util.i18n :refer [deferred-tru tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [every? get-in some]])
+ [metabase.util.performance :refer [every? some]])
(:import
(java.sql Clob Connection ResultSet ResultSetMetaData SQLException Statement)
(java.time OffsetTime)
@@ -172,7 +173,7 @@
;; connection string. We don't allow SQL execution on H2 databases for the default admin account for security
;; reasons
(when (= (keyword query-type) :native)
- (let [{:keys [details]} (driver-api/database (driver-api/metadata-provider))
+ (let [details (-> (driver-api/metadata-provider) driver-api/database driver.conn/effective-details)
user (db-details->user details)]
(when (and config/is-prod? ;; we elevated permissions in workspace tests
(or (str/blank? user)
@@ -705,7 +706,7 @@
username (driver.u/workspace-isolation-user-name workspace)
password (driver.u/random-workspace-password)
;; H2 embeds credentials in the :db connection string, so we need to build a new one
- original-db (get-in database [:details :db])
+ original-db (:db (driver.conn/effective-details database))
new-db (replace-credentials original-db username password)]
(jdbc/with-db-transaction [t-conn (sql-jdbc.conn/db->pooled-connection-spec (:id database))]
(with-open [^Statement stmt (.createStatement ^Connection (:connection t-conn))]
diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.clj
index b0a69e73c15..418a6815fed 100644
--- a/src/metabase/driver/mysql.clj
+++ b/src/metabase/driver/mysql.clj
@@ -12,6 +12,7 @@
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
[metabase.driver.common :as driver.common]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.mysql.actions :as mysql.actions]
[metabase.driver.mysql.ddl :as mysql.ddl]
[metabase.driver.sql :as driver.sql]
@@ -1187,7 +1188,7 @@
(defmethod driver.sql/default-database-role :mysql
[_driver database]
- (-> database :details :role))
+ (-> database driver.conn/effective-details :role))
(defmethod driver.sql/set-role-statement :mysql
[_driver role]
@@ -1209,7 +1210,7 @@
(let [db (try
(driver-api/database (driver-api/metadata-provider))
(catch Throwable _ nil))
- tiny-int-1-is-bit? (not (some-> db :details :additional-options (str/includes? "tinyInt1isBit=false")))
+ tiny-int-1-is-bit? (not (some-> db driver.conn/effective-details :additional-options (str/includes? "tinyInt1isBit=false")))
db-type-name (.getColumnTypeName rsmeta column-index)
precision (try
(.getPrecision rsmeta column-index)
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index 66ec7de2e0c..49159e0fa31 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -121,7 +121,7 @@
(defn- create-table-and-insert-data!
[driver transform-details conn-spec]
- (let [create-query (driver/compile-transform driver transform-details)
+ (let [create-query (driver/compile-transform driver transform-details)
rows-affected (last (driver/execute-raw-queries! driver conn-spec [create-query]))]
rows-affected))
@@ -131,9 +131,9 @@
old-temp (driver.u/temp-table-name driver output-table)]
(try
(let [new-temp-details (assoc transform-details :output-table new-temp)
- rows-affected (create-table-and-insert-data! driver new-temp-details conn-spec)]
+ rows-affected (create-table-and-insert-data! driver new-temp-details conn-spec)]
(driver/rename-tables! driver (:id database) {output-table old-temp
- new-temp output-table})
+ new-temp output-table})
(driver/drop-table! driver (:id database) old-temp)
{:rows-affected rows-affected})
(catch Exception e
@@ -146,7 +146,7 @@
(let [tmp-table (driver.u/temp-table-name driver output-table)]
(try
(let [tmp-table-details (assoc transform-details :output-table tmp-table)
- rows-affected (create-table-and-insert-data! driver tmp-table-details conn-spec)]
+ rows-affected (create-table-and-insert-data! driver tmp-table-details conn-spec)]
(driver/drop-table! driver (:id database) output-table)
(driver/rename-table! driver (:id database) tmp-table output-table)
{:rows-affected rows-affected})
@@ -169,7 +169,7 @@
[driver {:keys [conn-spec output-table database] :as transform-details} _opts]
(let [table-exists? (driver/table-exists? driver database
{:schema (namespace output-table)
- :name (name output-table)})]
+ :name (name output-table)})]
(cond
(or (not table-exists?)
(driver/database-supports? driver :create-or-replace-table database))
diff --git a/src/metabase/driver/sql_jdbc/connection.clj b/src/metabase/driver/sql_jdbc/connection.clj
index 27a2432f9a4..119c4e772b7 100644
--- a/src/metabase/driver/sql_jdbc/connection.clj
+++ b/src/metabase/driver/sql_jdbc/connection.clj
@@ -6,6 +6,8 @@
[clojure.java.jdbc :as jdbc]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
+ [metabase.driver.connection :as driver.conn]
+ [metabase.driver.connection.workspaces :as driver.w]
[metabase.driver.settings :as driver.settings]
[metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh]
[metabase.driver.util :as driver.u]
@@ -90,19 +92,19 @@
[driver database]
{;; only fetch one new connection at a time, rather than batching fetches (default = 3 at a time). This is done in
;; interest of minimizing memory consumption
- "acquireIncrement" 1
+ "acquireIncrement" 1
;; Never retry instead of the default of retrying 30 times (#51176)
;; While a couple queries may fail during a reboot, this should allow quicker recovery and less spinning on outdated
;; credentials
;; However, keep 1 retry for the tests to reduce flakiness.
- "acquireRetryAttempts" (if driver-api/is-test? 1 0)
+ "acquireRetryAttempts" (if driver-api/is-test? 1 0)
;; [From dox] Seconds a Connection can remain pooled but unused before being discarded.
- "maxIdleTime" (* 3 60 60) ; 3 hours
+ "maxIdleTime" (* 3 60 60) ; 3 hours
;; In the case of serverless databases, we don't want to periodically
;; wake them up to keep a connection open (#58373).
- "minPoolSize" 0
- "initialPoolSize" 0
- "maxPoolSize" (driver.settings/jdbc-data-warehouse-max-connection-pool-size)
+ "minPoolSize" 0
+ "initialPoolSize" 0
+ "maxPoolSize" (driver.settings/jdbc-data-warehouse-max-connection-pool-size)
;; [From dox] If true, an operation will be performed at every connection checkout to verify that the connection is
;; valid. [...] ;; Testing Connections in checkout is the simplest and most reliable form of Connection testing,
;; but for better performance, consider verifying connections periodically using `idleConnectionTestPeriod`. [...]
@@ -118,7 +120,7 @@
;; request. IRL the Metabase server and data warehouse are likely to be located in closer geographical proximity to
;; one another than my trans-contintental tests. Thus in the majority of cases the overhead should be next to
;; nothing, and in the worst case close to imperceptible.
- "testConnectionOnCheckout" true
+ "testConnectionOnCheckout" true
;; [From dox] Number of seconds that Connections in excess of minPoolSize should be permitted to remain idle in the
;; pool before being culled. Intended for applications that wish to aggressively minimize the number of open
;; Connections, shrinking the pool back towards minPoolSize if, following a spike, the load level diminishes and
@@ -126,7 +128,7 @@
;; if the parameter is to have any effect.
;;
;; Kill idle connections above the minPoolSize after 5 minutes.
- "maxIdleTimeExcessConnections" (* 5 60)
+ "maxIdleTimeExcessConnections" (* 5 60)
;; [From dox] Seconds. If set, if an application checks out but then fails to check-in [i.e. close()] a Connection
;; within the specified period of time, the pool will unceremoniously destroy() the Connection. This permits
;; applications with occasional Connection leaks to survive, rather than eventually exhausting the Connection
@@ -137,7 +139,7 @@
;; This should be the same as the query timeout. This theoretically shouldn't happen since the QP should kill
;; things after a certain timeout but it's better to be safe than sorry -- it seems like in practice some
;; connections disappear into the ether
- "unreturnedConnectionTimeout" (driver.settings/jdbc-data-warehouse-unreturned-connection-timeout-seconds)
+ "unreturnedConnectionTimeout" (driver.settings/jdbc-data-warehouse-unreturned-connection-timeout-seconds)
;; [From dox] If true, and if unreturnedConnectionTimeout is set to a positive value, then the pool will capture
;; the stack trace (via an Exception) of all Connection checkouts, and the stack traces will be printed when
;; unreturned checked-out Connections timeout. This is intended to debug applications with Connection leaks, that
@@ -159,10 +161,10 @@
"see stacktraces in the logs.")))
;; Set the data source name so that the c3p0 JMX bean has a useful identifier, which incorporates the DB ID, driver,
;; and name from the details
- "dataSourceName" (format "db-%d-%s-%s"
- (u/the-id database)
- (name driver)
- (data-source-name driver (:details database)))})
+ "dataSourceName" (format "db-%d-%s-%s"
+ (u/the-id database)
+ (name driver)
+ (data-source-name driver (driver.conn/effective-details database)))})
(defn- connection-pool-spec
"Like [[connection-pool/connection-pool-spec]] but also handles situations when the unpooled spec is a `:datasource`."
@@ -171,7 +173,7 @@
{:datasource (DataSources/pooledDataSource datasource (driver-api/map->properties pool-properties))}
(driver-api/connection-pool-spec spec pool-properties)))
-(defn ^:private default-ssh-tunnel-target-port [driver]
+(defn ^:private default-ssh-tunnel-target-port [driver]
(when-let [port-info (some
#(when (= "port" (:name %)) %)
(driver/connection-properties driver))]
@@ -184,11 +186,15 @@
(select-keys spec-or-details [:tunnel-enabled :tunnel-session :tunnel-tracker :tunnel-entrance-port :tunnel-entrance-host]))
(defn- create-pool!
- "Create a new C3P0 `ComboPooledDataSource` for connecting to the given `database`."
- [{:keys [id details], driver :engine, :as database}]
+ "Create a new C3P0 `ComboPooledDataSource` for connecting to the given `database`.
+ Uses [[driver.conn/effective-details]] to select the appropriate connection details
+ based on the current [[driver.conn/*connection-type*]]."
+ [{:keys [id], driver :engine, :as database}]
{:pre [(map? database)]}
- (log/debug (u/format-color :cyan "Creating new connection pool for %s database %s ..." driver id))
- (let [details-with-tunnel (driver/incorporate-ssh-tunnel-details ;; If the tunnel is disabled this returned unchanged
+ (log/debug (u/format-color :cyan "Creating new connection pool for %s database %s (connection-type: %s) ..."
+ driver id driver.conn/*connection-type*))
+ (let [details (driver.conn/effective-details database)
+ details-with-tunnel (driver/incorporate-ssh-tunnel-details ;; If the tunnel is disabled this returned unchanged
driver
(update details :port #(or % (default-ssh-tunnel-target-port driver))))
details-with-auth (driver.u/fetch-and-incorporate-auth-provider-details
@@ -216,8 +222,8 @@
15)
(defonce ^:private ^{:doc "A map of our currently open connection pools for canonical (non-swapped) connections,
- keyed by database-id. Each database has at most one canonical pool."}
- database-id->connection-pool
+ keyed by pool cache key `[database-id, connection-type]`. Each database+connection-type pair has at most one canonical pool."}
+ pool-cache-key->connection-pool
(atom {}))
(defonce ^:private ^Cache ^{:doc "A Guava cache of swapped connection pools with TTL-based expiration,
@@ -237,53 +243,70 @@
nil))))
(build)))
-(defonce ^:private ^{:doc "A map of DB details hash values for the canonical (non-swapped) details, keyed by Database `:id`.
+(defonce ^:private ^{:doc "A map of DB details hash values for the canonical (non-swapped) details,
+ keyed by pool cache key `[database-id, connection-type]`.
This is used to detect when database details have been updated in the application database."}
- database-id->jdbc-spec-hash
+ pool-cache-key->jdbc-spec-hash
(atom {}))
+(defn- pool-cache-key
+ "Returns the cache key for connection pools: `[database-id, connection-type]`.
+ Uses [[driver.conn/effective-connection-type]] so that a requested write connection
+ without configured `:write-data-details` resolves to `:default`, reusing the
+ existing pool instead of creating a duplicate."
+ [database]
+ [(u/the-id database) (driver.conn/effective-connection-type database)])
+
(mu/defn- jdbc-spec-hash
- "Computes a hash value for the JDBC connection spec based on `database`'s `:details` map, for the purpose of
- determining if details changed and therefore the existing connection pool needs to be invalidated."
- [{driver :engine, :keys [details], :as database} :- [:maybe :map]]
+ "Computes a hash value for the JDBC connection spec based on the effective connection details, for the purpose of
+ determining if details changed and therefore the existing connection pool needs to be invalidated.
+ Uses [[driver.conn/effective-details]] to select the appropriate details based on [[driver.conn/*connection-type*]]."
+ [{driver :engine, :as database} :- [:maybe :map]]
(when (some? database)
- (hash (connection-details->spec driver details))))
+ (hash (connection-details->spec driver (driver.conn/effective-details database)))))
(defn- set-canonical-pool!
- "Atomically update the canonical connection pool for a database.
+ "Atomically update the canonical connection pool for a database and connection type.
Use this function instead of modifying connection pool atoms directly because it properly closes down old pools in a
- thread-safe way, ensuring no more than one canonical pool is ever open for a specific database.
+ thread-safe way, ensuring no more than one canonical pool is ever open for a specific database and connection type.
This function is only for canonical (non-swapped) pools. Swapped pools are managed by the Guava cache."
- [database-id details-hash pool-spec-or-nil]
- {:pre [(integer? database-id) (some? details-hash)]}
- (let [[old-pool-map] (if pool-spec-or-nil
- (swap-vals! database-id->connection-pool assoc database-id pool-spec-or-nil)
- (swap-vals! database-id->connection-pool dissoc database-id))]
+ [cache-key details-hash pool-spec-or-nil]
+ {:pre [(vector? cache-key) (some? details-hash)]}
+ (let [[database-id _connection-type] cache-key
+ [old-pool-map] (if pool-spec-or-nil
+ (swap-vals! pool-cache-key->connection-pool assoc cache-key pool-spec-or-nil)
+ (swap-vals! pool-cache-key->connection-pool dissoc cache-key))]
;; if we replaced a different pool with the new pool that is different from the old one, destroy the old pool
- (when-let [old-pool-spec (get old-pool-map database-id)]
+ (when-let [old-pool-spec (get old-pool-map cache-key)]
(when-not (identical? old-pool-spec pool-spec-or-nil)
(destroy-pool! database-id old-pool-spec))))
;; Update canonical hash cache
- (swap! database-id->jdbc-spec-hash assoc database-id details-hash)
+ (swap! pool-cache-key->jdbc-spec-hash assoc cache-key details-hash)
nil)
(defn invalidate-pool-for-db!
- "Invalidates all connection pools for the given database (canonical and swapped) by closing them and removing from cache."
+ "Invalidates all connection pools for the given database (canonical -- all connection types -- and swapped) by closing
+ them and removing from cache."
[database]
- (let [db-id (u/the-id database)
- has-canonical? (contains? @database-id->connection-pool db-id)
+ (let [db-id (u/the-id database)
+ canonical-keys [[db-id :default] [db-id :write-data]]
+ pool-map @pool-cache-key->connection-pool
+ canonical-count (count (filter pool-map canonical-keys))
;; Find all swapped pool keys for this database (keys are [db-id, details-hash] tuples)
- swapped-keys (filter (fn [[cached-db-id _details-hash]]
- (= cached-db-id db-id))
- (keys (.asMap ^Cache swapped-connection-pools)))]
- (log/debugf "Invalidating connection pools for database %d (canonical: %s, swapped count: %d)"
- db-id has-canonical? (count swapped-keys))
- ;; Clear canonical pool
- (when-let [pool-spec (get @database-id->connection-pool db-id)]
+ swapped-keys (filter (fn [[cached-db-id _details-hash]]
+ (= cached-db-id db-id))
+ (keys (.asMap ^Cache swapped-connection-pools)))]
+ (log/debugf "Invalidating connection pools for database %d (canonical count: %d, swapped count: %d)"
+ db-id canonical-count (count swapped-keys))
+ ;; Clear canonical pools for both connection types
+ (doseq [cache-key canonical-keys
+ :let [[old-map] (swap-vals! pool-cache-key->connection-pool dissoc cache-key)
+ pool-spec (get old-map cache-key)]
+ :when pool-spec]
(destroy-pool! db-id pool-spec)
- (swap! database-id->connection-pool dissoc db-id))
+ (swap! pool-cache-key->jdbc-spec-hash dissoc cache-key))
;; Clear all swapped pools for this DB (removal listener will call destroy-pool!)
(doseq [cache-key swapped-keys]
(.invalidate ^Cache swapped-connection-pools cache-key))))
@@ -381,26 +404,30 @@
(defn- canonical-pool-hash-changed?
"Check if the canonical pool's hash differs from the expected hash.
- Handles stale DatabaseInstance by re-fetching from app DB."
- [database-id expected-hash]
- (let [curr-hash (get @database-id->jdbc-spec-hash database-id)]
+ Handles stale DatabaseInstance by re-fetching from app DB.
+ `cache-key` is a `[database-id, connection-type]` tuple."
+ [cache-key expected-hash]
+ (let [database-id (first cache-key)
+ curr-hash (get @pool-cache-key->jdbc-spec-hash cache-key)]
(when (and (some? curr-hash) (not= curr-hash expected-hash))
;; the hash didn't match, but it's possible that a stale instance of `DatabaseInstance`
;; was passed in (ex: from a long-running sync operation); fetch the latest one from
;; our app DB, and see if it STILL doesn't match
- (not= curr-hash (-> (t2/select-one [:model/Database :id :engine :details] :id database-id)
+ (not= curr-hash (-> (t2/select-one [:model/Database :id :engine :details :write_data_details] :id database-id)
jdbc-spec-hash)))))
(defn- get-canonical-pool
- "Get a canonical pool if it exists and is valid, otherwise return nil."
- [database-id details-hash log-invalidation?]
- (let [pool-spec (get @database-id->connection-pool database-id ::not-found)]
+ "Get a canonical pool if it exists and is valid, otherwise return nil.
+ `cache-key` is a `[database-id, connection-type]` tuple."
+ [cache-key details-hash log-invalidation?]
+ (let [database-id (first cache-key)
+ pool-spec (get @pool-cache-key->connection-pool cache-key ::not-found)]
(cond
(= ::not-found pool-spec)
nil
;; Check if the hash has changed (details were updated in DB)
- (canonical-pool-hash-changed? database-id details-hash)
+ (canonical-pool-hash-changed? cache-key details-hash)
(do
(when log-invalidation?
(log-pool-invalidation! database-id :hash-changed))
@@ -412,7 +439,8 @@
(defn db->pooled-connection-spec
"Return a JDBC connection spec that includes a c3p0 `ComboPooledDataSource`. These connection pools are cached so we
- don't create multiple ones for the same DB.
+ don't create multiple ones for the same DB and connection type. The connection type is determined by
+ [[driver.conn/*connection-type*]] - use [[driver.conn/with-write-connection]] to get a write connection pool.
When [[metabase.driver/with-swapped-connection-details]] is active for a database, the database details are
modified before creating the connection pool. Swapped pools are stored in a separate Guava cache with TTL-based
@@ -425,21 +453,23 @@
(u/id db-or-id-or-spec)
(let [database-id (u/the-id db-or-id-or-spec)
;; we need the Database instance no matter what (in order to calculate details hash)
- db-original (or (when (driver-api/instance-of? :model/Database db-or-id-or-spec)
+ db (or (when (driver-api/instance-of? :model/Database db-or-id-or-spec)
(driver-api/instance->metadata db-or-id-or-spec :metadata/database))
(when (= (:lib/type db-or-id-or-spec) :metadata/database)
db-or-id-or-spec)
(driver-api/with-metadata-provider database-id
(driver-api/database (driver-api/metadata-provider))))
- ;; Apply connection detail swaps if present
- has-swap? (driver/has-connection-swap? database-id)
- db (update db-original :details #(driver/maybe-swap-details database-id %))
- ;; Calculate hash from final (possibly swapped) details
+ cache-key (pool-cache-key db)
+ ;; Check for workspace detail swaps (for pool routing: canonical atom vs Guava TTL cache).
+ ;; The actual swap is applied inside effective-details, not here.
+ has-swap? (driver.w/has-connection-swap? database-id)
+ ;; Calculate hash from effective details (includes write-connection merge + workspace swap)
details-hash (jdbc-spec-hash db)]
+ (driver.conn/track-connection-acquisition! (driver.conn/effective-details db))
(cond
;; for the audit db, we pass the datasource for the app-db. This lets us use fewer db
;; connections with *application-db* and 1 less connection pool. Note: This data-source is
- ;; not in [[database-id->connection-pool]].
+ ;; not in [[pool-cache-key->connection-pool]].
(or (:is-audit db) (get-in db [:details :is-audit-dev]))
{:datasource (driver-api/data-source)}
@@ -451,17 +481,17 @@
:else
(or
;; we have an existing valid pool for this database, so use it
- (get-canonical-pool database-id details-hash true)
+ (get-canonical-pool cache-key details-hash true)
;; We don't want to end up with a bunch of simultaneous threads creating pools only to have them destroyed
;; the very next instant. This will cause their queries to fail. Thus we should do the usual locking here
;; and make sure only one thread will be creating a pool at a given instant.
- (locking database-id->connection-pool
+ (locking pool-cache-key->connection-pool
(or
;; check if another thread created the pool while we were waiting to acquire the lock
- (get-canonical-pool database-id details-hash false)
+ (get-canonical-pool cache-key details-hash false)
;; create a new pool and add it to our cache, then return it
(u/prog1 (create-pool! db)
- (set-canonical-pool! database-id details-hash <>)))))))
+ (set-canonical-pool! cache-key details-hash <>)))))))
;; already a `clojure.java.jdbc` spec map
(map? db-or-id-or-spec)
diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.clj
index c39a0e003b2..d535e7fd2d9 100644
--- a/src/metabase/driver/sql_jdbc/execute.clj
+++ b/src/metabase/driver/sql_jdbc/execute.clj
@@ -14,6 +14,7 @@
[medley.core :as m]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.settings :as driver.settings]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute.diagnostic :as sql-jdbc.execute.diagnostic]
@@ -773,7 +774,7 @@
[driver {{sql :query, params :params} :native, :as outer-query} _context respond]
{:pre [(string? sql) (seq sql)]}
(let [database (driver-api/database (driver-api/metadata-provider))
- sql (if (get-in database [:details :include-user-id-and-hash] true)
+ sql (if (get-in (driver.conn/effective-details database) [:include-user-id-and-hash] true)
(->> (driver-api/query->remark driver outer-query)
(inject-remark driver sql))
sql)
diff --git a/src/metabase/driver/sql_jdbc/sync/describe_table.clj b/src/metabase/driver/sql_jdbc/sync/describe_table.clj
index 9225400f05c..12b4dd89f2a 100644
--- a/src/metabase/driver/sql_jdbc/sync/describe_table.clj
+++ b/src/metabase/driver/sql_jdbc/sync/describe_table.clj
@@ -10,6 +10,7 @@
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
[metabase.driver.common.table-rows-sample :as table-rows-sample]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.settings :as driver.settings]
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
@@ -370,7 +371,7 @@
(if (or (and schema-names (empty? schema-names))
(and table-names (empty? table-names)))
[]
- (let [sql (describe-fields-sql driver (assoc args :details (:details db)))]
+ (let [sql (describe-fields-sql driver (assoc args :details (driver.conn/effective-details db)))]
(try
(log/debugf "`describe-fields` sql query:\n```\n%s\n```\n`describe-fields` args:\n```\n%s\n```"
(driver/prettify-native-form driver (first sql))
@@ -400,7 +401,7 @@
[]
(eduction
(map (fn [col] (select-keys col [:table-schema :table-name :field-name])))
- (sql-jdbc.execute/reducible-query db (describe-indexes-sql driver (assoc args :details (:details db)))))))
+ (sql-jdbc.execute/reducible-query db (describe-indexes-sql driver (assoc args :details (driver.conn/effective-details db)))))))
(defn- describe-table-fks*
[_driver ^Connection conn {^String schema :schema, ^String table-name :name} & [^String db-name-or-nil]]
@@ -438,7 +439,7 @@
(if (or (and schema-names (empty? schema-names))
(and table-names (empty? table-names)))
[]
- (sql-jdbc.execute/reducible-query db (describe-fks-sql driver (assoc args :details (:details db))))))
+ (sql-jdbc.execute/reducible-query db (describe-fks-sql driver (assoc args :details (driver.conn/effective-details db))))))
(defn describe-table-indexes
"Default implementation of [[metabase.driver/describe-table-indexes]] for SQL JDBC drivers. Uses JDBC DatabaseMetaData."
diff --git a/src/metabase/driver/sync.clj b/src/metabase/driver/sync.clj
index 271d34ce58d..d2ef36e2386 100644
--- a/src/metabase/driver/sync.clj
+++ b/src/metabase/driver/sync.clj
@@ -2,6 +2,7 @@
"General functions and utilities for sync operations across multiple drivers."
(:require
[clojure.string :as str]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.util :as driver.u])
(:import
(java.util.regex Pattern)))
@@ -60,9 +61,10 @@
([database]
(let [{prop-name :name} (driver.u/find-schema-filters-prop (driver.u/database->driver database))]
(db-details->schema-filter-patterns prop-name database)))
- ([prop-nm {db-details :details :as _database}]
+ ([prop-nm database]
(when prop-nm
- (let [schema-filter-type (get db-details (keyword (str prop-nm "-type")))
+ (let [db-details (driver.conn/effective-details database)
+ schema-filter-type (get db-details (keyword (str prop-nm "-type")))
schema-filter-patterns (get db-details (keyword (str prop-nm "-patterns")))]
(case schema-filter-type
"exclusion" [nil schema-filter-patterns]
diff --git a/src/metabase/driver/util.clj b/src/metabase/driver/util.clj
index d621710d5a9..6a2dcc1a6e1 100644
--- a/src/metabase/driver/util.clj
+++ b/src/metabase/driver/util.clj
@@ -223,8 +223,10 @@
[driver feature (mdb/unique-identifier) (:id database) (:updated-at database)])))))
;;; this can get called in post-select which doesn't always have ID
-(mu/defn- ensure-lib-database :- [:map
- [:lib/type [:= :metadata/database]]]
+(mu/defn ensure-lib-database :- [:map
+ [:lib/type [:= :metadata/database]]]
+ "Ensures the database is in MLv2 metadata format (SnakeHatingMap with kebab-case keys).
+ If passed a Toucan2 instance, converts it. If already MLv2 metadata, returns as-is."
[database :- [:or
[:map
[:lib/type [:= :metadata/database]]]
@@ -729,6 +731,21 @@
(into default-sensitive-fields (map (comp keyword :name) password-fields)))
default-sensitive-fields))
+(defn fields-hidden-for-write-data-connection
+ "Returns the set of field names (strings) that should NOT appear in `write_data_details` for the given `driver`.
+ These are fields whose resolved `visible-if` includes `\"write-data-connection\" false`, meaning they are hidden
+ when the write-data-connection form marker is true."
+ [driver]
+ (when-some [conn-prop-fn (get-method driver/connection-properties driver)]
+ (let [all-props (conn-prop-fn driver)
+ resolved (connection-props-server->client driver all-props)
+ props-by-name (collect-all-props-by-name resolved)]
+ (into #{}
+ (keep (fn [[field-name {:keys [visible-if]}]]
+ (when (false? (get visible-if "write-data-connection"))
+ field-name)))
+ props-by-name))))
+
(defn fetch-and-incorporate-auth-provider-details
"Incorporates auth-provider responses with db-details.
diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.clj
index fb816492fbe..1e4182d15bd 100644
--- a/src/metabase/lib_be/metadata/jvm.clj
+++ b/src/metabase/lib_be/metadata/jvm.clj
@@ -79,7 +79,7 @@
#_resolved-query clojure.lang.IPersistentMap]
[query-type model parsed-args honeysql]
(merge (next-method query-type model parsed-args honeysql)
- {:select [:id :engine :name :dbms_version :settings :is_audit :details :timezone :router_database_id]}))
+ {:select [:id :engine :name :dbms_version :settings :is_audit :details :write_data_details :timezone :router_database_id]}))
(t2/define-after-select :metadata/database
[database]
diff --git a/src/metabase/secrets/models/secret.clj b/src/metabase/secrets/models/secret.clj
index 89fcdf2bc29..790d11c6e66 100644
--- a/src/metabase/secrets/models/secret.clj
+++ b/src/metabase/secrets/models/secret.clj
@@ -277,18 +277,19 @@
(defn delete-orphaned-secrets!
"Delete Secret instances from the app DB, that will become orphaned when `database` is deleted. For now, this will
- simply delete any Secret whose ID appears in the details blob, since every Secret instance that is currently created
- is exclusively associated with a single Database.
+ simply delete any Secret whose ID appears in the details blobs (both `:details` and `:write_data_details`), since
+ every Secret instance that is currently created is exclusively associated with a single Database.
In the future, if/when we allow arbitrary association of secret instances to database instances, this will need to
change and become more complicated (likely by consulting a many-to-many join table)."
- [{:keys [id details] :as database}]
+ [{:keys [id details write_data_details] :as database}]
(when-let [possible-secret-prop-names (seq (keys (secret-conn-props-by-name (driver.u/database->driver database))))]
(doseq [secret-id (reduce (fn [acc prop-name]
- (if-let [secret-id (get details (->id-kw prop-name))]
- (conj acc secret-id)
- acc))
- []
+ (let [id-kw (->id-kw prop-name)]
+ (into acc
+ (keep #(get % id-kw))
+ [details write_data_details])))
+ #{}
possible-secret-prop-names)]
(log/infof "Deleting secret ID %s from app DB because the owning database (%s) is being deleted" secret-id id)
(t2/delete! :model/Secret :id secret-id))))
@@ -325,17 +326,16 @@
"To satisfy clients we need to return the keys they send us in details.
This is a transformation on `:model/Database` `to-json`
- Fetches the stored secret and fills in `-path` `-options` `-value` for each secret property"
+ Fetches the stored secret and fills in `-path` `-options` `-value` for each secret property.
+ Operates on both `:details` and `:write_data_details`."
[database]
- (let [driver (driver.u/database->driver database)]
- (m/update-existing
- database
- :details
- (fn [details]
- (reduce-over-details-secret-values
- driver
- details
- hydrate-redacted-secret)))))
+ (let [driver (driver.u/database->driver database)
+ hydrate (fn [details]
+ (reduce-over-details-secret-values driver details hydrate-redacted-secret))]
+ ;; Very low-level operation here, so not using driver.conn/* utils:
+ (-> database
+ (m/update-existing :details hydrate)
+ (m/update-existing :write_data_details hydrate))))
(defn clean-secret-properties-from-details
"Ensures that all possible secret property values are removed from `:details`.
@@ -350,21 +350,52 @@
(apply dissoc db-details (vals (->possible-secret-property-names conn-prop-nm))))))
(defn clean-secret-properties-from-database
- "Ensures that all possible secret property values are removed from `:details`.
+ "Ensures that all possible secret property values are removed from `:details` and `:write_data_details`.
This is a transformation on `:model/Database` `results-transform`."
[database]
- (m/update-existing
- database
- :details
- #(clean-secret-properties-from-details
- %
- (driver.u/database->driver database))))
+ (let [clean #(clean-secret-properties-from-details % (driver.u/database->driver database))]
+ ;; Very low-level operation here, so not using driver.conn/* utils:
+ (-> database
+ (m/update-existing :details clean)
+ (m/update-existing :write_data_details clean))))
+
+(defn- handle-secrets-for-details-key
+ "Process secret-type connection properties in `details-key` of `database`, converting raw secret
+ values into Secret records and storing only the secret ID. Returns the updated database."
+ [database details-key]
+ (if-let [details (get database details-key)]
+ (let [original-details (get (t2/original database) details-key)
+ updated-details (reduce-over-details-secret-values
+ (driver.u/database->driver database)
+ details
+ (fn [db-details conn-prop-nm conn-prop]
+ (let [kws (->possible-secret-property-names conn-prop-nm)
+ id-kw (keyword (str conn-prop-nm "-id"))
+ secret-id (get original-details id-kw)
+ secret (secret-map-from-details db-details conn-prop)
+ cleared-details (apply dissoc db-details (vals kws))]
+ (if secret
+ (if (:value secret)
+ (let [{:keys [id]} (upsert-secret-value!
+ secret-id
+ (format "%s for %s" (:display-name conn-prop) (:name database))
+ (:secret-kind conn-prop)
+ (:source secret)
+ (:value secret))]
+ (assoc cleared-details id-kw id))
+ (do
+ (t2/delete! :model/Secret :id secret-id)
+ (dissoc cleared-details id-kw)))
+ ;; Don't throw out a secret even if the client didn't send it back
+ (m/assoc-some cleared-details id-kw secret-id)))))]
+ (assoc database details-key updated-details))
+ database))
(defn handle-incoming-client-secrets!
- "Converts incoming secret values in `:details` into Secrets.
+ "Converts incoming secret values in `:details` and `:write_data_details` into Secrets.
This is a transformation on `:model/Database` `before-insert` and `before-update`.
- Only the Secret id should be stored in `:details`. All other secret props should be cleared.
+ Only the Secret id should be stored in the details maps. All other secret props should be cleared.
A secret prop like `{:type :secret, :secret-kind :pem-cert :name :private-key}` can get expanded by
[[driver.u/connection-props-server->client]] into these connection properties:
@@ -378,29 +409,7 @@
In this case, we want to look for a `:private-key-id` in the `original-details` which would indicate
that we have stored a Secret before. We upsert the secret-value (which could be `-path` or `-value`).
We clear out all the possible secret-keys `-value`, `-options`, `-path` and store the `-id`."
- [{:keys [details] :as database}]
- (let [{original-details :details} (t2/original database)
- updated-details (reduce-over-details-secret-values
- (driver.u/database->driver database)
- details
- (fn [db-details conn-prop-nm conn-prop]
- (let [kws (->possible-secret-property-names conn-prop-nm)
- id-kw (keyword (str conn-prop-nm "-id"))
- secret-id (get original-details id-kw)
- secret (secret-map-from-details db-details conn-prop)
- cleared-details (apply dissoc db-details (vals kws))]
- (if secret
- (if (:value secret)
- (let [{:keys [id]} (upsert-secret-value!
- secret-id
- (format "%s for %s" (:display-name conn-prop) (:name database))
- (:secret-kind conn-prop)
- (:source secret)
- (:value secret))]
- (assoc cleared-details id-kw id))
- (do
- (t2/delete! :model/Secret :id secret-id)
- (dissoc cleared-details id-kw)))
- ;; Don't throw out a secret even if the client didn't sent it back
- (m/assoc-some cleared-details id-kw secret-id)))))]
- (assoc database :details updated-details)))
+ [database]
+ (-> database
+ (handle-secrets-for-details-key :details)
+ (handle-secrets-for-details-key :write_data_details)))
diff --git a/src/metabase/transforms/api/transform.clj b/src/metabase/transforms/api/transform.clj
index aed523ae748..2eb6f0b8021 100644
--- a/src/metabase/transforms/api/transform.clj
+++ b/src/metabase/transforms/api/transform.clj
@@ -424,6 +424,15 @@
:limit (request/limit)))
(update :data #(map transforms.util/localize-run-timestamps %))))
+(api.macros/defendpoint :get "/run/:run-id" :- TransformRunResponse
+ "Get a transform run by ID."
+ [{:keys [run-id]} :- [:map
+ [:run-id ms/PositiveInt]]]
+ (api/check-data-analyst)
+ (let [run (api/check-404 (t2/select-one :model/TransformRun :id run-id))]
+ (-> (t2/hydrate run [:transform :collection :transform_tag_ids])
+ transforms.util/localize-run-timestamps)))
+
(defn update-transform!
"Update a transform. Validates features, database support, cycles, and target conflicts.
Returns the updated transform with hydrated associations."
diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.clj
index ebd2b65b457..e53845b5814 100644
--- a/src/metabase/transforms/query_impl.clj
+++ b/src/metabase/transforms/query_impl.clj
@@ -2,6 +2,7 @@
(:require
[metabase.database-routing.core :as database-routing]
[metabase.driver :as driver]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.util :as driver.u]
[metabase.lib.schema.common :as schema.common]
[metabase.query-processor.compile :as qp.compile]
@@ -54,40 +55,45 @@
([transform] (run-mbql-transform! transform nil))
([{:keys [id source target owner_user_id creator_id] :as transform} {:keys [run-method start-promise user-id]}]
(try
- (let [db (get-in source [:query :database])
+ (let [db (get-in source [:query :database])
{driver :engine :as database} (t2/select-one :model/Database db)
;; important to test routing before calling compile-source (whose qp middleware will also throw)
- _ (throw-if-db-routing-enabled transform driver database)
- transform-details {:db-id db
- :database database
- :transform-id id
- :transform-type (keyword (:type target))
- :conn-spec (driver/connection-spec driver database)
- :query (transforms.util/compile-source transform)
- :output-schema (:schema target)
- :output-table (transforms.util/qualified-table-name driver target)}
- opts (transform-opts transform-details)
- features (transforms.util/required-database-features transform)
- ;; For manual runs, use the triggering user; for cron, use owner/creator
- run-user-id (if (and (= run-method :manual) user-id)
- user-id
- (or owner_user_id creator_id))]
- (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
- (throw (ex-info "The database does not support the requested transform target type."
- {:driver driver, :database database, :features features})))
- ;; mark the execution as started and notify any observers
- (let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method run-user-id)]
- (when start-promise
- (deliver start-promise [:started run-id]))
- (log/info "Executing transform" id "with target" (pr-str target))
- (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
- (transforms.util/run-cancelable-transform!
- run-id driver transform-details
- (fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))
- (transforms.util/handle-transform-complete!
- :run-id run-id
- :transform transform
- :db database)))
+ _ (throw-if-db-routing-enabled transform driver database)]
+ (driver.conn/with-write-connection
+ (let [conn-spec (driver/connection-spec driver database)
+ transform-details {:db-id db
+ :database database
+ :transform-id id
+ :transform-type (keyword (:type target))
+ :conn-spec conn-spec
+ :query (transforms.util/compile-source transform)
+ :output-schema (:schema target)
+ :output-table (transforms.util/qualified-table-name driver target)}
+ opts (transform-opts transform-details)
+ features (transforms.util/required-database-features transform)
+ ;; For manual runs, use the triggering user; for cron, use owner/creator
+ run-user-id (if (and (= run-method :manual) user-id)
+ user-id
+ (or owner_user_id creator_id))]
+ (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
+ (throw (ex-info "The database does not support the requested transform target type."
+ {:driver driver, :database database, :features features})))
+ ;; mark the execution as started and notify any observers
+ (let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method run-user-id)]
+ (when start-promise
+ (deliver start-promise [:started run-id]))
+ (log/info "Executing transform" id "with target" (pr-str target)
+ (when (driver.conn/write-connection-requested?)
+ " using write connection"))
+ (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
+ (transforms.util/run-cancelable-transform!
+ run-id driver transform-details
+ (fn [_cancel-chan]
+ (driver/run-transform! driver transform-details opts))))
+ (transforms.util/handle-transform-complete!
+ :run-id run-id
+ :transform transform
+ :db database)))))
(catch Throwable t
(if (= :already-running (:error (ex-data t)))
(log/warnf "Transform %d is already running" id)
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 19e12b3ac5c..212f616498a 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -8,6 +8,7 @@
[metabase.app-db.core :as mdb]
[metabase.audit-app.core :as audit]
[metabase.driver :as driver]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.impl :as driver.impl]
[metabase.driver.settings :as driver.settings]
[metabase.driver.util :as driver.u]
@@ -50,6 +51,7 @@
(t2/deftransforms :model/Database
{:details mi/transform-encrypted-json
+ :write_data_details mi/transform-encrypted-json
:engine mi/transform-keyword
:metadata_sync_schedule mi/transform-cron-string
:cache_field_values_schedule mi/transform-cron-string
@@ -220,45 +222,68 @@
(defn maybe-test-and-migrate-details!
"When a driver has db-details to test and migrate:
we loop through them until we find one that works and update the database with the working details."
- [{:keys [engine details] :as database}]
- (if-let [details-to-test (seq (driver/db-details-to-test-and-migrate (keyword engine) details))]
- (do
- (log/infof "Attempting to connect to %d possible legacy details" (count details-to-test))
- (loop [[test-details & tail] details-to-test]
- (if test-details
- (if (driver.u/can-connect-with-details? engine (assoc test-details :engine engine))
- (let [keys-remaining (-> test-details keys set)
- [_ removed _] (data/diff keys-remaining (-> details keys set))]
- (log/infof "Successfully connected, migrating to: %s" (pr-str {:keys keys-remaining :keys-removed removed}))
- (t2/update! :model/Database (:id database) {:details test-details})
- test-details)
- (recur tail))
- ;; if we go through the list and we can't fine a working detail to test, keep original value
- details)))
- details))
+ [{:keys [engine] :as database}]
+ (let [details (driver.conn/default-details database)
+ test-and-migrate! (fn [details-to-test]
+ (log/infof "Attempting to connect to %d possible legacy details" (count details-to-test))
+ (loop [[test-details & tail] details-to-test]
+ (if test-details
+ (if (driver.u/can-connect-with-details? engine (assoc test-details :engine engine))
+ (let [keys-remaining (-> test-details keys set)
+ [_ removed _] (data/diff keys-remaining (-> details keys set))]
+ (log/infof "Successfully connected, migrating to: %s" (pr-str {:keys keys-remaining :keys-removed removed}))
+ (t2/update! :model/Database (:id database) {:details test-details})
+ test-details)
+ (recur tail))
+ ;; if we go through the list and we can't fine a working detail to test, keep original value
+ details)))]
+ (if-let [details-to-test (seq (driver/db-details-to-test-and-migrate (keyword engine) details))]
+ (test-and-migrate! details-to-test)
+ details)))
+
+(defn- check-connection!
+ "Checks connectivity for a set of database details. Returns true on success, false on failure.
+ Reports analytics with the given connection-type label."
+ [database driver engine details-map connection-type]
+ (try
+ (log/info (u/format-color :cyan "Health check [%s]: checking %s {:id %d}"
+ connection-type (:name database) (:id database)))
+ (u/with-timeout (driver.settings/db-connection-timeout-ms)
+ (or (driver/can-connect? driver details-map)
+ (throw (Exception. (format "Failed to connect to Database (%s)" connection-type)))))
+ (log/info (u/format-color :green "Health check [%s]: success %s {:id %d}"
+ connection-type (:name database) (:id database)))
+ (analytics/inc! :metabase-database/status {:driver engine :healthy true :connection-type connection-type})
+ true
+ (catch Throwable e
+ (let [humanized-message (some->> (u/all-ex-messages e)
+ (driver/humanize-connection-error-message driver))
+ reason (if (keyword? humanized-message) "user-input" "exception")]
+ (log/error e (u/format-color :red "Health check [%s]: failure with error %s {:id %d :reason %s :message %s}"
+ connection-type
+ (:name database)
+ (:id database)
+ reason
+ humanized-message))
+ (analytics/inc! :metabase-database/status {:driver engine :healthy false :reason reason :connection-type connection-type}))
+ false)))
(defn health-check-database!
"Checks database health off-thread.
- - checks connectivity
- - cleans-up ambiguous legacy, db-details"
+ - checks connectivity for the default connection
+ - checks connectivity for the write connection (if configured)
+ - cleans-up ambiguous legacy db-details"
[{:keys [engine] :as database}]
(when-not (or (:is_audit database) (:is_sample database))
(log/info (u/format-color :cyan "Health check: queueing %s {:id %d}" (:name database) (:id database)))
(quick-task/submit-task!
(fn []
- (let [details (maybe-test-and-migrate-details! database)
- engine (name engine)
- driver (keyword engine)
- details-map (assoc details :engine engine)]
- (try
- (log/info (u/format-color :cyan "Health check: checking %s {:id %d}" (:name database) (:id database)))
- (u/with-timeout (driver.settings/db-connection-timeout-ms)
- (or (driver/can-connect? driver details-map)
- (throw (Exception. "Failed to connect to Database"))))
- (log/info (u/format-color :green "Health check: success %s {:id %d}" (:name database) (:id database)))
- (analytics/inc! :metabase-database/status {:driver engine :healthy true})
-
- ;; Detect and update provider name
+ (let [details (maybe-test-and-migrate-details! database)
+ database (assoc database :details details)
+ engine-str (name engine)
+ driver (keyword engine-str)
+ details-map (assoc details :engine engine-str)]
+ (when (check-connection! database driver engine-str details-map "default")
(let [provider (provider-detection/detect-provider-from-database database)]
(when (not= provider (:provider_name database))
(try
@@ -267,18 +292,12 @@
(:provider_name database) provider))
(t2/update! :model/Database (:id database) {:provider_name provider})
(catch Throwable provider-e
- (log/warnf provider-e "Error during provider detection for database {:id %d}" (:id database))))))
-
- (catch Throwable e
- (let [humanized-message (some->> (u/all-ex-messages e)
- (driver/humanize-connection-error-message driver))
- reason (if (keyword? humanized-message) "user-input" "exception")]
- (log/error e (u/format-color :red "Health check: failure with error %s {:id %d :reason %s :message %s}"
- (:name database)
- (:id database)
- reason
- humanized-message))
- (analytics/inc! :metabase-database/status {:driver engine :healthy false :reason reason})))))))))
+ (log/warnf provider-e "Error during provider detection for database {:id %d}" (:id database)))))))
+ (when (:write_data_details database)
+ (let [write-details (driver.conn/without-resolution-telemetry
+ (driver.conn/with-write-connection
+ (driver.conn/effective-details database)))]
+ (check-connection! database driver engine-str (assoc write-details :engine engine-str) "write-data"))))))))
(defn check-health!
"Health checks databases connected to metabase asynchronously using a thread pool."
@@ -348,7 +367,9 @@
(binding [*normalizing-details* true]
(driver/normalize-db-details
driver
- (m/update-existing-in db [:details :auth-provider] keyword))))]
+ (-> db
+ (m/update-existing-in [:details :auth-provider] keyword)
+ (m/update-existing-in [:write_data_details :auth-provider] keyword)))))]
(cond-> database
;; TODO - this is only really needed for API responses. This should be a `hydrate` thing instead!
(and driver
@@ -453,7 +474,8 @@
(some some? [(:cache_field_values_schedule changes) (:metadata_sync_schedule changes)]))
infer-db-schedules
- (some? (:details changes))
+ (or (some? (:details changes))
+ (some? (:write_data_details changes)))
secret/handle-incoming-client-secrets!
(:uploads_enabled changes)
@@ -548,46 +570,48 @@
driver.u/default-sensitive-fields))
(methodical/defmethod mi/to-json :model/Database
- "When encoding a Database as JSON remove the `details` for any User without write perms for the DB.
- Users with write perms can see the `details` but remove anything resembling a password. No one gets to see this in
- an API response!
+ "When encoding a Database as JSON remove the `details` and `write_data_details` for any User without write perms
+ for the DB. Users with write perms can see the details but remove anything resembling a password. No one gets to
+ see this in an API response!
Also remove settings that the User doesn't have read perms for."
[db json-generator]
- (next-method
- (let [db (if (not (mi/can-write? db))
- (do (log/debug "Fully redacting database details during json encoding.")
- (dissoc db :details))
- (do (log/debug "Redacting sensitive fields within database details during json encoding.")
- (-> db
- (secret/to-json-hydrate-redacted-secrets)
- (update :details (fn [details]
- (reduce
- #(m/update-existing %1 %2 (fn [v] (when v secret/protected-password)))
- details
- (sensitive-fields-for-db db)))))))]
- (update db :settings
- (fn [settings]
- (when (map? settings)
- (u/prog1
- (m/filter-keys
- (fn [setting-name]
- (try
- (setting/can-read-setting? setting-name
- (setting/current-user-readable-visibilities))
- (catch Throwable e
+ (letfn [(redact-sensitive-fields [details]
+ (reduce
+ #(m/update-existing %1 %2 (fn [v] (when v secret/protected-password)))
+ details
+ (sensitive-fields-for-db db)))]
+ (next-method
+ (let [db (if (not (mi/can-write? db))
+ (do (log/debug "Fully redacting database details during json encoding.")
+ (dissoc db :details :write_data_details))
+ (do (log/debug "Redacting sensitive fields within database details during json encoding.")
+ (-> db
+ (secret/to-json-hydrate-redacted-secrets)
+ (update :details redact-sensitive-fields)
+ (m/update-existing :write_data_details redact-sensitive-fields))))]
+ (update db :settings
+ (fn [settings]
+ (when (map? settings)
+ (u/prog1
+ (m/filter-keys
+ (fn [setting-name]
+ (try
+ (setting/can-read-setting? setting-name
+ (setting/current-user-readable-visibilities))
+ (catch Throwable e
;; there is an known issue with exception is ignored when render API response (#32822)
;; If you see this error, you probably need to define a setting for `setting-name`.
;; But ideally, we should resolve the above issue, and remove this try/catch
- (log/errorf e "Error checking the readability of %s setting. The setting will be hidden in API response."
- setting-name)
+ (log/errorf e "Error checking the readability of %s setting. The setting will be hidden in API response."
+ setting-name)
;; let's be conservative and hide it by defaults, if you want to see it,
;; you need to define it :)
- false)))
- settings)
- (when (not= <> settings)
- (log/debug "Redacting non-user-readable database settings during json encoding.")))))))
- json-generator))
+ false)))
+ settings)
+ (when (not= <> settings)
+ (log/debug "Redacting non-user-readable database settings during json encoding.")))))))
+ json-generator)))
;;; ------------------------------------------ Workspace Permissions Cache --------------------------------------------
@@ -634,16 +658,23 @@
details
::serdes/skip))
:import identity}
+ :write_data_details {:export-with-context
+ (fn [current _ details]
+ (if (and include-database-secrets
+ (not (:is_attached_dwh current)))
+ details
+ ::serdes/skip))
+ :import identity}
:creator_id (serdes/fk :model/User)
:router_database_id (serdes/fk :model/Database)
:initial_sync_status {:export identity :import (constantly "complete")}}})
(defmethod serdes/extract-query "Database"
[model-name {:keys [where]}]
- (t2/reducible-select (keyword "model" model-name) {:where
- [:and
- (or where true)
- [:= :router_database_id nil]]}))
+ (t2/reducible-select (keyword "model" model-name)
+ {:where [:and
+ (or where true)
+ [:= :router_database_id nil]]}))
(defmethod serdes/entity-id "Database"
[_ {:keys [name]}]
@@ -681,7 +712,7 @@
:updated-at true}
:search-terms {:name search.spec/explode-camel-case
:description true}
- :where [:= :router_database_id nil]
+ :where [:= :router_database_id nil]
:render-terms {:initial-sync-status true}})
(defenterprise hydrate-router-user-attribute
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.clj
index 390efddaa12..289eb5c3872 100644
--- a/src/metabase/warehouses_rest/api.clj
+++ b/src/metabase/warehouses_rest/api.clj
@@ -12,6 +12,7 @@
[metabase.config.core :as config]
[metabase.database-routing.core :as database-routing]
[metabase.driver :as driver]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.settings :as driver.settings]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
@@ -938,17 +939,43 @@
;;; --------------------------------------------- PUT /api/database/:id ----------------------------------------------
(defn- upsert-sensitive-fields
- "Replace any sensitive values not overridden in the PUT with the original values"
- [database details]
- (when details
- (merge (:details database)
- (reduce
- (fn [details k]
- (if (= secret/protected-password (get details k))
- (m/update-existing details k (constantly (get-in database [:details k])))
- details))
- details
- (database/sensitive-fields-for-db database)))))
+ "Replace any sensitive values not overridden in the PUT with the original values.
+ `details-key` is the key in the database map to use (e.g., :details or :write_data_details)."
+ ([database details]
+ (upsert-sensitive-fields database details :details))
+ ([database details details-key]
+ (when details
+ (merge (get database details-key)
+ (reduce
+ (fn [details k]
+ (if (= secret/protected-password (get details k))
+ (m/update-existing details k (constantly (get-in database [details-key k])))
+ details))
+ details
+ (database/sensitive-fields-for-db database))))))
+
+(defn- validate-write-data-details!
+ "Validates write_data_details guardrails. Throws 400 on violation."
+ [existing-database write-data-details]
+ (api/check-400 (not (:router_database_id existing-database))
+ (tru "Cannot configure a write connection on a destination database"))
+ (api/check-400 (not (t2/exists? :model/Database :router_database_id (:id existing-database)))
+ (tru "Cannot configure a write connection on a router database"))
+ (when-not (:write-data-connection write-data-details)
+ (throw (ex-info (tru "write-data-connection must be set in write_data_details")
+ {:status-code 400})))
+ (when (:destination-database write-data-details)
+ (throw (ex-info (tru "destination-database must be false in write_data_details")
+ {:status-code 400})))
+ (let [engine (keyword (:engine existing-database))]
+ (when-let [hidden-fields (not-empty (driver.u/fields-hidden-for-write-data-connection engine))]
+ (let [hidden-kws (into #{} (map keyword) hidden-fields)
+ disallowed-keys (filterv #(contains? hidden-kws %) (keys write-data-details))]
+ (when (seq disallowed-keys)
+ (throw (ex-info (tru "write_data_details must not contain fields hidden for write connections: {0}"
+ (str/join ", " (sort (map name disallowed-keys))))
+ {:status-code 400
+ :disallowed-keys disallowed-keys})))))))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
@@ -959,80 +986,102 @@
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]
_query-params
- {:keys [name engine details is_full_sync is_on_demand description caveats points_of_interest schedules
- auto_run_queries refingerprint cache_ttl settings provider_name]} :- [:map
- [:name {:optional true} [:maybe ms/NonBlankString]]
- [:engine {:optional true} [:maybe DBEngineString]]
- [:refingerprint {:optional true} [:maybe :boolean]]
- [:details {:optional true} [:maybe ms/Map]]
- [:schedules {:optional true} [:maybe sync.schedules/ExpandedSchedulesMap]]
- [:description {:optional true} [:maybe :string]]
- [:caveats {:optional true} [:maybe :string]]
- [:points_of_interest {:optional true} [:maybe :string]]
- [:auto_run_queries {:optional true} [:maybe :boolean]]
- [:cache_ttl {:optional true} [:maybe ms/PositiveInt]]
- [:provider_name {:optional true} [:maybe :string]]
- [:settings {:optional true} [:maybe ms/Map]]]]
+ {:keys [name engine details write_data_details is_full_sync is_on_demand description caveats points_of_interest
+ schedules auto_run_queries refingerprint cache_ttl settings provider_name]}
+ :- [:map
+ [:name {:optional true} [:maybe ms/NonBlankString]]
+ [:engine {:optional true} [:maybe DBEngineString]]
+ [:refingerprint {:optional true} [:maybe :boolean]]
+ [:details {:optional true} [:maybe ms/Map]]
+ [:write_data_details {:optional true} [:maybe ms/Map]]
+ [:schedules {:optional true} [:maybe sync.schedules/ExpandedSchedulesMap]]
+ [:description {:optional true} [:maybe :string]]
+ [:caveats {:optional true} [:maybe :string]]
+ [:points_of_interest {:optional true} [:maybe :string]]
+ [:auto_run_queries {:optional true} [:maybe :boolean]]
+ [:cache_ttl {:optional true} [:maybe ms/PositiveInt]]
+ [:provider_name {:optional true} [:maybe :string]]
+ [:settings {:optional true} [:maybe ms/Map]]]]
;; TODO - ensure that custom schedules and let-user-control-scheduling go in lockstep
- (let [existing-database (api/write-check (t2/select-one :model/Database :id id))
- incoming-details details
- details (some->> details
- (upsert-sensitive-fields existing-database))
- ;; verify that we can connect to the database if `:details` OR `:engine` have changed.
- details-changed? (some-> details (not= (:details existing-database)))
- engine-changed? (some-> engine keyword (not= (:engine existing-database)))
- conn-error (when (or details-changed? engine-changed?)
- (warehouses/test-database-connection (or engine (:engine existing-database))
- (or details (:details existing-database))))
- full-sync? (some-> is_full_sync boolean)
- on-demand? (boolean is_on_demand)]
- (if conn-error
+ (when (some? write_data_details)
+ (premium-features/assert-has-feature :advanced-permissions (tru "Advanced Permissions")))
+ (when (:write-data-connection details)
+ (throw (ex-info (tru "write-data-connection must not be set in details")
+ {:status-code 400})))
+ (let [existing-database (api/write-check (t2/select-one :model/Database :id id))
+ _ (when write_data_details
+ (validate-write-data-details! existing-database write_data_details))
+ incoming-details details
+ incoming-write-data-details write_data_details
+ details-with-secrets (some->> details
+ (upsert-sensitive-fields existing-database))
+ write-data-details-with-secrets (when write_data_details
+ (upsert-sensitive-fields existing-database write_data_details :write_data_details))
+ ;; verify that we can connect to the database if details OR `:engine` have changed.
+ details-changed? (some-> details-with-secrets (not= (:details existing-database)))
+ write-details-changed? (some-> write-data-details-with-secrets (not= (:write_data_details existing-database)))
+ engine-changed? (some-> engine keyword (not= (:engine existing-database)))
+ main-conn-error (when (or details-changed? engine-changed?)
+ (warehouses/test-database-connection (or engine (:engine existing-database))
+ (or details-with-secrets (driver.conn/default-details existing-database))))
+ write-conn-error (when (or write-details-changed? engine-changed?)
+ (let [would-be-database (cond-> existing-database
+ details-with-secrets (assoc :details details-with-secrets)
+ write-data-details-with-secrets (assoc :write_data_details write-data-details-with-secrets))]
+ (driver.conn/with-write-connection
+ (warehouses/test-database-connection (or engine (:engine would-be-database))
+ (driver.conn/effective-details would-be-database)))))
+ full-sync? (some-> is_full_sync boolean)
+ on-demand? (boolean is_on_demand)]
+ (if (or main-conn-error write-conn-error)
;; failed to connect, return error
{:status 400
- :body conn-error}
+ ;; Write details are a merge over the top of default connection:
+ :body (or main-conn-error write-conn-error)}
;; no error, proceed with update
(let [existing-settings (:settings existing-database)
- pending-settings (into {}
- ;; upsert settings with a PATCH-style update. `nil` key means unset the Setting.
- (remove (fn [[_k v]] (nil? v)))
- (merge existing-settings settings))
- updates (merge
- ;; TODO - is there really a reason to let someone change the engine on an existing database?
- ;; that seems like the kind of thing that will almost never work in any practical way
- ;; TODO - this means one cannot unset the description. Does that matter?
- (u/select-keys-when
- {:name name
- :engine engine
- :details details
- :refingerprint refingerprint
- :is_full_sync full-sync?
- :is_on_demand on-demand?
- :description description
- :caveats caveats
- :points_of_interest points_of_interest
- :auto_run_queries auto_run_queries
- :settings (when (seq settings) pending-settings)
- :provider_name provider_name}
- :non-nil #{:name :engine :details :refingerprint :is_full_sync :is_on_demand
- :description :caveats :points_of_interest :auto_run_queries :settings}
- :present #{:provider_name})
- ;; cache_field_values_schedule can be nil
- (when schedules
- (sync.schedules/schedule-map->cron-strings schedules)))
- pending-db (merge existing-database updates)]
+ pending-settings (into {}
+ ;; upsert settings with a PATCH-style update. `nil` key means unset the Setting.
+ (remove (fn [[_k v]] (nil? v)))
+ (merge existing-settings settings))
+ updates (merge
+ ;; TODO - is there really a reason to let someone change the engine on an existing database?
+ ;; that seems like the kind of thing that will almost never work in any practical way
+ ;; TODO - this means one cannot unset the description. Does that matter?
+ (u/select-keys-when
+ {:name name
+ :engine engine
+ :details details-with-secrets
+ :write_data_details write-data-details-with-secrets
+ :refingerprint refingerprint
+ :is_full_sync full-sync?
+ :is_on_demand on-demand?
+ :description description
+ :caveats caveats
+ :points_of_interest points_of_interest
+ :auto_run_queries auto_run_queries
+ :settings (when (seq settings) pending-settings)
+ :provider_name provider_name}
+ :non-nil #{:name :engine :details :refingerprint :is_full_sync :is_on_demand
+ :description :caveats :points_of_interest :auto_run_queries :settings}
+ :present #{:provider_name :write_data_details})
+ ;; cache_field_values_schedule can be nil
+ (when schedules
+ (sync.schedules/schedule-map->cron-strings schedules)))
+ pending-db (merge existing-database updates)]
;; pass in this predicate to break circular dependency
(let [driver-supports? (fn [db feature] (driver.u/supports? (driver.u/database->driver db) feature db))]
;; ensure we're not trying to set anything we should not be able to.
;; Note: it's also possible for existing settings to become invalid when changing things like the engine.
;; We skip validation for: unchanged values and nil values (resetting to default is always allowed).
(doseq [[setting-kw new-value] settings
- :when (and (some? new-value)
+ :when (and (some? new-value)
;; Allow explicit default value as well (typically this is what FE will actually do)
;; Should we translate this into setting it to NULL? That seems too opinionated.
- (not= new-value (try (setting/default-value setting-kw)
+ (not= new-value (try (setting/default-value setting-kw)
;; fallback to a redundant nil check
- (catch Exception _)))
- (not= new-value (get existing-settings setting-kw)))]
+ (catch Exception _)))
+ (not= new-value (get existing-settings setting-kw)))]
(try
(setting/validate-settable-for-db! setting-kw pending-db driver-supports?)
(catch Exception e
@@ -1047,15 +1096,16 @@
;; the details in db and existing-database have been normalized so they are the same here
;; we need to pass through details-changed? which is calculated before detail normalization
;; to ensure the pool is invalidated and [[driver-api/secret-value-as-file!]] memoization is cleared
- (events/publish-event! :event/database-update {:object db
- :user-id api/*current-user-id*
- :previous-object existing-database
+ (events/publish-event! :event/database-update {:object db
+ :user-id api/*current-user-id*
+ :previous-object existing-database
:details-changed? details-changed?})
(-> db
;; return the DB with the expanded schedules back in place
add-expanded-schedules
;; return the DB with the passed in details in place
- (m/update-existing :details #(merge incoming-details %))))))))
+ (m/update-existing :details #(merge incoming-details %))
+ (m/update-existing :write_data_details #(merge incoming-write-data-details %))))))))
;;; -------------------------------------------- DELETE /api/database/:id --------------------------------------------
@@ -1099,7 +1149,7 @@
;; it's okay to allow testing H2 connections during sync. We only want to disallow you from testing them for the
;; purposes of creating a new H2 database.
(binding [driver.settings/*allow-testing-h2-connections* true]
- (driver.u/can-connect-with-details? (:engine db) (:details db) :throw-exceptions))
+ (driver.u/can-connect-with-details? (:engine db) (driver.conn/default-details db) :throw-exceptions))
nil
(catch Throwable e
e))]
@@ -1486,11 +1536,15 @@
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :get "/:id/healthcheck"
"Reports whether the database can currently connect"
- [{:keys [id]} :- [:map [:id ms/PositiveInt]]]
- (let [{:keys [engine details]} (t2/select-one :model/Database :id id)]
+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]
+ {:keys [connection-type]} :- [:map [:connection-type {:optional true} ::driver.conn/connection-type]]]
+ (let [{:as database :keys [engine]} (t2/select-one :model/Database :id id)
+ connection-type (or connection-type :default)
+ connection-details (driver.conn/details-for-exact-type database connection-type)]
+ (api/check-400 connection-details (tru "No {0} connection configured for this database" (name connection-type)))
;; we only want to prevent creating new H2 databases. Testing the existing database is fine.
(binding [driver.settings/*allow-testing-h2-connections* true]
- (if-let [err-map (warehouses/test-database-connection engine details)]
+ (if-let [err-map (warehouses/test-database-connection engine connection-details)]
(merge err-map {:status "error"})
{:status "ok"}))))
diff --git a/test/metabase/driver/common_test.clj b/test/metabase/driver/common_test.clj
index 4e279cd1317..451e75a5fb5 100644
--- a/test/metabase/driver/common_test.clj
+++ b/test/metabase/driver/common_test.clj
@@ -73,7 +73,9 @@
(testing "JSON Unfolding database support details behave as they're supposed to"
#_{:clj-kondo/ignore [:equals-true]}
(are [details expected] (= expected
- (driver.common/json-unfolding-default {:details details}))
+ (driver.common/json-unfolding-default
+ {:lib/type :metadata/database
+ :details details}))
{} true
{:json-unfolding nil} true
{:json-unfolding true} true
diff --git a/test/metabase/driver/connection/workspaces_test.clj b/test/metabase/driver/connection/workspaces_test.clj
new file mode 100644
index 00000000000..4a240f2434a
--- /dev/null
+++ b/test/metabase/driver/connection/workspaces_test.clj
@@ -0,0 +1,48 @@
+(ns metabase.driver.connection.workspaces-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.driver.connection.workspaces :as driver.w]))
+
+(deftest ^:parallel maybe-swap-details-test
+ (testing "maybe-swap-details merges swap map into details"
+ (driver.w/with-swapped-connection-details 1 {:user "swap-user" :password "swap-pass"}
+ (is (= {:host "localhost" :user "swap-user" :password "swap-pass"}
+ (driver.w/maybe-swap-details 1 {:host "localhost" :user "original-user" :password "original-pass"})))))
+
+ (testing "maybe-swap-details returns details unchanged when no swap exists"
+ (driver.w/with-swapped-connection-details 1 {:user "swap-user"}
+ (is (= {:host "localhost" :user "original-user"}
+ (driver.w/maybe-swap-details 2 {:host "localhost" :user "original-user"})))))
+
+ (testing "maybe-swap-details supports deep merge for nested maps"
+ (driver.w/with-swapped-connection-details 1 {:ssl {:key-store-password "new-pass"}}
+ (is (= {:host "localhost" :ssl {:enabled true :key-store-password "new-pass"}}
+ (driver.w/maybe-swap-details 1 {:host "localhost" :ssl {:enabled true :key-store-password "old-pass"}})))))
+
+ (testing "deep merge adds new keys to nested maps"
+ (driver.w/with-swapped-connection-details 1 {:ssl {:new-key "new-value"}}
+ (is (= {:host "localhost" :ssl {:enabled true :new-key "new-value"}}
+ (driver.w/maybe-swap-details 1 {:host "localhost" :ssl {:enabled true}})))))
+
+ (testing "deep merge replaces nested map with non-map value"
+ (driver.w/with-swapped-connection-details 1 {:ssl "disabled"}
+ (is (= {:host "localhost" :ssl "disabled"}
+ (driver.w/maybe-swap-details 1 {:host "localhost" :ssl {:enabled true :key-store "path"}})))))
+
+ (testing "deep merge adds nested map where none existed"
+ (driver.w/with-swapped-connection-details 1 {:ssl {:enabled true}}
+ (is (= {:host "localhost" :ssl {:enabled true}}
+ (driver.w/maybe-swap-details 1 {:host "localhost"})))))
+
+ (testing "deep merge works with multiple levels of nesting"
+ (driver.w/with-swapped-connection-details 1 {:advanced {:ssl {:cert {:path "/new/path"}}}}
+ (is (= {:host "localhost" :advanced {:timeout 30 :ssl {:enabled true :cert {:path "/new/path"}}}}
+ (driver.w/maybe-swap-details 1 {:host "localhost" :advanced {:timeout 30 :ssl {:enabled true :cert {:path "/old/path"}}}})))))
+
+ (testing "nested swaps for the same database throw an exception"
+ (driver.w/with-swapped-connection-details 1 {:user "outer"}
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Nested connection detail swaps are not supported"
+ (driver.w/with-swapped-connection-details 1 {:user "inner"}
+ nil))))))
diff --git a/test/metabase/driver/connection_test.clj b/test/metabase/driver/connection_test.clj
new file mode 100644
index 00000000000..7ae1a2cc61d
--- /dev/null
+++ b/test/metabase/driver/connection_test.clj
@@ -0,0 +1,153 @@
+(ns metabase.driver.connection-test
+ (:require
+ [clojure.test :refer :all]
+ [mb.hawk.assert-exprs.approximately-equal :as =?]
+ [metabase.analytics.prometheus-test :as prometheus-test]
+ [metabase.driver.connection :as driver.conn]
+ [metabase.driver.connection.workspaces :as driver.w]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(comment =?/keep-me)
+
+(deftest effective-details-default-test
+ (testing "effective-details returns :details when *connection-type* is :default"
+ (let [details {:host "read-host" :port 5432}
+ write-data-details {:host "write-host" :port 5432}
+ database {:lib/type :metadata/database
+ :details details
+ :write-data-details write-data-details}]
+ (is (=? details
+ (driver.conn/effective-details database))))))
+
+(deftest effective-details-write-with-write-details-test
+ (let [details {:host "read-host" :port 5432}
+ write-data-details {:host "write-host" :port 5432}]
+ (testing "effective-details returns write_data_details when :write-data and details exist"
+ (testing "with kebab-case key (MLv2 metadata style)"
+ (let [database {:lib/type :metadata/database
+ :details details
+ :write-data-details write-data-details}]
+ (driver.conn/with-write-connection
+ (is (=? write-data-details
+ (driver.conn/effective-details database))))))
+ (testing "with snake_case key (Toucan2 instance style)"
+ (let [database (t2/instance :model/Database
+ {:id 1
+ :details details
+ :write_data_details write-data-details})]
+ (driver.conn/with-write-connection
+ (is (=? write-data-details
+ (driver.conn/effective-details database)))))))))
+
+(deftest effective-details-write-fallback-test
+ (let [details {:host "read-host" :port 5432}]
+ (testing "effective-details falls back to :details when :write-data but no write details"
+ (let [database {:lib/type :metadata/database
+ :details details
+ :write-data-details nil}]
+ (driver.conn/with-write-connection
+ (is (=? details
+ (driver.conn/effective-details database)))))
+ (testing "also when :write_data_details key is missing entirely"
+ (let [database {:lib/type :metadata/database
+ :details details}]
+ (driver.conn/with-write-connection
+ (is (=? details
+ (driver.conn/effective-details database)))))))))
+
+(deftest effective-details-nil-test
+ (testing "effective-details returns nil when database is nil"
+ (is (nil? (driver.conn/effective-details nil)))
+ (driver.conn/with-write-connection
+ (is (nil? (driver.conn/effective-details nil))))))
+
+(deftest write-connection-requested?-test
+ (testing "write-connection-requested? returns false by default"
+ (is (false? (driver.conn/write-connection-requested?))))
+ (testing "write-connection-requested? returns true inside with-write-connection"
+ (driver.conn/with-write-connection
+ (is (true? (driver.conn/write-connection-requested?))))))
+
+(deftest effective-details-with-workspace-swap-test
+ (testing "effective-details applies workspace swap in :default connection type"
+ (let [database {:lib/type :metadata/database
+ :id 1
+ :details {:host "read-host" :user "admin" :port 5432}}]
+ (driver.w/with-swapped-connection-details 1 {:user "ws-user" :password "ws-pass"}
+ (is (=? {:host "read-host" :user "ws-user" :password "ws-pass" :port 5432}
+ (driver.conn/effective-details database))))))
+
+ (testing "effective-details applies workspace swap AFTER write-data merge"
+ (let [database {:lib/type :metadata/database
+ :id 1
+ :details {:host "host" :user "admin" :port 5432}
+ :write-data-details {:user "writer" :password "write-pass" :write true}}]
+ (driver.w/with-swapped-connection-details 1 {:user "ws-user" :password "ws-pass"}
+ (driver.conn/with-write-connection
+ (is (=? {:host "host" :user "ws-user" :password "ws-pass" :port 5432 :write true}
+ (driver.conn/effective-details database)))))))
+
+ (testing "without workspace swap, effective-details is unchanged (regression)"
+ (let [database {:lib/type :metadata/database
+ :id 1
+ :details {:host "host" :user "admin"}
+ :write-data-details {:user "writer"}}]
+ (is (=? {:host "host" :user "admin"}
+ (driver.conn/effective-details database)))
+ (driver.conn/with-write-connection
+ (is (=? {:host "host" :user "writer"}
+ (driver.conn/effective-details database)))))))
+
+(deftest type-resolved-metric-test
+ (testing "type-resolved counter increments when write-data-details are genuinely used"
+ (mt/with-prometheus-system! [_ system]
+ (let [database {:lib/type :metadata/database
+ :id 1
+ :details {:host "read-host" :port 5432}
+ :write-data-details {:host "write-host"}}]
+ (driver.conn/with-write-connection
+ (driver.conn/effective-details database))
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-db-connection/type-resolved
+ {:connection-type "write-data"}))))))
+ (testing "type-resolved counter does NOT increment on fallback (no write-data-details)"
+ (mt/with-prometheus-system! [_ system]
+ (let [database {:lib/type :metadata/database
+ :id 1
+ :details {:host "read-host" :port 5432}}]
+ (driver.conn/with-write-connection
+ (driver.conn/effective-details database))
+ (is (prometheus-test/approx= 0 (mt/metric-value system :metabase-db-connection/type-resolved
+ {:connection-type "write-data"}))))))
+ (testing "type-resolved counter does NOT increment for default connection type"
+ (mt/with-prometheus-system! [_ system]
+ (let [database {:lib/type :metadata/database
+ :id 1
+ :details {:host "read-host" :port 5432}
+ :write-data-details {:host "write-host"}}]
+ (driver.conn/effective-details database)
+ (is (prometheus-test/approx= 0 (mt/metric-value system :metabase-db-connection/type-resolved
+ {:connection-type "write-data"}))))))
+ (testing "type-resolved counter does NOT increment when workspace swap is active"
+ (mt/with-prometheus-system! [_ system]
+ (let [database {:lib/type :metadata/database
+ :id 1
+ :details {:host "read-host" :port 5432}
+ :write-data-details {:host "write-host"}}]
+ (driver.w/with-swapped-connection-details 1 {:user "ws-user"}
+ (driver.conn/with-write-connection
+ (driver.conn/effective-details database)))
+ (is (prometheus-test/approx= 0 (mt/metric-value system :metabase-db-connection/type-resolved
+ {:connection-type "write-data"}))))))
+ (testing "type-resolved counter does NOT increment inside without-resolution-telemetry"
+ (mt/with-prometheus-system! [_ system]
+ (let [database {:lib/type :metadata/database
+ :id 1
+ :details {:host "read-host" :port 5432}
+ :write-data-details {:host "write-host"}}]
+ (driver.conn/without-resolution-telemetry
+ (driver.conn/with-write-connection
+ (is (=? {:host "write-host" :port 5432}
+ (driver.conn/effective-details database)))))
+ (is (prometheus-test/approx= 0 (mt/metric-value system :metabase-db-connection/type-resolved
+ {:connection-type "write-data"})))))))
diff --git a/test/metabase/driver/sql_jdbc/connection_test.clj b/test/metabase/driver/sql_jdbc/connection_test.clj
index 9473a376946..838706fc1b5 100644
--- a/test/metabase/driver/sql_jdbc/connection_test.clj
+++ b/test/metabase/driver/sql_jdbc/connection_test.clj
@@ -12,6 +12,8 @@
[metabase.config.core :as config]
[metabase.core.core :as mbc]
[metabase.driver :as driver]
+ [metabase.driver.connection :as driver.conn]
+ [metabase.driver.connection.workspaces :as driver.w]
[metabase.driver.settings :as driver.settings]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh]
@@ -34,6 +36,7 @@
[toucan2.core :as t2])
(:import
(com.google.common.cache Cache)
+ (com.mchange.v2.c3p0 DataSources)
(org.h2.tools Server)))
(set! *warn-on-reflection* true)
@@ -64,6 +67,7 @@
;; need to create a new, nonexistent h2 db
(let [destroyed? (atom false)
original-destroy @#'sql-jdbc.conn/destroy-pool!
+ pool-cache-key @#'sql-jdbc.conn/pool-cache-key
connection-details {:db "mem:connection_test"}
spec (mdb/spec :h2 connection-details)]
(with-redefs [sql-jdbc.conn/destroy-pool! (fn [id destroyed-spec]
@@ -77,22 +81,153 @@
(next.jdbc/execute! conn ["CREATE TABLE birds (name varchar)"])
(next.jdbc/execute! conn ["INSERT INTO birds values ('rasta'),('lucky')"])
(mt/with-temp [:model/Database database {:engine :h2, :details connection-details}]
- (testing "database id is not in our connection map initially"
+ (let [cache-key (pool-cache-key database)]
+ (testing "database id is not in our connection map initially"
;; deref'ing a var to get the atom. looks weird
- (is (not (contains? @@#'sql-jdbc.conn/database-id->connection-pool
- (u/id database)))))
- (testing "when getting a pooled connection it is now in our connection map"
- (let [stored-spec (sql-jdbc.conn/db->pooled-connection-spec database)
- birds (jdbc/query stored-spec ["SELECT * FROM birds"])]
- (is (seq birds))
- (is (contains? @@#'sql-jdbc.conn/database-id->connection-pool
- (u/id database)))))
- (testing "and is no longer in our connection map after cleanup"
- (driver/notify-database-updated :h2 database)
- (is (not (contains? @@#'sql-jdbc.conn/database-id->connection-pool
- (u/id database)))))
- (testing "the pool has been destroyed"
- (is @destroyed?))))))))))
+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool cache-key))))
+ (testing "when getting a pooled connection it is now in our connection map"
+ (let [stored-spec (sql-jdbc.conn/db->pooled-connection-spec database)
+ birds (jdbc/query stored-spec ["SELECT * FROM birds"])]
+ (is (seq birds))
+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool cache-key))))
+ (testing "and is no longer in our connection map after cleanup"
+ (driver/notify-database-updated :h2 database)
+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool cache-key))))
+ (testing "the pool has been destroyed"
+ (is @destroyed?)))))))))))
+
+(deftest connection-type-pool-separation-test
+ (mt/test-driver :h2
+ (testing "Different connection types get separate pools for the same database"
+ (let [read-details {:db "mem:read_pool_test"}
+ write-details {:db "mem:write_pool_test"}
+ spec (mdb/spec :h2 read-details)]
+ ;; Create an in-memory H2 db we can use for the test
+ (sql-jdbc.execute/do-with-connection-with-options
+ :h2
+ spec
+ {:write? true}
+ (fn [conn]
+ (next.jdbc/execute! conn ["CREATE TABLE IF NOT EXISTS test_tbl (id int)"])
+ ;; Use snake_case for column name since deftransforms uses snake_case keys
+ (mt/with-temp [:model/Database database {:engine :h2
+ :details read-details
+ :write_data_details write-details}]
+ (let [db-id (u/the-id database)
+ default-cache-key [db-id :default]
+ write-cache-key [db-id :write-data]]
+ ;; Ensure pools are cleared
+ (sql-jdbc.conn/invalidate-pool-for-db! database)
+
+ (testing "initially no pools exist"
+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key)))
+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))
+
+ (testing "getting a default connection creates only the default pool"
+ (sql-jdbc.conn/db->pooled-connection-spec database)
+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key))
+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))
+
+ (testing "getting a write connection creates a separate write pool"
+ (driver.conn/with-write-connection
+ (sql-jdbc.conn/db->pooled-connection-spec database))
+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key))
+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key)))
+
+ (testing "the two pools are different objects"
+ (let [default-pool (get @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key)
+ write-pool (get @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key)]
+ (is (some? default-pool))
+ (is (some? write-pool))
+ (is (not (identical? default-pool write-pool)))))
+
+ ;; Cleanup
+ (sql-jdbc.conn/invalidate-pool-for-db! database)))))))))
+
+(deftest write-connection-reuses-default-pool-when-unconfigured-test
+ (mt/test-driver :h2
+ (testing "When write-data-details is not configured, with-write-connection reuses the default pool"
+ (mt/with-temp [:model/Database database {:engine :h2
+ :details {:db "mem:no_write_details_test"}}]
+ (let [db-id (u/the-id database)
+ default-cache-key [db-id :default]
+ write-cache-key [db-id :write-data]]
+ (sql-jdbc.conn/invalidate-pool-for-db! database)
+ (testing "get the default pool"
+ (sql-jdbc.conn/db->pooled-connection-spec database)
+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key)))
+ (testing "with-write-connection reuses the default pool, no duplicate created"
+ (let [default-pool (get @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key)
+ write-pool (driver.conn/with-write-connection
+ (sql-jdbc.conn/db->pooled-connection-spec database))]
+ (is (identical? default-pool write-pool)
+ "Should return the exact same pool object")
+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))
+ "No separate write pool entry should exist")))
+ (sql-jdbc.conn/invalidate-pool-for-db! database))))))
+
+(deftest write-pool-uses-write-details-test
+ (mt/test-driver :h2
+ (testing "Write connection pool uses :write-data-details when available"
+ (let [read-details {:db "mem:read_details_db"}
+ write-details {:db "mem:write_details_db"}]
+ ;; Use snake_case for column name since deftransforms uses snake_case keys
+ (mt/with-temp [:model/Database database {:engine :h2
+ :details read-details
+ :write_data_details write-details}]
+ (let [db-id (u/the-id database)]
+ ;; Ensure pools are cleared
+ (sql-jdbc.conn/invalidate-pool-for-db! database)
+
+ (testing "jdbc-spec-hash differs between default and write connection types"
+ (let [default-hash (#'sql-jdbc.conn/jdbc-spec-hash database)
+ write-hash (driver.conn/with-write-connection
+ (#'sql-jdbc.conn/jdbc-spec-hash database))]
+ (is (integer? default-hash))
+ (is (integer? write-hash))
+ (is (not= default-hash write-hash)
+ "Hash should differ because effective-details returns different details")))
+
+ (testing "hash cache uses composite keys"
+ ;; Get both pools
+ (sql-jdbc.conn/db->pooled-connection-spec database)
+ (driver.conn/with-write-connection
+ (sql-jdbc.conn/db->pooled-connection-spec database))
+
+ (let [default-cached-hash (get @@#'sql-jdbc.conn/pool-cache-key->jdbc-spec-hash [db-id :default])
+ write-cached-hash (get @@#'sql-jdbc.conn/pool-cache-key->jdbc-spec-hash [db-id :write-data])]
+ (is (some? default-cached-hash))
+ (is (some? write-cached-hash))
+ (is (not= default-cached-hash write-cached-hash))))
+
+ ;; Cleanup
+ (sql-jdbc.conn/invalidate-pool-for-db! database)))))))
+
+(deftest invalidate-pool-clears-both-connection-types-test
+ (mt/test-driver :h2
+ (testing "invalidate-pool-for-db! clears both default and write pools"
+ (let [read-details {:db "mem:invalidate_test"}
+ write-details {:db "mem:invalidate_write_test"}]
+ ;; Use snake_case for column name since deftransforms uses snake_case keys
+ (mt/with-temp [:model/Database database {:engine :h2
+ :details read-details
+ :write_data_details write-details}]
+ (let [db-id (u/the-id database)
+ default-cache-key [db-id :default]
+ write-cache-key [db-id :write-data]]
+ ;; Create both pools
+ (sql-jdbc.conn/db->pooled-connection-spec database)
+ (driver.conn/with-write-connection
+ (sql-jdbc.conn/db->pooled-connection-spec database))
+
+ (testing "both pools exist before invalidation"
+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key))
+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key)))
+
+ (testing "invalidate-pool-for-db! removes both pools"
+ (sql-jdbc.conn/invalidate-pool-for-db! database)
+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key)))
+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))))))))
(deftest ^:parallel c3p0-datasource-name-test
(mt/test-drivers (mt/driver-select {:+parent :sql-jdbc})
@@ -163,7 +298,7 @@
(with-redefs [sql-jdbc.conn/log-jdbc-spec-hash-change-msg! hash-change-fn
driver.u/supports? supports?-fn]
(let [pool-spec-1 (sql-jdbc.conn/db->pooled-connection-spec db)
- db-hash-1 (get @@#'sql-jdbc.conn/database-id->jdbc-spec-hash (u/the-id db))]
+ db-hash-1 (get @@#'sql-jdbc.conn/pool-cache-key->jdbc-spec-hash (#'sql-jdbc.conn/pool-cache-key db))]
(testing "hash value calculated correctly for new pooled conn"
(is (some? pool-spec-1))
(is (integer? db-hash-1))
@@ -176,8 +311,8 @@
(t2/update! :model/Database (mt/id) {:details (:details db-perturbed)})
(let [;; this call should result in the connection pool becoming invalidated, and the new hash value
;; being stored based upon these updated details
- pool-spec-2 (sql-jdbc.conn/db->pooled-connection-spec db-perturbed)
- db-hash-2 (get @@#'sql-jdbc.conn/database-id->jdbc-spec-hash (u/the-id db))]
+ pool-spec-2 (sql-jdbc.conn/db->pooled-connection-spec db-perturbed)
+ db-hash-2 (get @@#'sql-jdbc.conn/pool-cache-key->jdbc-spec-hash (#'sql-jdbc.conn/pool-cache-key db))]
;; to throw a wrench into things, kick off a sync of the original db (unperturbed); this
;; simulates a long running sync that began before the perturbed details were saved to the app DB
;; the sync steps SHOULD NOT invalidate the connection pool, because doing so could cause a seesaw
@@ -209,14 +344,15 @@
:source nil
:value (.getBytes "super secret")
:creator_id (mt/user->id :crowberto)}]
- (let [db {:engine :postgres
- :details {:ssl true
- :ssl-mode "verify-ca"
- :ssl-root-cert-options "uploaded"
- :ssl-root-cert-creator-id (mt/user->id :crowberto)
- :ssl-root-cert-source nil
- :ssl-root-cert-id (:id secret)
- :ssl-root-cert-created-at "2022-07-25T15:57:51.556-05:00"}}]
+ (let [db {:lib/type :metadata/database
+ :engine :postgres
+ :details {:ssl true
+ :ssl-mode "verify-ca"
+ :ssl-root-cert-options "uploaded"
+ :ssl-root-cert-creator-id (mt/user->id :crowberto)
+ :ssl-root-cert-source nil
+ :ssl-root-cert-id (:id secret)
+ :ssl-root-cert-created-at "2022-07-25T15:57:51.556-05:00"}}]
(is (instance? java.io.File
(:sslrootcert (#'sql-jdbc.conn/connection-details->spec :postgres
(:details db))))
@@ -236,15 +372,15 @@
;; TODO (Cam 9/30/25) -- sort of evil to delete databases like this in a test, shouldn't we do this in a
;; transaction or something?
(t2/delete! :model/Database {:where [:= :is_audit true]})
- (let [status (mbc/ensure-audit-db-installed!)
+ (let [status (mbc/ensure-audit-db-installed!)
audit-db-id (t2/select-one-fn :id :model/Database {:where [:= :is_audit true]})
- _ (is (= :metabase-enterprise.audit-app.audit/installed status))
- _ (is (= 13371337 audit-db-id))
- first-pool (sql-jdbc.conn/db->pooled-connection-spec audit-db-id)
+ _ (is (= :metabase-enterprise.audit-app.audit/installed status))
+ _ (is (= 13371337 audit-db-id))
+ first-pool (sql-jdbc.conn/db->pooled-connection-spec audit-db-id)
second-pool (sql-jdbc.conn/db->pooled-connection-spec audit-db-id)]
(is (= first-pool second-pool))
(is (= ::audit-db-not-in-cache!
- (get @#'sql-jdbc.conn/database-id->connection-pool audit-db-id ::audit-db-not-in-cache!)))))))
+ (get @#'sql-jdbc.conn/pool-cache-key->connection-pool audit-db-id ::audit-db-not-in-cache!)))))))
(deftest ^:parallel include-unreturned-connection-timeout-test
(testing "We should be setting unreturnedConnectionTimeout; it should be the same as the query timeout (#33646)"
@@ -659,7 +795,7 @@
(sql-jdbc.conn/invalidate-pool-for-db! db)
(let [original-spec (sql-jdbc.conn/db->pooled-connection-spec db)]
(testing "Swap map is merged into details when creating connection"
- (driver/with-swapped-connection-details db-id {:test-swap true}
+ (driver.w/with-swapped-connection-details db-id {:test-swap true}
(testing "spec is swapped"
(is (not= original-spec (sql-jdbc.conn/db->pooled-connection-spec db))))
(testing "Pool was created with swap in swapped pools cache"
@@ -680,16 +816,16 @@
pool-b (atom nil)
pool-a-2 (atom nil)]
(testing "User A swaps with their credentials"
- (driver/with-swapped-connection-details db-id {:user "user-a" :password "pass-a" :log-level 100}
+ (driver.w/with-swapped-connection-details db-id {:user "user-a" :password "pass-a" :log-level 100}
(reset! pool-a-1 (sql-jdbc.conn/db->pooled-connection-spec db))
(is (= 1 (count-swapped-pools-for-db db-id)) "First swap creates one pool")))
(testing "User B swaps with different credentials"
- (driver/with-swapped-connection-details db-id {:user "user-b" :password "pass-b" :log-level 99}
+ (driver.w/with-swapped-connection-details db-id {:user "user-b" :password "pass-b" :log-level 99}
(reset! pool-b (sql-jdbc.conn/db->pooled-connection-spec db))
(is (= 2 (count-swapped-pools-for-db db-id)) "Different swap details create a second pool")
(is (not (identical? @pool-a-1 @pool-b)) "Different swap details return different pool instances")))
(testing "User A returns - should reuse their original pool (still in cache due to TTL)"
- (driver/with-swapped-connection-details db-id {:user "user-a" :password "pass-a" :log-level 100}
+ (driver.w/with-swapped-connection-details db-id {:user "user-a" :password "pass-a" :log-level 100}
(reset! pool-a-2 (sql-jdbc.conn/db->pooled-connection-spec db))
(is (= 2 (count-swapped-pools-for-db db-id)) "Identical swap details reuse existing pool")
(is (identical? @pool-a-1 @pool-a-2) "Identical swap details return the same pool instance"))))))))
@@ -700,13 +836,13 @@
(let [db (mt/db)
db-id (u/the-id db)]
(sql-jdbc.conn/invalidate-pool-for-db! db)
- (driver/with-swapped-connection-details db-id {:outer-swap true}
+ (driver.w/with-swapped-connection-details db-id {:outer-swap true}
(sql-jdbc.conn/db->pooled-connection-spec db)
(testing "Attempting nested swap for same database throws"
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Nested connection detail swaps are not supported"
- (driver/with-swapped-connection-details db-id {:inner-swap true}
+ (driver.w/with-swapped-connection-details db-id {:inner-swap true}
(sql-jdbc.conn/db->pooled-connection-spec db)))))))))
(testing "Different databases can have concurrent swaps"
@@ -714,29 +850,46 @@
(let [db-1 (mt/db)
db-1-id (u/the-id db-1)]
;; We can only test this with one db in most test setups, but the code path works
- (driver/with-swapped-connection-details db-1-id {:swap-1 true}
+ (driver.w/with-swapped-connection-details db-1-id {:swap-1 true}
;; This would work for a different db-id
(is (some? (sql-jdbc.conn/db->pooled-connection-spec db-1))))))))
(deftest invalidate-pool-clears-both-canonical-and-swapped-test
- (testing "invalidate-pool-for-db! clears both canonical and swapped pools"
+ (testing "invalidate-pool-for-db! clears canonical pools (default and write) and swapped pools"
(mt/test-drivers (mt/normal-driver-select {:+parent :sql-jdbc})
- (let [db (mt/db)
- db-id (u/the-id db)]
+ (let [db (mt/db)
+ db-id (u/the-id db)
+ default-cache-key [db-id :default]
+ write-cache-key [db-id :write-data]]
(sql-jdbc.conn/invalidate-pool-for-db! db)
+ ;; Create default canonical pool
(sql-jdbc.conn/db->pooled-connection-spec db)
- (is (contains? @@#'sql-jdbc.conn/database-id->connection-pool db-id))
-
- (driver/with-swapped-connection-details db-id {:test-swap true}
+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key)
+ "default canonical pool exists")
+
+ ;; Insert a fake write pool entry to verify it gets cleared.
+ ;; Needs :datasource so destroy-pool! can handle it.
+ (let [fake-ds (DataSources/pooledDataSource
+ (DataSources/unpooledDataSource "jdbc:h2:mem:fake_write_pool"))]
+ (swap! @#'sql-jdbc.conn/pool-cache-key->connection-pool
+ assoc write-cache-key {:datasource fake-ds}))
+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key)
+ "write canonical pool exists")
+
+ ;; Create swapped pool (for default connection type)
+ (driver.w/with-swapped-connection-details db-id {:test-swap true}
(sql-jdbc.conn/db->pooled-connection-spec db))
- (is (= 1 (count-swapped-pools-for-db db-id)))
+ (is (= 1 (count-swapped-pools-for-db db-id))
+ "swapped pool exists")
- ;; Now invalidate - should clear both
+ ;; Now invalidate - should clear all pools
(sql-jdbc.conn/invalidate-pool-for-db! db)
- (testing "Canonical pool is cleared"
- (is (not (contains? @@#'sql-jdbc.conn/database-id->connection-pool db-id))))
+ (testing "Default canonical pool is cleared"
+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key))))
+ (testing "Write canonical pool is cleared"
+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))
(testing "Swapped pool is cleared"
(is (= 0 (count-swapped-pools-for-db db-id))))))))
@@ -752,7 +905,7 @@
(fn [db]
(swap! create-count inc)
(original db)))]
- (driver/with-swapped-connection-details db-id swap-details
+ (driver.w/with-swapped-connection-details db-id swap-details
;; First call creates a pool
(let [pool-1 (sql-jdbc.conn/db->pooled-connection-spec db)]
(is (= 1 @create-count))
@@ -785,7 +938,7 @@
(fn [db]
(swap! create-count inc)
(original db)))]
- (driver/with-swapped-connection-details db-id swap-details
+ (driver.w/with-swapped-connection-details db-id swap-details
;; First call creates a pool
(let [pool-1 (sql-jdbc.conn/db->pooled-connection-spec db)]
(is (= 1 @create-count))
@@ -819,7 +972,7 @@
(fn [db]
(swap! create-count inc)
(original db)))]
- (driver/with-swapped-connection-details db-id {:test-swap true}
+ (driver.w/with-swapped-connection-details db-id {:test-swap true}
;; First call creates a pool
(let [pool-1 (sql-jdbc.conn/db->pooled-connection-spec db)]
(is (= 1 @create-count))
diff --git a/test/metabase/driver/sql_jdbc/execute_test.clj b/test/metabase/driver/sql_jdbc/execute_test.clj
index 128ffff02ca..a930f894a04 100644
--- a/test/metabase/driver/sql_jdbc/execute_test.clj
+++ b/test/metabase/driver/sql_jdbc/execute_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer :all]
[malli.error :as me]
[metabase.driver :as driver]
+ [metabase.driver.connection :as driver.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.test :as mt]
[metabase.util.malli.registry :as mr])
@@ -55,6 +56,29 @@
(sql-jdbc.execute/try-ensure-open-conn! driver closed-conn)
(is (= 2 @connection-count))))))))))
+(deftest resilient-reconnect-preserves-connection-type-test
+ (testing "resilient reconnection preserves *connection-type* binding"
+ (mt/test-drivers (descendants driver/hierarchy :sql-jdbc)
+ (let [test-db-id (mt/id)
+ captured-connection-type (volatile! nil)
+ orig-fn @#'sql-jdbc.execute/do-with-resolved-connection-data-source]
+ (with-redefs [sql-jdbc.execute/do-with-resolved-connection-data-source
+ (fn [driver db opts]
+ (when (and (= db test-db-id) (:keep-open? opts))
+ (vreset! captured-connection-type driver.conn/*connection-type*))
+ (orig-fn driver db opts))]
+ (let [closed-conn (doto (.getConnection ^DataSource
+ (orig-fn driver/*driver* test-db-id {}))
+ (.close))]
+ (driver.conn/with-write-connection
+ (driver/do-with-resilient-connection
+ driver/*driver*
+ test-db-id
+ (fn [driver _]
+ (sql-jdbc.execute/try-ensure-open-conn! driver closed-conn)
+ (is (= :write-data @captured-connection-type)
+ "Reconnection should preserve write-data connection type"))))))))))
+
(deftest try-ensure-open-conn-sets-non-recursive-options-test
(testing "try-ensure-open-conn! sets connection options as non-recursive"
#_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}
@@ -166,3 +190,24 @@
(is (false? (.isClosed prepared-stmt)))
(.close prepared-stmt)
(is (true? (.isClosed prepared-stmt)))))))))
+
+(deftest write-op-metric-test
+ (testing "write-op counter tracks default connection acquisitions"
+ (mt/with-prometheus-system! [_ system]
+ (mt/test-drivers (descendants driver/hierarchy :sql-jdbc)
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver/*driver* (mt/id) nil
+ (fn [_conn] nil))
+ (is (pos? (mt/metric-value system :metabase-db-connection/write-op
+ {:connection-type "default"}))))))
+ (testing "write-op counter tracks write-data connection acquisitions"
+ (mt/with-prometheus-system! [_ system]
+ (mt/test-drivers (descendants driver/hierarchy :sql-jdbc)
+ (let [db (mt/db)]
+ (mt/with-temp-vals-in-db :model/Database (:id db) {:write_data_details (:details db)}
+ (driver.conn/with-write-connection
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver/*driver* (mt/id) nil
+ (fn [_conn] nil)))
+ (is (pos? (mt/metric-value system :metabase-db-connection/write-op
+ {:connection-type "write-data"})))))))))
diff --git a/test/metabase/driver/sync_test.clj b/test/metabase/driver/sync_test.clj
index 7d0cfc93144..69f7409fc0c 100644
--- a/test/metabase/driver/sync_test.clj
+++ b/test/metabase/driver/sync_test.clj
@@ -34,12 +34,14 @@
(deftest ^:parallel schema-filter-NPE-test
(testing "Schema filter function should not NPE if you pass in a `nil` schema (#38156)"
(testing "inclusion -- don't include nil schemas"
- (let [db {:details {:schema-filters-type "inclusion"
- :schema-filters-patterns "x"}
- :engine :postgres}]
+ (let [db {:lib/type :metadata/database
+ :details {:schema-filters-type "inclusion"
+ :schema-filters-patterns "x"}
+ :engine :postgres}]
(is (not (driver.s/include-schema? db nil)))))
(testing "exclusion -- don't exclude nil schemas"
- (let [db {:details {:schema-filters-type "exclusion"
- :schema-filters-patterns "x"}
- :engine :postgres}]
+ (let [db {:lib/type :metadata/database
+ :details {:schema-filters-type "exclusion"
+ :schema-filters-patterns "x"}
+ :engine :postgres}]
(is (driver.s/include-schema? db nil))))))
diff --git a/test/metabase/driver_test.clj b/test/metabase/driver_test.clj
index 1a1980a4cbf..512cba9e393 100644
--- a/test/metabase/driver_test.clj
+++ b/test/metabase/driver_test.clj
@@ -481,47 +481,3 @@
(deftest deps-flags-when-supported-driver-is-not-covered-test
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Database that supports :dependencies/native does not provide an implementation of driver/native-query-deps"
(driver/native-query-deps ::mock-deps-driver nil nil))))
-
-(deftest ^:parallel maybe-swap-details-test
- (testing "maybe-swap-details merges swap map into details"
- (driver/with-swapped-connection-details 1 {:user "swap-user" :password "swap-pass"}
- (is (= {:host "localhost" :user "swap-user" :password "swap-pass"}
- (driver/maybe-swap-details 1 {:host "localhost" :user "original-user" :password "original-pass"})))))
-
- (testing "maybe-swap-details returns details unchanged when no swap exists"
- (driver/with-swapped-connection-details 1 {:user "swap-user"}
- (is (= {:host "localhost" :user "original-user"}
- (driver/maybe-swap-details 2 {:host "localhost" :user "original-user"})))))
-
- (testing "maybe-swap-details supports deep merge for nested maps"
- (driver/with-swapped-connection-details 1 {:ssl {:key-store-password "new-pass"}}
- (is (= {:host "localhost" :ssl {:enabled true :key-store-password "new-pass"}}
- (driver/maybe-swap-details 1 {:host "localhost" :ssl {:enabled true :key-store-password "old-pass"}})))))
-
- (testing "deep merge adds new keys to nested maps"
- (driver/with-swapped-connection-details 1 {:ssl {:new-key "new-value"}}
- (is (= {:host "localhost" :ssl {:enabled true :new-key "new-value"}}
- (driver/maybe-swap-details 1 {:host "localhost" :ssl {:enabled true}})))))
-
- (testing "deep merge replaces nested map with non-map value"
- (driver/with-swapped-connection-details 1 {:ssl "disabled"}
- (is (= {:host "localhost" :ssl "disabled"}
- (driver/maybe-swap-details 1 {:host "localhost" :ssl {:enabled true :key-store "path"}})))))
-
- (testing "deep merge adds nested map where none existed"
- (driver/with-swapped-connection-details 1 {:ssl {:enabled true}}
- (is (= {:host "localhost" :ssl {:enabled true}}
- (driver/maybe-swap-details 1 {:host "localhost"})))))
-
- (testing "deep merge works with multiple levels of nesting"
- (driver/with-swapped-connection-details 1 {:advanced {:ssl {:cert {:path "/new/path"}}}}
- (is (= {:host "localhost" :advanced {:timeout 30 :ssl {:enabled true :cert {:path "/new/path"}}}}
- (driver/maybe-swap-details 1 {:host "localhost" :advanced {:timeout 30 :ssl {:enabled true :cert {:path "/old/path"}}}})))))
-
- (testing "nested swaps for the same database throw an exception"
- (driver/with-swapped-connection-details 1 {:user "outer"}
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Nested connection detail swaps are not supported"
- (driver/with-swapped-connection-details 1 {:user "inner"}
- nil))))))
diff --git a/test/metabase/transforms/execute_test.clj b/test/metabase/transforms/execute_test.clj
index 998159bdcad..dd86df90910 100644
--- a/test/metabase/transforms/execute_test.clj
+++ b/test/metabase/transforms/execute_test.clj
@@ -16,8 +16,8 @@
[metabase.transforms.test-dataset :as transforms-dataset]
[metabase.transforms.test-util :as transforms.tu :refer [delete-schema! with-transform-cleanup!]]
[metabase.transforms.util :as transforms.util]
- [toucan2.core :as t2]
- [toucan2.util :as u]))
+ [metabase.util :as u]
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -368,8 +368,7 @@
record-call! (fn [call-name] (swap! calls conj call-name))
mock-query "CREATE TABLE test_schema.test_table AS (SELECT 1 AS x);"
mock-rows [[1]]
- transform-details {:conn-spec {}
- :output-table :test_schema/test_table
+ transform-details {:output-table :test_schema/test_table
:database {:id 1}
:transform-type :table}]
(with-redefs [driver/table-exists? (constantly true)
@@ -429,3 +428,110 @@
clojure.lang.ExceptionInfo
#"Failed to run the transform \(Test transform\) because the database \(.+\) has database routing turned on. Running transforms on databases with db routing enabled is not supported."
(transforms.execute/execute! transform {:run-method :cron})))))))))
+
+;; TODO(Timothy, 02-16-26):
+;; Like no-create-schema-permissions-test, this would be improved by being driver agnostic
+(deftest write-connection-used-for-transforms-test
+ (mt/test-driver :postgres
+ (mt/dataset transforms-dataset/transforms-test
+ (let [details (:details (mt/db))
+ password (:password details)
+ readonly-user (u/lower-case-en (mt/random-name))
+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))
+ schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+ make-widget-query
+ (fn [mp]
+ (let [transforms-products (lib.metadata/table mp (mt/id :transforms_products))
+ products-category (lib.metadata/field mp (mt/id :transforms_products :category))
+ products-id (lib.metadata/field mp (mt/id :transforms_products :id))]
+ (-> (lib/query mp transforms-products)
+ (lib/filter (lib/= products-category "Widget"))
+ (lib/order-by products-id :asc))))]
+ (try
+ (driver/execute-raw-queries! driver/*driver* spec
+ [[(format "CREATE ROLE %s WITH LOGIN PASSWORD '%s';" readonly-user password)]
+ [(format "GRANT USAGE ON SCHEMA %s TO %s;" schema readonly-user)]
+ [(format "GRANT SELECT ON ALL TABLES IN SCHEMA %s TO %s;" schema readonly-user)]
+ ;; Grant SELECT on future tables created by the write user,
+ ;; so the readonly user can query transform output tables.
+ [(format "ALTER DEFAULT PRIVILEGES FOR ROLE %s IN SCHEMA %s GRANT SELECT ON TABLES TO %s;"
+ (:user details) schema readonly-user)]
+ ;; PG < 15 grants CREATE on `public` to PUBLIC by default.
+ ;; Revoke it so the readonly user truly cannot create tables. :thinking_face:
+ [(format "REVOKE CREATE ON SCHEMA %s FROM PUBLIC;" schema)]])
+ (let [readonly-details (assoc details :user readonly-user)
+ write-details {:user (:user details)
+ :password (:password details)}
+ run-transform!
+ (fn [db-opts target-name]
+ (mt/with-temp [:model/Database db (merge {:engine :postgres} db-opts)]
+ (mt/with-db db
+ (sync/sync-database! db {:scan :schema})
+ (let [query (make-widget-query (mt/metadata-provider))]
+ (with-transform-cleanup! [target-table {:type "table"
+ :schema schema
+ :name target-name}]
+ (mt/with-temp [:model/Transform transform {:name target-name
+ :source {:type :query
+ :query query}
+ :target target-table}]
+ (transforms.execute/execute! transform {:run-method :manual})
+ (let [_ (transforms.tu/wait-for-table (:name target-table) 10000)
+ mp (mt/metadata-provider)
+ table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))]
+ (->> (lib/query mp table-result)
+ (qp/process-query)
+ (mt/formatted-rows [int str str 2.0 str])
+ (sort-by first <)))))))))]
+ (testing "Negative: read-only user without write_data_details fails to run transform"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"permission denied"
+ (run-transform! {:details readonly-details} "write_conn_neg"))))
+ (testing "Positive: read-only user with write_data_details succeeds"
+ (is (= [[1 "Widget A" "Widget" 19.99 "2024-01-01T10:00:00Z"]
+ [7 "Widget B" "Widget" 24.99 "2024-01-07T10:00:00Z"]
+ [9 "Widget C" "Widget" 14.99 "2024-01-09T10:00:00Z"]
+ [10 "Widget D" "Widget" 34.99 "2024-01-10T10:00:00Z"]
+ [15 "Widget E" "Widget" 44.99 "2024-01-15T10:00:00Z"]]
+ (run-transform! {:details readonly-details
+ :write_data_details write-details}
+ "write_conn_pos")))))
+ (finally
+ (driver/execute-raw-queries! driver/*driver* spec
+ [;; [(format "GRANT CREATE ON SCHEMA %s TO PUBLIC;" schema)]
+ [(format "DROP OWNED BY %s;" readonly-user)]
+ [(format "DROP ROLE IF EXISTS %s;" readonly-user)]])))))))
+
+(deftest transform-creates-write-pool-test
+ (mt/test-drivers (mt/normal-driver-select {:+parent :sql-jdbc, :+features [:transforms/table]})
+ (mt/dataset transforms-dataset/transforms-test
+ (let [db-id (mt/id)
+ write-cache-key [db-id :write-data]
+ schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+ old-write-details (:write_data_details (mt/db))]
+ (when-not old-write-details
+ (t2/update! :model/Database db-id {:write_data_details (:details (mt/db))}))
+ (try
+ (sql-jdbc.conn/invalidate-pool-for-db! (mt/db))
+ (testing "write pool does not exist before transform execution"
+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))
+ (with-transform-cleanup! [target-table {:type "table"
+ :schema schema
+ :name "pool_test"}]
+ (let [mp (mt/metadata-provider)
+ transforms-products (lib.metadata/table mp (mt/id :transforms_products))
+ products-category (lib.metadata/field mp (mt/id :transforms_products :category))
+ query (-> (lib/query mp transforms-products)
+ (lib/filter (lib/= products-category "Widget")))]
+ (mt/with-temp [:model/Transform transform {:name "pool-test"
+ :source {:type :query
+ :query query}
+ :target target-table}]
+ (transforms.execute/execute! transform {:run-method :manual})
+ (transforms.tu/wait-for-table (:name target-table) 10000)
+ (testing "write pool is created during transform execution"
+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))))
+ (finally
+ (t2/update! :model/Database db-id {:write_data_details old-write-details})
+ (sql-jdbc.conn/invalidate-pool-for-db! (mt/db))))))))
diff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.clj
index 2801e7606dd..21503cef2e9 100644
--- a/test/metabase/warehouse_schema_rest/api/table_test.clj
+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj
@@ -425,7 +425,7 @@
(is (= (merge
(-> (table-defaults)
(dissoc :segments :field_values :metrics :measures :updated_at)
- (update :db merge (select-keys (mt/db) [:details])))
+ (update :db merge (select-keys (mt/db) [:details :write_data_details])))
(t2/hydrate (t2/select-one [:model/Table :id :schema :name :created_at :initial_sync_status] :id (u/the-id table))
:pk_field :collection)
{:description "What a nice table!"
diff --git a/test/metabase/warehouses/models/database_test.clj b/test/metabase/warehouses/models/database_test.clj
index b32160182ef..38e3a983682 100644
--- a/test/metabase/warehouses/models/database_test.clj
+++ b/test/metabase/warehouses/models/database_test.clj
@@ -77,9 +77,9 @@
(mt/with-prometheus-system! [_ system]
(mt/with-temporary-setting-values [db-connection-timeout-ms 30000]
(database/check-health!)
- (is (== 1.0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})))
+ (is (== 1.0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})))
(database/check-health!)
- (is (== 1.0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true}))))))))))
+ (is (== 1.0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"}))))))))))
(deftest health-check-database-test
(mt/test-drivers (mt/normal-drivers)
@@ -89,47 +89,83 @@
(mt/with-prometheus-system! [_ system]
(mt/with-temporary-setting-values [db-connection-timeout-ms 30000]
(database/health-check-database! (mt/db))
- (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})) "healthy")
- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input"})) "unhealthy user-input")
- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception"})) "unhealthy exception"))))
+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})) "healthy")
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input" :connection-type "default"})) "unhealthy user-input")
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "default"})) "unhealthy exception"))))
(testing "skip audit"
(mt/with-prometheus-system! [_ system]
(database/health-check-database! (assoc (mt/db) :is_audit true))
- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})) "healthy")
- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input"})) "unhealthy user-input")
- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception"})) "unhealthy exception")))
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})) "healthy")
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input" :connection-type "default"})) "unhealthy user-input")
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "default"})) "unhealthy exception")))
(testing "skip sample"
(mt/with-prometheus-system! [_ system]
(database/health-check-database! (assoc (mt/db) :is_sample true))
- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})) "healthy")
- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input"})) "unhealthy user-input")
- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception"})) "unhealthy exception")))
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})) "healthy")
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input" :connection-type "default"})) "unhealthy user-input")
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "default"})) "unhealthy exception")))
(testing "failures for timeout"
(mt/with-prometheus-system! [_ system]
- (mt/with-temporary-setting-values [db-connection-timeout-ms -1] ;; setting to -1 because 0 sometimes flakes
+ (mt/with-temporary-setting-values [db-connection-timeout-ms -1]
(database/health-check-database! (mt/db))
- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})) "healthy")
- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input"})) "unhealthy user-input")
- (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception"})) "unhealthy exception"))))
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})) "healthy")
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input" :connection-type "default"})) "unhealthy user-input")
+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "default"})) "unhealthy exception"))))
(testing "failures for bad connections"
(when-let [bad-conn (tx/bad-connection-details driver/*driver*)]
(mt/with-prometheus-system! [_ system]
(database/health-check-database! (update (mt/db) :details merge bad-conn))
- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})) "healthy")
- (is (or (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input"}))
- (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception"}))) "unhealthy user-input or exception"))))
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})) "healthy")
+ (is (or (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input" :connection-type "default"}))
+ (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "default"}))) "unhealthy user-input or exception"))))
(testing "failures for exception"
(with-redefs [driver/can-connect? (fn [& _args] (throw (Exception. "boom")))]
(mt/with-prometheus-system! [_ system]
(database/health-check-database! (mt/db))
- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})) "healthy")
- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input"})) "unhealthy user-input")
- (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception"})) "unhealthy exception"))))))))
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})) "healthy")
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input" :connection-type "default"})) "unhealthy user-input")
+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "default"})) "unhealthy exception"))))))))
+
+(deftest health-check-write-connection-test
+ (mt/test-drivers (mt/normal-drivers)
+ (with-redefs [quick-task/submit-task! (fn [task] (task))]
+ (binding [driver.settings/*allow-testing-h2-connections* true]
+ (testing "database with write_data_details checks both connections"
+ (mt/with-prometheus-system! [_ system]
+ (mt/with-temporary-setting-values [db-connection-timeout-ms 30000]
+ (database/health-check-database! (assoc (mt/db) :write_data_details (:details (mt/db))))
+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"}))
+ "default connection healthy")
+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "write-data"}))
+ "write-data connection healthy"))))
+
+ (testing "database without write_data_details only checks default connection"
+ (mt/with-prometheus-system! [_ system]
+ (mt/with-temporary-setting-values [db-connection-timeout-ms 30000]
+ (database/health-check-database! (mt/db))
+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"}))
+ "default connection healthy")
+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "write-data"}))
+ "no write-data connection checked"))))
+
+ (testing "write connection failure does not prevent default check"
+ (let [call-count (atom 0)]
+ (with-redefs [driver/can-connect? (fn [& _args]
+ (if (< (swap! call-count inc) 2)
+ true
+ (throw (Exception. "write connection failed"))))]
+ (mt/with-prometheus-system! [_ system]
+ (mt/with-temporary-setting-values [db-connection-timeout-ms 30000]
+ (database/health-check-database! (assoc (mt/db) :write_data_details (:details (mt/db))))
+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"}))
+ "default connection healthy")
+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "write-data"}))
+ "write-data connection failed"))))))))))
(deftest can-read-database-setting-test
(let [encode-decode (comp json/decode json/encode)
@@ -570,6 +606,98 @@
(is (nil? (t2/select-one :model/Secret :id secret-id))
(format "Secret ID %d was not removed from the app DB" secret-id)))))))))))
+(deftest write-data-details-secrets-on-insert-test
+ (testing "handle-incoming-client-secrets! processes write_data_details on insert"
+ (mt/with-temp [:model/Database db {:engine "secret-test-driver"
+ :name "Secret Test"
+ :details {:host "localhost"}
+ :write_data_details {:keystore-value "write-secret"}}]
+ (let [db-id (:id db)
+ raw-write-details (t2/select-one-fn
+ (comp json/decode+kw :write_data_details)
+ (t2/table-name :model/Database) db-id)]
+ (testing "keystore-value replaced with keystore-id"
+ (is (contains? raw-write-details :keystore-id))
+ (is (not (contains? raw-write-details :keystore-value))))
+ (testing "Secret record created with correct value"
+ (when-let [secret-id (:keystore-id raw-write-details)]
+ (is (=? {:value (u/string-to-bytes "write-secret") :source :uploaded :version 1}
+ (secret/latest-for-id secret-id)))))))))
+
+(deftest write-data-details-secrets-on-update-test
+ (testing "handle-incoming-client-secrets! processes write_data_details on update"
+ (mt/with-temp [:model/Database db {:engine "secret-test-driver"
+ :name "Secret Test"
+ :details {:host "localhost"}}]
+ (let [db-id (:id db)]
+ (t2/update! :model/Database db-id {:write_data_details {:keystore-value "write-secret"}})
+ (let [raw-write-details (t2/select-one-fn
+ (comp json/decode+kw :write_data_details)
+ (t2/table-name :model/Database) db-id)]
+ (testing "keystore-value replaced with keystore-id"
+ (is (contains? raw-write-details :keystore-id))
+ (is (not (contains? raw-write-details :keystore-value))))
+ (testing "Secret record created with correct value"
+ (when-let [secret-id (:keystore-id raw-write-details)]
+ (is (=? {:value (u/string-to-bytes "write-secret") :source :uploaded :version 1}
+ (secret/latest-for-id secret-id))))))))))
+
+(deftest delete-orphaned-secrets-includes-write-data-details-test
+ (testing "Secret records referenced in write_data_details are deleted when database is deleted"
+ (let [db-table (t2/table-name :model/Database)]
+ (mt/with-temp [:model/Secret {write-secret-id :id} {:name "write-secret" :value "secret-value" :kind "keystore"}]
+ (mt/with-temp [db-table {db-id :id} {:engine (name :secret-test-driver)
+ :name "Secret Test"
+ :created_at (t/instant)
+ :updated_at (t/instant)
+ :details (json/encode {:host "localhost"})
+ :write_data_details (json/encode {:keystore-id write-secret-id})}]
+ (t2/delete! :model/Database :id db-id)
+ (is (nil? (t2/select-one :model/Secret :id write-secret-id))
+ "Secret referenced in write_data_details should be deleted"))))))
+
+(deftest write-data-details-redaction-to-json-test
+ (testing "to-json-hydrate-redacted-secrets hydrates redacted values in write_data_details"
+ (mt/with-temp [:model/Secret {secret-id :id} {:name "secret" :value "stored-secret-value" :kind "s" :source "uploaded"}
+ :model/Database db {:engine (name :secret-test-driver)
+ :name "Secret Test"
+ :details {:host "localhost"}}]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (let [db-with-write-details (assoc db :write_data_details {:keystore-id secret-id})
+ json-result (-> db-with-write-details json/encode json/decode+kw)]
+ (testing "keystore-value hydrated with redacted placeholder"
+ (is (= secret/protected-password
+ (get-in json-result [:write_data_details :keystore-value]))))
+ (testing "keystore-options hydrated"
+ (is (= "uploaded"
+ (get-in json-result [:write_data_details :keystore-options])))))))))
+
+(deftest clean-secret-properties-from-write-data-details-test
+ (testing "results-transform cleans secret properties from write_data_details"
+ (let [db-table (t2/table-name :model/Database)]
+ (mt/with-temp [db-table {db-id :id} {:engine (name :secret-test-driver)
+ :name "Secret Test"
+ :created_at (t/instant)
+ :updated_at (t/instant)
+ :details (json/encode {:host "localhost"})
+ :write_data_details (json/encode {:keystore-value "should-be-cleaned"
+ :keystore-id 999})}]
+ (let [db (t2/select-one :model/Database db-id)]
+ (testing "keystore-value cleaned from write_data_details"
+ (is (not (contains? (:write_data_details db) :keystore-value)))))))))
+
+(deftest after-select-keywordizes-auth-provider-in-details-test
+ (testing "after-select keywordizes :auth-provider in both :details and :write_data_details"
+ (mt/with-temp [:model/Database db {:engine :h2
+ :name "Auth Provider Test"
+ :details {:auth-provider "aws-iam"}
+ :write_data_details {:auth-provider "aws-iam"}}]
+ (let [db (t2/select-one :model/Database (:id db))]
+ (testing ":details :auth-provider is keywordized"
+ (is (keyword? (get-in db [:details :auth-provider]))))
+ (testing ":write_data_details :auth-provider is keywordized"
+ (is (keyword? (get-in db [:write_data_details :auth-provider]))))))))
+
(deftest user-may-not-update-sample-database-test
(mt/with-temp [:model/Database {:keys [id] :as _sample-database} {:engine :h2
:is_sample true
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index 4fb2308652b..5fe7ee7aed2 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -155,7 +155,7 @@
(testing "DB details visibility"
(testing "Regular users should not see DB details"
(is (= (-> (db-details)
- (dissoc :details :schedules))
+ (dissoc :details :write_data_details :schedules))
(-> (mt/user-http-request :rasta :get 200 (format "database/%d" (mt/id)))
(dissoc :schedules :can_upload)))))
(testing "Superusers should see DB details"
@@ -714,7 +714,7 @@
(deftest ^:parallel fetch-database-metadata-test
(testing "GET /api/database/:id/metadata"
- (is (= (merge (dissoc (db-details) :details :router_user_attribute)
+ (is (= (merge (dissoc (db-details) :details :write_data_details :router_user_attribute)
{:engine "h2"
:name "test-data (h2)"
:features (map u/qualified-name (driver.u/features :h2 (mt/db)))
@@ -963,7 +963,7 @@
(testing "Database details/settings *should not* come back for Rasta since she's not a superuser"
(let [expected-keys (-> #{:features :native_permissions :can_upload :router_user_attribute :transforms_permissions}
(into (keys (t2/select-one :model/Database :id (mt/id))))
- (disj :details))]
+ (disj :details :write_data_details))]
(doseq [db (:data (mt/user-http-request :rasta :get 200 "database"))]
(testing (format "Database %s %d %s" (:engine db) (u/the-id db) (pr-str (:name db)))
(is (= expected-keys
@@ -2425,7 +2425,21 @@
driver/can-connect? (constantly false)]
(is (= {:status "error"
:message "Failed to connect to Database"}
- (mt/user-http-request :crowberto :get 200 (str "database/" id "/healthcheck")))))))))
+ (mt/user-http-request :crowberto :get 200 (str "database/" id "/healthcheck")))))))
+ (testing "connection-type passed and configured"
+ (mt/with-temp [:model/Database {id :id} {:details {:host "primary"}
+ :write_data_details {:host "write"}}]
+ (with-redefs [driver/available? (constantly true)
+ driver/can-connect? (constantly true)]
+ (is (= {:status "ok"}
+ (mt/user-http-request :crowberto :get 200 (str "database/" id "/healthcheck?connection-type=write-data")))))))
+ (testing "connection-type passed but not configured returns 400"
+ (mt/with-temp [:model/Database {id :id} {:details {:host "primary"}}]
+ (with-redefs [driver/available? (constantly true)]
+ (is (mt/user-http-request :crowberto :get 400 (str "database/" id "/healthcheck?connection-type=write-data"))))))
+ (testing "invalid connection-type value returns 400"
+ (mt/with-temp [:model/Database {id :id} {}]
+ (is (mt/user-http-request :crowberto :get 400 (str "database/" id "/healthcheck?connection-type=invalid")))))))
(setting/defsetting api-test-missing-premium-feature
"A feature used for testing /settings-available (1)"
@@ -2649,3 +2663,172 @@
(is (contains? response "Manage DB")))
(testing "Database with no permissions should not be included"
(is (not (contains? response "No Access DB"))))))))
+
+;;; ----------------------------------------- write_data_details tests -----------------------------------------
+
+(deftest ^:parallel upsert-sensitive-fields-write-data-details-test
+ (testing "upsert-sensitive-fields works with :write_data_details key"
+ (is (= {:host "localhost"
+ :port 5432
+ :password "new-password"}
+ (#'api.database/upsert-sensitive-fields
+ {:engine :h2
+ :id (mt/id)
+ :details {:host "localhost" :port 5432 :password "main-pass"}
+ :write_data_details {:host "localhost" :port 5432 :password "write-pass"}}
+ {:host "localhost"
+ :port 5432
+ :password "new-password"}
+ :write_data_details)))
+ (testing "protected passwords are replaced from original"
+ (is (= {:host "localhost"
+ :port 5432
+ :password "write-pass"}
+ (#'api.database/upsert-sensitive-fields
+ {:engine :h2
+ :id (mt/id)
+ :details {:host "localhost" :port 5432 :password "main-pass"}
+ :write_data_details {:host "localhost" :port 5432 :password "write-pass"}}
+ {:host "localhost"
+ :port 5432
+ :password secret/protected-password}
+ :write_data_details))))))
+
+(deftest get-database-write-data-details-test
+ (testing "GET /api/database/:id"
+ (testing "Superusers see write_data_details with sensitive fields redacted"
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2
+ :details {:host "localhost"}
+ :write_data_details {:host "write-host"
+ :password "secret-write-pass"}}]
+ (let [response (mt/user-http-request :crowberto :get 200 (format "database/%d" db-id))]
+ (is (some? (:write_data_details response)))
+ (is (= "write-host" (get-in response [:write_data_details :host])))
+ (is (= secret/protected-password (get-in response [:write_data_details :password]))))))
+ (testing "Regular users do not see write_data_details"
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2
+ :details {:host "localhost"}
+ :write_data_details {:host "write-host"}}]
+ (let [response (mt/user-http-request :rasta :get 200 (format "database/%d" db-id))]
+ (is (not (contains? response :write_data_details)))
+ (is (not (contains? response :details))))))))
+
+(deftest update-database-write-data-details-test
+ (testing "PUT /api/database/:id with write_data_details"
+ (testing "Superusers can set write_data_details"
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2
+ :details {:host "localhost"}}]
+ (with-redefs [driver/can-connect? (constantly true)]
+ (let [response (mt/user-http-request :crowberto :put 200 (format "database/%d" db-id)
+ {:write_data_details {:host "write-host"
+ :password "write-pass"
+ :write-data-connection true}})]
+ (is (= "write-host" (get-in response [:write_data_details :host])))
+ (is (= secret/protected-password (get-in response [:write_data_details :password])))
+ (let [db (t2/select-one :model/Database :id db-id)]
+ (is (= {:host "write-host" :password "write-pass" :write-data-connection true}
+ (:write_data_details db)))))))))
+ (testing "Superusers can clear write_data_details by setting it to nil"
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2
+ :details {:host "localhost"}
+ :write_data_details {:host "write-host"}}]
+ (with-redefs [driver/can-connect? (constantly true)]
+ (mt/user-http-request :crowberto :put 200 (format "database/%d" db-id)
+ {:write_data_details nil})
+ (let [db (t2/select-one :model/Database :id db-id)]
+ (is (nil? (:write_data_details db))))))))
+ (testing "Sensitive fields are preserved when protected-password is sent"
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2
+ :details {:host "localhost"}
+ :write_data_details {:host "write-host"
+ :password "original-pass"}}]
+ (with-redefs [driver/can-connect? (constantly true)]
+ (mt/user-http-request :crowberto :put 200 (format "database/%d" db-id)
+ {:write_data_details {:host "new-write-host"
+ :password secret/protected-password
+ :write-data-connection true}})
+ (let [db (t2/select-one :model/Database :id db-id)]
+ (is (= "new-write-host" (get-in db [:write_data_details :host])))
+ (is (= "original-pass" (get-in db [:write_data_details :password]))))))))
+ (testing "Returns 402 without :advanced-permissions feature"
+ (with-redefs [premium-features/has-feature? (constantly false)]
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2
+ :details {:host "localhost"}}]
+ (mt/user-http-request :crowberto :put 402 (format "database/%d" db-id)
+ {:write_data_details {:host "write-host"}}))))))
+
+(deftest put-validates-write-data-details-connection-test
+ (testing "PUT /api/database/:id returns 400 when write connection test fails"
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2
+ :details {:host "localhost"}}]
+ (with-redefs [driver/can-connect? (fn [_engine details]
+ (if (:write-data-connection details)
+ (throw (Exception. "Write connection failed"))
+ true))]
+ (let [response (mt/user-http-request :crowberto :put 400 (format "database/%d" db-id)
+ {:write_data_details {:host "totally-bogus-host"
+ :write-data-connection true}})]
+ (is (= "Write connection failed" (:message response)))))))))
+
+(deftest write-data-details-guardrails-test
+ (testing "PUT /api/database/:id write_data_details guardrails"
+ (testing "write-data-connection must not be truthy in details"
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2
+ :details {:host "localhost"}}]
+ (is (= "write-data-connection must not be set in details"
+ (mt/user-http-request :crowberto :put 400 (format "database/%d" db-id)
+ {:details {:host "localhost"
+ :write-data-connection true}}))))))
+ (testing "write-data-connection must be truthy in write_data_details"
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2
+ :details {:host "localhost"}}]
+ (is (= "write-data-connection must be set in write_data_details"
+ (mt/user-http-request :crowberto :put 400 (format "database/%d" db-id)
+ {:write_data_details {:host "write-host"}}))))))
+ (testing "Destination-database must be false in write_data_details"
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2
+ :details {:host "localhost"}}]
+ (is (= "destination-database must be false in write_data_details"
+ (mt/user-http-request :crowberto :put 400 (format "database/%d" db-id)
+ {:write_data_details {:host "write-host"
+ :write-data-connection true
+ :destination-database true}}))))))
+ (testing "Fields hidden for write connections must not be in write_data_details"
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2
+ :details {:host "localhost"}}]
+ (is (str/includes?
+ (mt/user-http-request :crowberto :put 400 (format "database/%d" db-id)
+ {:write_data_details {:host "write-host"
+ :write-data-connection true
+ :auto_run_queries true}})
+ "write_data_details must not contain fields hidden for write connections")))))
+ (testing "Cannot set write_data_details on a destination database"
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-temp [:model/Database {router-id :id} {:engine :h2
+ :details {:host "localhost"}}
+ :model/Database {dest-id :id} {:engine :h2
+ :details {:host "localhost"}
+ :router_database_id router-id}]
+ (is (= "Cannot configure a write connection on a destination database"
+ (mt/user-http-request :crowberto :put 400 (format "database/%d" dest-id)
+ {:write_data_details {:host "write-host"
+ :write-data-connection true}}))))))
+ (testing "Cannot set write_data_details on a router database"
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-temp [:model/Database {router-id :id} {:engine :h2
+ :details {:host "localhost"}}
+ :model/Database {_dest-id :id} {:engine :h2
+ :details {:host "localhost"}
+ :router_database_id router-id}]
+ (is (= "Cannot configure a write connection on a router database"
+ (mt/user-http-request :crowberto :put 400 (format "database/%d" router-id)
+ {:write_data_details {:host "write-host"
+ :write-data-connection true}}))))))))