The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Transforms DB Connection (#68883)
Timothy S. Dean4ab6f9df02e8
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 47798754d77..1eaf8981e41 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -547,6 +547,8 @@metabase.driver.commonmetabase.driver.common.parametersmetabase.driver.common.parameters.parse+ metabase.driver.connection+ metabase.driver.connection.workspacesmetabase.driver.ddl.interfacemetabase.driver.h2metabase.driver.impl@@ -569,7 +571,8 @@metabase.driver.sql.utilmetabase.driver.syncmetabase.driver.util}- :uses #{app-db+ :uses #{analytics+ app-dbappearanceauth-providerclassloaderdiff --git a/e2e/support/helpers/e2e-request-helpers.js b/e2e/support/helpers/e2e-request-helpers.tssimilarity index 52%rename from e2e/support/helpers/e2e-request-helpers.jsrename to e2e/support/helpers/e2e-request-helpers.tsindex d19330a80eb..0b114742ac7 100644--- a/e2e/support/helpers/e2e-request-helpers.js+++ b/e2e/support/helpers/e2e-request-helpers.ts@@ -24,7 +24,27 @@export function spyRequestFinished(name = "requestFinishedSpy") {const spy = cy.spy().as(name);return {- interceptor: (req) => req.continue((res) => spy(req, res)),+ interceptor: (req: any) => req.continue((res: any) => spy(req, res)),spy,};}++const WAIT_TIMEOUT = 10000;+const WAIT_INTERVAL = 100;++export function retryRequest<T>(+ callback: () => Cypress.Chainable<T>,+ condition: (result: T) => boolean,+ timeout = WAIT_TIMEOUT,+): Cypress.Chainable<T> {+ return callback().then((result) => {+ if (condition(result)) {+ return cy.wrap(result);+ } else if (timeout > 0) {+ cy.wait(WAIT_INTERVAL);+ return retryRequest(callback, condition, timeout - WAIT_INTERVAL);+ } else {+ throw new Error("Retry timeout");+ }+ });+}diff --git a/e2e/support/helpers/e2e-transform-helpers.ts b/e2e/support/helpers/e2e-transform-helpers.tsindex f6ba93fcec7..8d7ca36d99d 100644--- a/e2e/support/helpers/e2e-transform-helpers.ts+++ b/e2e/support/helpers/e2e-transform-helpers.ts@@ -2,16 +2,17 @@ import { WRITABLE_DB_ID } from "e2e/support/cypress_data";import type {Collection,CollectionId,- ListTransformRunsResponse,PythonTransformTableAliases,TransformId,TransformRun,+ TransformRunStatus,TransformSourceCheckpointStrategy,TransformTagId,} from "metabase-types/api";import { createTransform } from "./api";import { getTableId } from "./e2e-qa-databases-helpers";+import { retryRequest } from "./e2e-request-helpers";export function createTransformCollection({name,@@ -33,34 +34,41 @@ export function visitTransform(transformId: TransformId) {}export function runTransform(transformId: TransformId) {- cy.request("POST", `/api/transform/${transformId}/run`);+ return cy.request("POST", `/api/transform/${transformId}/run`);}-const WAIT_TIMEOUT = 10000;-const WAIT_INTERVAL = 100;+export function runTransformAndWaitForStatus(+ transformId: TransformId,+ status: TransformRunStatus,+) {+ return runTransform(transformId).then(({ body: run }) => {+ return retryRequest(+ () => cy.request("GET", `/api/transform/run/${run.run_id}`),+ (response) => response.status === 200 && response.body.status === status,+ );+ });+}++export function runTransformAndWaitForSuccess(transformId: TransformId) {+ return runTransformAndWaitForStatus(transformId, "succeeded");+}++export function runTransformAndWaitForFailure(transformId: TransformId) {+ return runTransformAndWaitForStatus(transformId, "failed");+}export function waitForTransformRuns(filter: (runs: TransformRun[]) => boolean,- timeout = WAIT_TIMEOUT,-): Cypress.Chainable {- return cy- .request<ListTransformRunsResponse>("GET", "/api/transform/run")- .then((response) => {- if (filter(response.body.data)) {- return cy.wrap(response);- } else if (timeout > 0) {- cy.wait(WAIT_INTERVAL);- return waitForTransformRuns(filter, timeout - WAIT_INTERVAL);- } else {- throw new Error("Run retry timeout");- }- });+) {+ return retryRequest(+ () => cy.request("GET", "/api/transform/run"),+ ({ body }) => filter(body.data),+ );}export function waitForSucceededTransformRuns() {- waitForTransformRuns(- (runs) =>- runs.length > 0 && runs.every((run) => run.status === "succeeded"),+ return waitForTransformRuns((runs) =>+ runs.some((run) => run.status === "succeeded"),);}@@ -222,9 +230,7 @@ export function createAndRunMbqlTransform({visitTransform: false,}).then(({ body: transform }) => {// Run the transform- cy.request("POST", `/api/transform/${transform.id}/run`);- // Wait for it to complete successfully- waitForSucceededTransformRuns();+ runTransformAndWaitForSuccess(transform.id);return cy.wrap({transformId: transform.id,diff --git a/e2e/test/scenarios/admin/database-routing/database-routing-admin.cy.spec.ts b/e2e/test/scenarios/admin/database-routing/database-routing-admin.cy.spec.tsindex d159235a80c..9df7f6b373a 100644--- a/e2e/test/scenarios/admin/database-routing/database-routing-admin.cy.spec.ts+++ b/e2e/test/scenarios/admin/database-routing/database-routing-admin.cy.spec.ts@@ -90,7 +90,7 @@ describe("admin > database > database routing", () => {});dbRoutingSection().within(() => {cy.findByText("Destination DB 1");- cy.findByTestId("destination-db-health-info").realHover();+ cy.findByTestId("database-connection-health-info").realHover();});H.tooltip().should("contain.text", "Connected");diff --git a/e2e/test/scenarios/admin/databases/database-writable-connection.cy.spec.ts b/e2e/test/scenarios/admin/databases/database-writable-connection.cy.spec.tsnew file mode 100644index 00000000000..9bbba3604c9--- /dev/null+++ b/e2e/test/scenarios/admin/databases/database-writable-connection.cy.spec.ts@@ -0,0 +1,209 @@+import { WRITABLE_DB_CONFIG, WRITABLE_DB_ID } from "e2e/support/cypress_data";+import type { DatabaseId, TransformTagId } from "metabase-types/api";++const { H } = cy;++type DatabaseCredentials = {+ username: string;+ password: string;+};++const DEFAULT_USER: DatabaseCredentials = {+ username: WRITABLE_DB_CONFIG.mysql.connection.user,+ password: WRITABLE_DB_CONFIG.mysql.connection.password,+};++const READ_ONLY_USER: DatabaseCredentials = {+ username: "readonly_user",+ password: "readonly_user",+};++const DATABASE_NAME = WRITABLE_DB_CONFIG.mysql.connection.database;+const TRANSFORM_TABLE_NAME = "transform_table";++describe("scenarios > admin > databases > writable connection", () => {+ beforeEach(() => {+ H.restore("mysql-writable");+ cy.signInAsAdmin();+ H.activateToken("bleeding-edge");+ createUser(READ_ONLY_USER);+ });++ afterEach(() => {+ dropUser(READ_ONLY_USER);+ dropTable(TRANSFORM_TABLE_NAME);+ });++ it("should be able to create, edit, and remove a writable connection", () => {+ visitDatabase(WRITABLE_DB_ID);+ createWritableConnection(DEFAULT_USER);+ updateWritableConnection(READ_ONLY_USER);+ removeWritableConnection();+ getWritableConnectionInfoSection()+ .findByText("Add writable connection")+ .should("exist");+ });++ it("should validate writable connection details on save", () => {+ visitDatabase(WRITABLE_DB_ID);+ getWritableConnectionInfoSection()+ .findByText("Add writable connection")+ .click();+ fillInCredentials({+ username: "invalid",+ password: "invalid",+ });++ cy.button("Save").click();+ cy.findByRole("alert").should(+ "contain.text",+ "Metabase tried, but couldn't connect",+ );+ });++ it("should show up-to-date connection health status", () => {+ visitDatabase(WRITABLE_DB_ID);+ createWritableConnection(READ_ONLY_USER);+ getMainConnectionInfoSection().within(() => {+ getDatabaseConnectionHealthInfo().should("have.text", "Connected");+ });+ getWritableConnectionInfoSection().within(() => {+ getDatabaseConnectionHealthInfo().should("have.text", "Connected");+ });++ dropUser(READ_ONLY_USER);+ visitDatabase(WRITABLE_DB_ID);+ getMainConnectionInfoSection().within(() => {+ getDatabaseConnectionHealthInfo().should("have.text", "Connected");+ });+ getWritableConnectionInfoSection().within(() => {+ getDatabaseConnectionHealthInfo().should(+ "contain.text",+ "Could not connect",+ );+ });+ });++ it("should be able to run transforms with a writable connection", () => {+ visitDatabase(WRITABLE_DB_ID);++ createTransform().then(({ body: transform }) => {+ updateMainConnection(READ_ONLY_USER);+ H.runTransformAndWaitForFailure(transform.id);++ createWritableConnection(DEFAULT_USER);+ H.runTransformAndWaitForSuccess(transform.id);+ });+ });++ it("should be able to run transforms via a job with a writable connection", () => {+ visitDatabase(WRITABLE_DB_ID);+ updateMainConnection(READ_ONLY_USER);+ createWritableConnection(DEFAULT_USER);++ H.createTransformTag({ name: "New tag" }).then(({ body: tag }) => {+ createTransform({ tagIds: [tag.id] });+ H.createTransformJob({+ schedule: "* * * * * ? *", // every second+ tag_ids: [tag.id],+ });+ });+ H.waitForSucceededTransformRuns();+ });+});++function createTransform({ tagIds = [] }: { tagIds?: TransformTagId[] } = {}) {+ return H.createTransform({+ name: "Test transform",+ source: {+ type: "query",+ query: {+ database: WRITABLE_DB_ID,+ type: "native",+ native: {+ query: "SELECT 1 as num",+ },+ },+ },+ target: {+ type: "table",+ database: WRITABLE_DB_ID,+ name: TRANSFORM_TABLE_NAME,+ schema: null,+ },+ tag_ids: tagIds,+ });+}++function queryDB(query: string) {+ H.queryWritableDB(query, "mysql");+}++function createUser(credentials: DatabaseCredentials) {+ queryDB(+ `CREATE USER IF NOT EXISTS ${credentials.username} IDENTIFIED BY '${credentials.password}';+ GRANT SELECT ON ${DATABASE_NAME}.* TO '${credentials.username}'; `,+ );+}++function dropUser(credentials: DatabaseCredentials) {+ queryDB(`DROP USER IF EXISTS '${credentials.username}';`);+}++function dropTable(tableName: string) {+ queryDB(`DROP TABLE IF EXISTS ${tableName};`);+}++function visitDatabase(databaseId: DatabaseId) {+ cy.visit(`/admin/databases/${databaseId}`);+}++function getMainConnectionInfoSection() {+ return cy.findByTestId("database-connection-info-section");+}++function getWritableConnectionInfoSection() {+ return cy.findByTestId("writable-connection-info-section");+}++function getDatabaseConnectionHealthInfo() {+ return cy.findByTestId("database-connection-health-info");+}++function fillInCredentials(credentials: DatabaseCredentials) {+ cy.findByLabelText("Username").clear().type(credentials.username);+ cy.findByLabelText("Password").clear().type(credentials.password);+}++function createWritableConnection(credentials: DatabaseCredentials) {+ getWritableConnectionInfoSection()+ .findByText("Add writable connection")+ .click();+ fillInCredentials(credentials);+ cy.button("Save").click();+ getWritableConnectionInfoSection().should("be.visible");+}++function updateMainConnection(credentials: DatabaseCredentials) {+ getMainConnectionInfoSection().findByText("Edit connection details").click();+ fillInCredentials(credentials);+ cy.button("Save changes").click();+ getMainConnectionInfoSection().should("be.visible");+}++function updateWritableConnection(credentials: DatabaseCredentials) {+ getWritableConnectionInfoSection()+ .findByText("Edit connection details")+ .click();+ fillInCredentials(credentials);+ cy.button("Save changes").click();+ getWritableConnectionInfoSection().should("be.visible");+}++function removeWritableConnection() {+ getWritableConnectionInfoSection()+ .button("Remove writable connection")+ .click();+ H.modal().button("Remove").click();+ getWritableConnectionInfoSection().should("be.visible");+}diff --git a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.tsindex eeb69565fb3..48f1b9ed41a 100644--- a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts@@ -1618,8 +1618,7 @@ LIMITcy.get<TransformId>("@transformId").then((transformId) => {cy.log("run the transform to create the output table");- cy.request("POST", `/api/transform/${transformId}/run`);- H.waitForSucceededTransformRuns();+ H.runTransformAndWaitForSuccess(transformId);H.resyncDatabase({dbId: WRITABLE_DB_ID,tableName: transformTableName,diff --git a/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts b/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.tsindex e9b5f57bb30..c39a31321a6 100644--- a/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts+++ b/e2e/test/scenarios/dependencies/dependency-checks.cy.spec.ts@@ -364,8 +364,7 @@ function createSqlTransformWithDependentMbqlQuestions() {},}).then(({ body: transform }) => {cy.wrap(transform.id).as("transformId");- cy.request("POST", `/api/transform/${transform.id}/run`);- H.waitForSucceededTransformRuns();+ H.runTransformAndWaitForSuccess(transform.id);H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: transformTableName });H.getTableId({ databaseId: WRITABLE_DB_ID, name: transformTableName }).then(@@ -420,9 +419,8 @@ function createMbqlTransformWithDependentMbqlTransforms() {{ wrapId: true },);cy.get<number>("@transformId").then((transformId) =>- H.runTransform(transformId),+ H.runTransformAndWaitForSuccess(transformId),);- H.waitForSucceededTransformRuns();H.getTableId({ databaseId: WRITABLE_DB_ID, name: "base_transform" }).then((tableId) => {diff --git a/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts b/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.tsindex 981972195cb..617db43985f 100644--- a/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts+++ b/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts@@ -18,7 +18,6 @@ import type {NativeQuerySnippetId,SegmentId,TableId,- TransformId,} from "metabase-types/api";import { createMockCard } from "metabase-types/api/mocks";@@ -493,7 +492,7 @@ describe("scenarios > dependencies > dependency graph", () => {it("should display dependencies for a transform and navigate to them", () => {createTableBasedTransform({ tableName: TABLE_NAME }).then(({ body: transform }) => {- runTransformAndWaitForSuccess(transform.id);+ H.runTransformAndWaitForSuccess(transform.id);visitGraphForEntity(transform.id, "transform");},);@@ -1072,11 +1071,6 @@ function createSnippetBasedTransform({});}-function runTransformAndWaitForSuccess(transformId: TransformId) {- cy.request("POST", `/api/transform/${transformId}/run`);- H.waitForSucceededTransformRuns();-}-function createEmptySnippet() {return H.createSnippet({name: EMPTY_SNIPPET_NAME,diff --git a/enterprise/backend/src/metabase_enterprise/database_routing/api.clj b/enterprise/backend/src/metabase_enterprise/database_routing/api.cljindex d6018ad7426..0d5e08afead 100644--- a/enterprise/backend/src/metabase_enterprise/database_routing/api.clj+++ b/enterprise/backend/src/metabase_enterprise/database_routing/api.clj@@ -109,6 +109,7 @@(api/check-404 db)(api/check-400 (not (:router_database_id db)) "Cannot make a destination database a router database")(api/check-400 (not (:uploads_enabled db)) "Cannot enable database routing for a database with uploads enabled")+ (api/check-400 (not (:write_data_details db)) "Cannot enable database routing for a database with a write connection configured")(setting/with-database db(api/check-400 (not (setting/get :persist-models-enabled)) "Cannot enable database routing for a database with model persistence enabled")(api/check-400 (not (setting/get :database-enable-actions)) "Cannot enable database routing for a database with actions enabled")))diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljindex a32696eff53..cc134fe2db4 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -8,6 +8,7 @@[metabase-enterprise.transforms-python.settings :as transforms-python.settings][metabase.app-db.core :as app-db][metabase.driver :as driver]+ [metabase.driver.connection :as driver.conn][metabase.driver.util :as driver.u][metabase.transforms.core :as transforms][metabase.transforms.instrumentation :as transforms.instrumentation]@@ -332,8 +333,8 @@(log/error e "Failed to to create resulting table")(throw (ex-info "Failed to create the resulting table"{:transform-message (or (:transform-message (ex-data e))- ;; TODO keeping messaging the same at this level- ;; should be more specific in underlying calls+ ;; TODO keeping messaging the same at this level+ ;; should be more specific in underlying calls(i18n/tru "Failed to create the resulting table"))}e)))))))))@@ -351,37 +352,41 @@[transform {:keys [run-method start-promise user-id]}](assert (transforms.util/python-transform? transform) "Transform must be a python transform")(try- (let [message-log (empty-message-log)+ (let [message-log (empty-message-log){:keys [target owner_user_id creator_id] transform-id :id} transform- {driver :engine :as db} (t2/select-one :model/Database (transforms.i/target-db-id transform))+ {driver :engine :as db} (t2/select-one :model/Database (transforms.i/target-db-id transform));; For manual runs, use the triggering user; for cron, use owner/creator- run-user-id (if (and (= run-method :manual) user-id)- user-id- (or owner_user_id creator_id))- {run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method run-user-id)]+ run-user-id (if (and (= run-method :manual) user-id)+ user-id+ (or owner_user_id creator_id))+ {run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method run-user-id)](some-> start-promise (deliver [:started run-id]))(log! message-log (i18n/tru "Executing Python transform"))- (log/info "Executing Python transform" transform-id "with target" (pr-str target))- (let [start-ms (u/start-timer)- transform-details {:db-id (:id db)- :transform-id transform-id- :transform-type (keyword (:type target))- :conn-spec (driver/connection-spec driver db)- :output-schema (:schema target)- :output-table (transforms.util/qualified-table-name driver target)}- run-fn (fn [cancel-chan]- (run-python-transform! transform db run-id cancel-chan message-log)- (log! message-log (i18n/tru "Python execution finished successfully in {0}" (u.format/format-milliseconds (u/since-ms start-ms))))- (save-log-to-transform-run-message! run-id message-log))- ex-message-fn #(exceptional-run-message message-log %)- result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]- (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]- (transforms.util/handle-transform-complete!- :run-id run-id- :transform transform- :db db)- {:run_id run-id- :result result}))+ (driver.conn/with-write-connection+ (log/info "Executing Python transform" transform-id "with target" (pr-str target)+ (when (driver.conn/write-connection-requested?)+ " using write connection"))+ (let [start-ms (u/start-timer)+ conn-spec (driver/connection-spec driver db)+ transform-details {:db-id (:id db)+ :transform-id transform-id+ :transform-type (keyword (:type target))+ :conn-spec conn-spec+ :output-schema (:schema target)+ :output-table (transforms.util/qualified-table-name driver target)}+ run-fn (fn [cancel-chan]+ (run-python-transform! transform db run-id cancel-chan message-log)+ (log! message-log (i18n/tru "Python execution finished successfully in {0}" (u.format/format-milliseconds (u/since-ms start-ms))))+ (save-log-to-transform-run-message! run-id message-log))+ ex-message-fn #(exceptional-run-message message-log %)+ result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]+ (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]+ (transforms.util/handle-transform-complete!+ :run-id run-id+ :transform transform+ :db db)+ {:run_id run-id+ :result result})))(catch Throwable t(log/error t "Error executing Python transform")(throw t))))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/isolation.clj b/enterprise/backend/src/metabase_enterprise/workspaces/isolation.cljindex 4754cf04984..295415d2345 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/isolation.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/isolation.clj@@ -13,6 +13,7 @@This namespace provides the public API for workspace isolation."(:require[metabase.driver :as driver]+ [metabase.driver.connection.workspaces :as driver.w][metabase.driver.util :as driver.u]));;;; Delegation to driver methods@@ -43,8 +44,8 @@(defn do-with-workspace-isolation"Impl of* with-workspace-isolation*."[workspace thunk]- (driver/with-swapped-connection-details (:database_id workspace)- (:database_details workspace)+ (driver.w/with-swapped-connection-details (:database_id workspace)+ (:database_details workspace)(thunk)))(defmacro with-workspace-isolationdiff --git a/enterprise/backend/test/metabase_enterprise/database_routing/api_test.clj b/enterprise/backend/test/metabase_enterprise/database_routing/api_test.cljindex 7b789676b68..90df56052db 100644--- a/enterprise/backend/test/metabase_enterprise/database_routing/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/database_routing/api_test.clj@@ -125,6 +125,10 @@(is (= "Cannot enable database routing for a database with uploads enabled"(mt/user-http-request :crowberto :put 400 (str "ee/database-routing/router-database/" db-id){:user_attribute "db_name"}))))+ (mt/with-temp [:model/Database {db-id :id} {:write_data_details {:host "other-host"}}]+ (is (= "Cannot enable database routing for a database with a write connection configured"+ (mt/user-http-request :crowberto :put 400 (str "ee/database-routing/router-database/" db-id)+ {:user_attribute "db_name"}))))(mt/with-temp [:model/Database {router-db-id :id} {}:model/Database {db-id :id} {:router_database_id router-db-id}](is (= "Cannot make a destination database a router database"diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/drivers_test.cljindex fafef33b444..ff44280a708 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/drivers_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/drivers_test.clj@@ -1,7 +1,7 @@(ns ^:mb/driver-tests metabase-enterprise.workspaces.drivers-test(:require[clojure.test :refer :all]- [metabase.driver :as driver]+ [metabase.driver.connection.workspaces :as driver.w][metabase.test :as mt]))(deftest drivers-support-workspaces-has-to-support-db-swapping-test@@ -13,7 +13,7 @@(catch Exception _etrue)))](testing "db is swapped if being executed inside a macros"- (driver/with-swapped-connection-details (mt/id) {:user "unicorn"}+ (driver.w/with-swapped-connection-details (mt/id) {:user "unicorn"}(is (db-swapped?))))(testing "sanity check that it's not swapped outside of the macros"diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/WritableConnectionInfoSection.tsx b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/WritableConnectionInfoSection.tsxnew file mode 100644index 00000000000..5523d927df9--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/WritableConnectionInfoSection.tsx@@ -0,0 +1,96 @@+import { Link } from "react-router";+import { t } from "ttag";++import { DatabaseConnectionHealthInfo } from "metabase/admin/databases/components/DatabaseConnectionHealthInfo";+import {+ DatabaseInfoSection,+ DatabaseInfoSectionDivider,+} from "metabase/admin/databases/components/DatabaseInfoSection";+import {+ hasDbRoutingEnabled,+ isDbModifiable,+} from "metabase/admin/databases/utils";+import { useUpdateDatabaseMutation } from "metabase/api";+import { useConfirmation } from "metabase/common/hooks/use-confirmation";+import * as Urls from "metabase/lib/urls";+import type { WritableConnectionInfoSectionProps } from "metabase/plugins/oss/permissions";+import { Alert, Button, Group, Icon } from "metabase/ui";++export function WritableConnectionInfoSection({+ database,+}: WritableConnectionInfoSectionProps) {+ const hasWritableConnection = database.write_data_details !== null;+ const [updateDatabase] = useUpdateDatabaseMutation();+ const { modalContent, show: showConfirmation } = useConfirmation();+ const isDbRoutingEnabled = hasDbRoutingEnabled(database);++ if (!isDbModifiable(database)) {+ return null;+ }++ const handleRemove = () => {+ showConfirmation({+ title: t`Remove writable connection?`,+ message: t`This will remove the writable connection for this database. Any actions that depend on it will stop working.`,+ confirmButtonText: t`Remove`,+ onConfirm: async () => {+ await updateDatabase({+ id: database.id,+ write_data_details: null,+ }).unwrap();+ },+ });+ };++ return (+ <DatabaseInfoSection+ name={t`Writable connection`}+ description={t`You can add a separate writable connection to use with features like transforms and editable tables.`}+ data-testid="writable-connection-info-section"+ >+ <Group justify="space-between" gap="lg">+ {hasWritableConnection && (+ <DatabaseConnectionHealthInfo+ databaseId={database.id}+ connectionType="write-data"+ />+ )}+ <Button+ component={isDbRoutingEnabled ? undefined : Link}+ to={Urls.editDatabaseWritableConnection(database.id)}+ disabled={isDbRoutingEnabled}+ >+ {hasWritableConnection+ ? t`Edit connection details`+ : t`Add writable connection`}+ </Button>+ </Group>++ {isDbRoutingEnabled && (+ <>+ <DatabaseInfoSectionDivider />+ <Alert+ variant="light"+ color="info"+ icon={<Icon name="info" />}+ mb="md"+ >+ {t`Writable connection can't be enabled when Database Routing is enabled.`}+ </Alert>+ </>+ )}++ {hasWritableConnection && (+ <>+ <DatabaseInfoSectionDivider condensed />+ <Group>+ <Button variant="filled" color="error" onClick={handleRemove}>+ {t`Remove writable connection`}+ </Button>+ </Group>+ </>+ )}+ {modalContent}+ </DatabaseInfoSection>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/WritableConnectionInfoSection.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/WritableConnectionInfoSection.unit.spec.tsxnew file mode 100644index 00000000000..1c256dc02e5--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/WritableConnectionInfoSection.unit.spec.tsx@@ -0,0 +1,59 @@+import { setupDatabaseEndpoints } from "__support__/server-mocks/database";+import { renderWithProviders, screen } from "__support__/ui";+import type { Database } from "metabase-types/api";+import { createMockDatabase } from "metabase-types/api/mocks";++import { WritableConnectionInfoSection } from "./WritableConnectionInfoSection";++interface SetupOpts {+ database?: Database;+}++function setup({ database = createMockDatabase() }: SetupOpts = {}) {+ setupDatabaseEndpoints(database);+ renderWithProviders(<WritableConnectionInfoSection database={database} />);+}++describe("WritableConnectionInfoSection", () => {+ it("should not render for a non-modifiable database (attached DWH)", () => {+ setup({ database: createMockDatabase({ is_attached_dwh: true }) });++ expect(+ screen.queryByTestId("writable-connection-info-section"),+ ).not.toBeInTheDocument();+ });++ it("should render for a modifiable database", () => {+ setup({ database: createMockDatabase({ is_attached_dwh: false }) });++ expect(+ screen.getByTestId("writable-connection-info-section"),+ ).toBeInTheDocument();+ });++ it("should show a warning when database routing is enabled", () => {+ setup({+ database: createMockDatabase({+ router_user_attribute: "some_attribute",+ }),+ });++ expect(+ screen.getByText(+ "Writable connection can't be enabled when Database Routing is enabled.",+ ),+ ).toBeInTheDocument();+ });++ it("should not show a warning when database routing is not enabled", () => {+ setup({+ database: createMockDatabase({ router_user_attribute: null }),+ });++ expect(+ screen.queryByText(+ "Writable connection can't be enabled when Database Routing is enabled.",+ ),+ ).not.toBeInTheDocument();+ });+});diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/index.ts b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/index.tsnew file mode 100644index 00000000000..22c5fb40b9f--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/components/WritableConnectionInfoSection/index.ts@@ -0,0 +1 @@+export * from "./WritableConnectionInfoSection";diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/index.js b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/index.jsindex cab3d9bbd54..bc7218fd797 100644--- a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/index.js+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/index.js@@ -21,11 +21,13 @@ import {import { hasPremiumFeature } from "metabase-enterprise/settings";import { ImpersonationModal } from "./components/ImpersonationModal";+import { WritableConnectionInfoSection } from "./components/WritableConnectionInfoSection";import {shouldRestrictNativeQueryPermissions,upgradeViewPermissionsIfNeeded,} from "./graph";import { advancedPermissionsSlice, getImpersonatedPostAction } from "./reducer";+import { getWritableConnectionInfoRoutes } from "./routes";import { getImpersonations } from "./selectors";const IMPERSONATED_PERMISSION_OPTION = {@@ -157,6 +159,11 @@ export function initializePlugin() {upgradeViewPermissionsIfNeeded;PLUGIN_DATA_PERMISSIONS.shouldRestrictNativeQueryPermissions =shouldRestrictNativeQueryPermissions;++ PLUGIN_ADVANCED_PERMISSIONS.getWritableConnectionInfoRoutes =+ getWritableConnectionInfoRoutes;+ PLUGIN_ADVANCED_PERMISSIONS.WritableConnectionInfoSection =+ WritableConnectionInfoSection;}}diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/pages/WritableConnectionInfoPage/WritableConnectionInfoPage.tsx b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/pages/WritableConnectionInfoPage/WritableConnectionInfoPage.tsxnew file mode 100644index 00000000000..d20fc632ef1--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/pages/WritableConnectionInfoPage/WritableConnectionInfoPage.tsx@@ -0,0 +1,138 @@+import { useMemo, useState } from "react";+import type { Route } from "react-router";+import { push } from "react-router-redux";+import { t } from "ttag";++import { SettingsSection } from "metabase/admin/components/SettingsSection";+import {+ skipToken,+ useGetDatabaseQuery,+ useUpdateDatabaseMutation,+} from "metabase/api";+import { LeaveRouteConfirmModal } from "metabase/common/components/LeaveConfirmModal";+import { DelayedLoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper/DelayedLoadingAndErrorWrapper";+import { DatabaseForm } from "metabase/databases/components/DatabaseForm";+import type { DatabaseFormConfig } from "metabase/databases/types";+import { useDispatch } from "metabase/lib/redux";+import * as Urls from "metabase/lib/urls";+import { Box, Flex, ScrollArea, Title } from "metabase/ui";+import type { Database, DatabaseData } from "metabase-types/api";++const FORM_CONFIG: DatabaseFormConfig = {+ engine: {+ fieldState: "disabled",+ },+ name: {+ fieldState: "hidden",+ },+ isAdvanced: true,+};++type WritableConnectionInfoPageParams = {+ databaseId: string;+};++type WritableConnectionInfoPageProps = {+ params: WritableConnectionInfoPageParams;+ route: Route;+};++export function WritableConnectionInfoPage({+ params,+ route,+}: WritableConnectionInfoPageProps) {+ const databaseId = Urls.extractEntityId(params.databaseId);+ const {+ data: database,+ isLoading,+ error,+ } = useGetDatabaseQuery(databaseId != null ? { id: databaseId } : skipToken);++ if (isLoading || error != null || database == null) {+ return <DelayedLoadingAndErrorWrapper loading={isLoading} error={error} />;+ }++ return <WritableConnectionInfoPageBody database={database} route={route} />;+}++type WritableConnectionInfoPageBodyProps = {+ database: Database;+ route: Route;+};++function WritableConnectionInfoPageBody({+ database,+ route,+}: WritableConnectionInfoPageBodyProps) {+ const title = getTitle(database);+ const initialValues = useMemo(() => getInitialValues(database), [database]);+ const [isDirty, setIsDirty] = useState(false);+ const [updateDatabase, { isLoading: isSaving }] = useUpdateDatabaseMutation();+ const dispatch = useDispatch();++ const handleSubmit = async (newValues: DatabaseData) => {+ await updateDatabase({+ id: database.id,+ write_data_details: getSubmitDetails(newValues),+ }).unwrap();+ dispatch(push(Urls.viewDatabase(database.id)));+ };++ const handleCancel = () => {+ dispatch(push(Urls.viewDatabase(database.id)));+ };++ return (+ <Flex+ direction="row"+ h="100%"+ bg="background-secondary"+ data-testid="writable-connection-info-page"+ >+ <Box h="100%" w="100%" component={ScrollArea}>+ <Box w="100%" maw="54rem" mx="auto" p={{ base: "md", sm: "xl" }}>+ <Flex mb="lg" align="center">+ <Title order={1} fz="h2">+ {title}+ </Title>+ </Flex>+ <SettingsSection>+ <DatabaseForm+ initialValues={initialValues}+ config={FORM_CONFIG}+ location="full-page"+ onSubmit={handleSubmit}+ onCancel={handleCancel}+ onDirtyStateChange={setIsDirty}+ />+ </SettingsSection>+ </Box>+ </Box>+ <LeaveRouteConfirmModal isEnabled={isDirty && !isSaving} route={route} />+ </Flex>+ );+}++function getTitle(database: Database): string {+ return database.write_data_details == null+ ? t`Add writable connection`+ : t`Edit writable connection details`;+}++function getInitialValues(database: Database): DatabaseData {+ const { password, ...details } = database.details ?? {};++ return {+ ...database,+ id: database.write_data_details ? database.id : undefined,+ details: {+ ...details,+ ...database.write_data_details,+ "write-data-connection": true,+ },+ };+}++function getSubmitDetails(values: DatabaseData) {+ return values.details ?? {};+}diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/pages/WritableConnectionInfoPage/index.ts b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/pages/WritableConnectionInfoPage/index.tsnew file mode 100644index 00000000000..051c3cd9b34--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/pages/WritableConnectionInfoPage/index.ts@@ -0,0 +1 @@+export * from "./WritableConnectionInfoPage";diff --git a/enterprise/frontend/src/metabase-enterprise/advanced_permissions/routes.tsx b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/routes.tsxnew file mode 100644index 00000000000..c4e789db0bd--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/advanced_permissions/routes.tsx@@ -0,0 +1,12 @@+import { Route } from "react-router";++import { WritableConnectionInfoPage } from "./pages/WritableConnectionInfoPage";++export function getWritableConnectionInfoRoutes() {+ return (+ <Route+ path=":databaseId/write-data"+ component={WritableConnectionInfoPage}+ />+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/auth/components/AuthSettingsPage.tsx b/enterprise/frontend/src/metabase-enterprise/auth/components/AuthSettingsPage.tsxindex 7e58758f446..735e65c4794 100644--- a/enterprise/frontend/src/metabase-enterprise/auth/components/AuthSettingsPage.tsx+++ b/enterprise/frontend/src/metabase-enterprise/auth/components/AuthSettingsPage.tsx@@ -9,7 +9,10 @@ import { LdapAuthCard } from "metabase/admin/settings/auth/containers/LdapAuthCaimport { ManageApiKeys } from "metabase/admin/settings/components/ApiKeys/ManageApiKeys";import { AdminSettingInput } from "metabase/admin/settings/components/widgets/AdminSettingInput";import { useHasTokenFeature } from "metabase/common/hooks";-import { PLUGIN_AUTH_PROVIDERS } from "metabase/plugins";+import {+ type AuthSettingsPageProps,+ PLUGIN_AUTH_PROVIDERS,+} from "metabase/plugins";import { JwtAuthCard } from "../containers/JwtAuthCard";import { SamlAuthCard } from "../containers/SamlAuthCard";@@ -17,9 +20,9 @@ import { useHasSsoEnabled } from "../utils";import { SessionTimeoutSetting } from "./SessionTimeoutSetting";-type Tabs = "authentication" | "user-provisioning" | "api-keys";--export function AuthSettingsPage({ tab = "authentication" }: { tab?: Tabs }) {+export function AuthSettingsPage({+ tab = "authentication",+}: AuthSettingsPageProps) {if (tab === "api-keys") {return <ManageApiKeys />;}diff --git a/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.tsx b/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.tsxindex 417a231d2fd..b675fa65e24 100644--- a/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.tsx+++ b/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.tsx@@ -143,12 +143,28 @@ export const DatabaseRoutingSection = ({/></Box></Tooltip>- <UnstyledButton onClick={() => setIsExpanded(!isExpanded)} px="xs">- <Icon name={isExpanded ? "chevronup" : "chevrondown"} />- </UnstyledButton>+ {disabledFeatMsg == null && (+ <UnstyledButton onClick={() => setIsExpanded(!isExpanded)} px="xs">+ <Icon name={isExpanded ? "chevronup" : "chevrondown"} />+ </UnstyledButton>+ )}</Flex></Flex>+ {disabledFeatMsg != null && (+ <>+ <DatabaseInfoSectionDivider />+ <Alert+ variant="light"+ color="info"+ icon={<Icon name="info" />}+ mb="md"+ >+ {disabledFeatMsg}+ </Alert>+ </>+ )}+{isExpanded && (<><DatabaseInfoSectionDivider />diff --git a/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.unit.spec.tsxindex da76d608f03..66f3110b7ff 100644--- a/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/DatabaseRoutingSection.unit.spec.tsx@@ -15,13 +15,15 @@ import { createMockSettingsState } from "metabase-types/store/mocks";import { DatabaseRoutingSection } from "./DatabaseRoutingSection";-const setup = (database: Partial<Database>) => {- const db = createMockDatabase(database);+interface SetupOpts {+ database?: Database;+}+const setup = ({ database = createMockDatabase() }: SetupOpts = {}) => {setupUserAttributesEndpoint(["cool_guy", "boss_gal"]);- setupDatabasesEndpoints([db]);+ setupDatabasesEndpoints([database]);- renderWithProviders(<DatabaseRoutingSection database={db} />, {+ renderWithProviders(<DatabaseRoutingSection database={database} />, {storeInitialState: {currentUser: createMockUser({ is_superuser: true }),settings: createMockSettingsState(@@ -44,7 +46,12 @@ const setup = (database: Partial<Database>) => {describe("DatabaseRoutingSection", () => {it("should render DatabaseRoutingSection", () => {- setup({ engine: "postgres", features: ["database-routing"] });+ setup({+ database: createMockDatabase({+ engine: "postgres",+ features: ["database-routing"],+ }),+ });expect(screen.getByText("Database routing")).toBeInTheDocument();expect(@@ -58,8 +65,10 @@ describe("DatabaseRoutingSection", () => {it("should render DatabaseRoutingSection with custom db_routing_info", () => {setup({- engine: "bigquery-cloud-sdk",- features: ["database-routing"],+ database: createMockDatabase({+ engine: "bigquery-cloud-sdk",+ features: ["database-routing"],+ }),});expect(screen.getByText("Database routing")).toBeInTheDocument();@@ -70,9 +79,11 @@ describe("DatabaseRoutingSection", () => {it("should hide section if database is attached DWH", () => {setup({- engine: "postgres",- is_attached_dwh: true,- features: ["database-routing"],+ database: createMockDatabase({+ engine: "postgres",+ is_attached_dwh: true,+ features: ["database-routing"],+ }),});expect(screen.queryByText("Database routing")).not.toBeInTheDocument();@@ -80,16 +91,36 @@ describe("DatabaseRoutingSection", () => {it("should hide section if database is sample", () => {setup({- engine: "postgres",- is_sample: true,- features: ["database-routing"],+ database: createMockDatabase({+ engine: "postgres",+ is_sample: true,+ features: ["database-routing"],+ }),});expect(screen.queryByText("Database routing")).not.toBeInTheDocument();});it("should hide section if database routing is not supported by the db engine", async () => {- setup({ engine: "clickhouse", features: [] });+ setup({+ database: createMockDatabase({ engine: "clickhouse", features: [] }),+ });expect(screen.queryByText("Database routing")).not.toBeInTheDocument();});++ it("should show a warning when writable connection is enabled", () => {+ setup({+ database: createMockDatabase({+ engine: "postgres",+ features: ["database-routing"],+ write_data_details: { host: "localhost" },+ }),+ });++ expect(+ screen.getByText(+ "Database routing can't be enabled when a Writable Connection is enabled.",+ ),+ ).toBeInTheDocument();+ });});diff --git a/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/utils.tsx b/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/utils.tsxindex 00f1efdcefb..79d200c5f32 100644--- a/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/utils.tsx+++ b/enterprise/frontend/src/metabase-enterprise/database_routing/DatabaseRoutingSection/utils.tsx@@ -2,7 +2,11 @@ import { Link } from "react-router";import { match } from "ts-pattern";import { t } from "ttag";-import { hasActionsEnabled, hasFeature } from "metabase/admin/databases/utils";+import {+ hasActionsEnabled,+ hasFeature,+ hasWritableConnectionDetails,+} from "metabase/admin/databases/utils";import { Text } from "metabase/ui";import type { Database } from "metabase-types/api";@@ -12,6 +16,7 @@ export const getDisabledFeatureMessage = (database: Database) => {isPersisted: hasFeature(database, "persist-models-enabled"),isUploadDb: database.uploads_enabled,supportsRouting: !!database.features?.includes("database-routing"),+ hasWritableConnection: hasWritableConnectionDetails(database),}).with({ supportsRouting: false },@@ -36,6 +41,11 @@ export const getDisabledFeatureMessage = (database: Database) => {() =>t`Database routing can't be enabled if uploads are enabled for this database.`,)+ .with(+ { hasWritableConnection: true },+ () =>+ t`Database routing can't be enabled when a Writable Connection is enabled.`,+ ).otherwise(() => undefined);};diff --git a/enterprise/frontend/src/metabase-enterprise/database_routing/DestinationDatabasesList/DestinationDatabasesList.tsx b/enterprise/frontend/src/metabase-enterprise/database_routing/DestinationDatabasesList/DestinationDatabasesList.tsxindex 67f371a92e7..dc906a16748 100644--- a/enterprise/frontend/src/metabase-enterprise/database_routing/DestinationDatabasesList/DestinationDatabasesList.tsx+++ b/enterprise/frontend/src/metabase-enterprise/database_routing/DestinationDatabasesList/DestinationDatabasesList.tsx@@ -92,7 +92,6 @@ const DestinationDatabasesListItem = ({<DatabaseConnectionHealthInfodatabaseId={database.id}displayText="tooltip"- data-testid="destination-db-health-info"/><Text>{database.name}</Text></Flex>diff --git a/enterprise/frontend/src/metabase-enterprise/semantic_search/SearchSettingsWidget.tsx b/enterprise/frontend/src/metabase-enterprise/semantic_search/SearchSettingsWidget.tsxindex 4bbc626519c..66cdeca4a3e 100644--- a/enterprise/frontend/src/metabase-enterprise/semantic_search/SearchSettingsWidget.tsx+++ b/enterprise/frontend/src/metabase-enterprise/semantic_search/SearchSettingsWidget.tsx@@ -8,6 +8,7 @@ import { UpsellSemanticSearchPill } from "metabase/admin/upsells/UpsellSemanticSimport { getErrorMessage, useAdminSetting } from "metabase/api/utils";import { getPlan, isProPlan } from "metabase/common/utils/plan";import { useSelector } from "metabase/lib/redux";+import type { SearchSettingsWidgetProps } from "metabase/plugins";import { getSetting } from "metabase/selectors/settings";import { Box, Progress, Stack, Text, Tooltip } from "metabase/ui";import { useGetSemanticSearchStatusQuery } from "metabase-enterprise/api/search";@@ -26,9 +27,7 @@ function useLatch(bool: boolean) {export function SearchSettingsWidget({statusPollingInterval = 5000,-}: {- statusPollingInterval?: number;-}) {+}: SearchSettingsWidgetProps) {const plan = useSelector((state) =>getPlan(getSetting(state, "token-features")),);diff --git a/frontend/src/metabase-types/api/database.ts b/frontend/src/metabase-types/api/database.tsindex 9ba8ff96064..7fba227d4f1 100644--- a/frontend/src/metabase-types/api/database.ts+++ b/frontend/src/metabase-types/api/database.ts@@ -96,6 +96,7 @@ export interface DatabaseData {// missing in responses from the backend, cf. implementation of// [[metabase.models.interface/to-json]] for `:model/Database`:details?: Record<string, unknown>;+ write_data_details?: Record<string, unknown> | null;schedules: DatabaseSchedules;auto_run_queries: boolean | null;refingerprint: boolean | null;@@ -138,6 +139,13 @@ export type DatabaseLocalSettingAvailability =| { enabled: true }| { enabled: false; reasons: DatabaseLocalSettingDisableReason[] };+export type DatabaseConnectionType = "default" | "write-data";++export type GetDatabaseHealthRequest = {+ id: DatabaseId;+ "connection-type"?: DatabaseConnectionType;+};+export type GetDatabaseHealthResponse =| { status: "ok" }| { status: "error"; message: string; errors: unknown };@@ -212,6 +220,7 @@ export interface UpdateDatabaseRequest {engine?: string;refingerprint?: boolean | null;details?: Record<string, unknown>;+ write_data_details?: Record<string, unknown> | null;schedules?: DatabaseSchedules;description?: string;caveats?: string;diff --git a/frontend/src/metabase/admin/databases/components/DatabaseConnectionHealthInfo.tsx b/frontend/src/metabase/admin/databases/components/DatabaseConnectionHealthInfo.tsxindex 2f54f80d7b1..6f1df04f5e2 100644--- a/frontend/src/metabase/admin/databases/components/DatabaseConnectionHealthInfo.tsx+++ b/frontend/src/metabase/admin/databases/components/DatabaseConnectionHealthInfo.tsx@@ -4,19 +4,24 @@ import { t } from "ttag";import { useGetDatabaseHealthQuery } from "metabase/api";import { Badge, Flex, type FlexProps, Text, Tooltip } from "metabase/ui";-import type { DatabaseId } from "metabase-types/api";+import type { DatabaseConnectionType, DatabaseId } from "metabase-types/api";interface DatabaseConnectionHealthInfoProps extends FlexProps {databaseId: DatabaseId;+ connectionType?: DatabaseConnectionType;displayText?: "inline" | "tooltip";}export const DatabaseConnectionHealthInfo = ({databaseId,displayText = "inline",+ connectionType = "default",...props}: DatabaseConnectionHealthInfoProps) => {- const healthQuery = useGetDatabaseHealthQuery(databaseId);+ const healthQuery = useGetDatabaseHealthQuery({+ id: databaseId,+ "connection-type": connectionType,+ });const health = useMemo(() => {return match(healthQuery).with(@@ -49,7 +54,12 @@ export const DatabaseConnectionHealthInfo = ({}, [healthQuery]);return (- <Flex align="center" gap="sm" {...props}>+ <Flex+ align="center"+ gap="sm"+ {...props}+ data-testid="database-connection-health-info"+ ><Tooltip label={health.message}><Badge size="12" circle bg={health.color} style={{ flexShrink: 0 }} /></Tooltip>diff --git a/frontend/src/metabase/admin/databases/containers/DatabaseEditApp.tsx b/frontend/src/metabase/admin/databases/containers/DatabaseEditApp.tsxindex db0d79481e7..ec7f2c060a9 100644--- a/frontend/src/metabase/admin/databases/containers/DatabaseEditApp.tsx+++ b/frontend/src/metabase/admin/databases/containers/DatabaseEditApp.tsx@@ -16,6 +16,7 @@ import CS from "metabase/css/core/index.css";import { usePageTitle } from "metabase/hooks/use-page-title";import { connect, useSelector } from "metabase/lib/redux";import {+ PLUGIN_ADVANCED_PERMISSIONS,PLUGIN_DATABASE_REPLICATION,PLUGIN_DB_ROUTING,PLUGIN_TABLE_EDITING,@@ -103,6 +104,10 @@ function DatabaseEditAppInner({><DatabaseConnectionInfoSection database={database} />+ <PLUGIN_ADVANCED_PERMISSIONS.WritableConnectionInfoSection+ database={database}+ />+<DatabaseModelFeaturesSectiondatabase={database}isModelPersistenceEnabled={isModelPersistenceEnabled}diff --git a/frontend/src/metabase/admin/databases/containers/DatabasePage.tsx b/frontend/src/metabase/admin/databases/containers/DatabasePage.tsxindex 47c3d506034..b0ef1e47a55 100644--- a/frontend/src/metabase/admin/databases/containers/DatabasePage.tsx+++ b/frontend/src/metabase/admin/databases/containers/DatabasePage.tsx@@ -62,21 +62,9 @@ export function DatabasePage({ params, route }: DatabasePageProps) {};return (- <Flex- direction="row"- h="100%"- style={{ backgroundColor: "var(--mb-color-background-secondary)" }}- >+ <Flex direction="row" h="100%" bg="background-secondary"><Box h="100%" w="100%" component={ScrollArea}>- <Box- w="100%"- maw="54rem"- mx="auto"- p={{- base: `md`,- sm: `xl`,- }}- >+ <Box w="100%" maw="54rem" mx="auto" p={{ base: "md", sm: "xl" }}><Flexmb="lg"align="center"diff --git a/frontend/src/metabase/admin/databases/utils.ts b/frontend/src/metabase/admin/databases/utils.tsindex 9b91207d5ed..0be3cecfcf7 100644--- a/frontend/src/metabase/admin/databases/utils.ts+++ b/frontend/src/metabase/admin/databases/utils.ts@@ -23,6 +23,12 @@ export const hasActionsEnabled = (database: Pick<Database, "settings">) => {return Boolean(database.settings?.["database-enable-actions"]);};+export const hasWritableConnectionDetails = (+ database: Pick<Database, "write_data_details">,+) => {+ return database.write_data_details != null;+};+export const hasDbRoutingEnabled = (database: Pick<Database, "router_user_attribute">,) => {diff --git a/frontend/src/metabase/admin/routes.jsx b/frontend/src/metabase/admin/routes.jsxindex 378272d6e70..9dda2dc7daa 100644--- a/frontend/src/metabase/admin/routes.jsx+++ b/frontend/src/metabase/admin/routes.jsx@@ -42,6 +42,7 @@ import { DataModelV1 } from "metabase/metadata/pages/DataModelV1";import {PLUGIN_ADMIN_TOOLS,PLUGIN_ADMIN_USER_MENU_ROUTES,+ PLUGIN_ADVANCED_PERMISSIONS,PLUGIN_CACHING,PLUGIN_DB_ROUTING,PLUGIN_DEPENDENCIES,@@ -83,6 +84,7 @@ export const getRoutes = (store, CanAccessSettings, IsAdmin) => {<Route path="create" component={DatabasePage} /></Route><Route path=":databaseId/edit" component={DatabasePage} />+ {PLUGIN_ADVANCED_PERMISSIONS.getWritableConnectionInfoRoutes(IsAdmin)}<Route path=":databaseId" component={DatabaseEditApp}>{PLUGIN_DB_ROUTING.getDestinationDatabaseRoutes(IsAdmin)}</Route>diff --git a/frontend/src/metabase/admin/settings/components/SettingsPages/AuthenticationSettingsPage.tsx b/frontend/src/metabase/admin/settings/components/SettingsPages/AuthenticationSettingsPage.tsxindex 2367b8a46cd..ccd296036f0 100644--- a/frontend/src/metabase/admin/settings/components/SettingsPages/AuthenticationSettingsPage.tsx+++ b/frontend/src/metabase/admin/settings/components/SettingsPages/AuthenticationSettingsPage.tsx@@ -4,7 +4,10 @@ import { SettingsPageWrapper } from "metabase/admin/components/SettingsSection";import { UpsellSSO } from "metabase/admin/upsells";import { useGetSettingsQuery } from "metabase/api";import { hasAnySsoFeature } from "metabase/common/utils/plan";-import { PLUGIN_AUTH_PROVIDERS } from "metabase/plugins";+import {+ type AuthSettingsPageTab,+ PLUGIN_AUTH_PROVIDERS,+} from "metabase/plugins";import { Box, Flex, Stack } from "metabase/ui";import { ApiKeysAuthCard } from "../../auth/components/ApiKeysAuthCard";@@ -12,7 +15,11 @@ import { GoogleAuthCard } from "../../auth/containers/GoogleAuthCard/GoogleAuthCimport { LdapAuthCard } from "../../auth/containers/LdapAuthCard";import { ManageApiKeys } from "../ApiKeys/ManageApiKeys";-export function AuthenticationSettingsPage({ tab }: { tab: string }) {+export function AuthenticationSettingsPage({+ tab,+}: {+ tab: AuthSettingsPageTab;+}) {const hasSSO = useHasSso();if (hasSSO) {diff --git a/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.enterprise.unit.spec.tsx b/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.enterprise.unit.spec.tsxindex a270b3f7cc5..8368ff591b7 100644--- a/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.enterprise.unit.spec.tsx+++ b/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.enterprise.unit.spec.tsx@@ -1,12 +1,13 @@import { setupEnterpriseOnlyPlugin } from "__support__/enterprise";import { screen } from "__support__/ui";+import type { AuthSettingsPageTab } from "metabase/plugins";import type { EnterpriseSettings } from "metabase-types/api";import { setup as OSSSetup } from "./AuthenticationSettingsPage.setup";const setup = async (extraSettings?: Partial<EnterpriseSettings>,- tab = "authentication",+ tab: AuthSettingsPageTab = "authentication",) => {setupEnterpriseOnlyPlugin("auth");return OSSSetup(extraSettings, true, tab);diff --git a/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.setup.tsx b/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.setup.tsxindex 256aaa04300..d2408b70e64 100644--- a/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.setup.tsx+++ b/frontend/src/metabase/admin/settings/components/SettingsPages/test/AuthenticationSettingsPage.setup.tsx@@ -7,6 +7,7 @@ import {} from "__support__/server-mocks";import { renderWithProviders } from "__support__/ui";import { UndoListing } from "metabase/common/components/UndoListing";+import type { AuthSettingsPageTab } from "metabase/plugins";import type { ApiKey, EnterpriseSettings } from "metabase-types/api";import {createMockSettings,@@ -54,7 +55,7 @@ export const testApiKeys: ApiKey[] = [export const setup = async (extraSettings?: Partial<EnterpriseSettings>,isEnterprise = false,- tab = "authentication",+ tab: AuthSettingsPageTab = "authentication",) => {const settings = createMockSettings({"google-auth-enabled": false,diff --git a/frontend/src/metabase/api/database.ts b/frontend/src/metabase/api/database.tsindex 7fae1024e9f..6b2fcb6c856 100644--- a/frontend/src/metabase/api/database.ts+++ b/frontend/src/metabase/api/database.ts@@ -11,6 +11,7 @@ import type {Database,DatabaseId,Field,+ GetDatabaseHealthRequest,GetDatabaseHealthResponse,GetDatabaseMetadataRequest,GetDatabaseRequest,@@ -69,13 +70,17 @@ export const databaseApi = Api.injectEndpoints({dispatch(updateMetadata(data, DatabaseSchema)),),}),- getDatabaseHealth: builder.query<GetDatabaseHealthResponse, DatabaseId>({- query: (id) => ({+ getDatabaseHealth: builder.query<+ GetDatabaseHealthResponse,+ GetDatabaseHealthRequest+ >({+ query: ({ id, ...params }) => ({method: "GET",url: `/api/database/${id}/healthcheck`,+ params,}),// invalidate health check in the case db connection info changes- providesTags: (_, __, id) => [idTag("database", id)],+ providesTags: (_, __, { id }) => [idTag("database", id)],}),getDatabaseMetadata: builder.query<Database, GetDatabaseMetadataRequest>({query: ({ id, ...params }) => ({diff --git a/frontend/src/metabase/databases/components/DatabaseForm/DatabaseForm.tsx b/frontend/src/metabase/databases/components/DatabaseForm/DatabaseForm.tsxindex 84062a1ff44..8b3c1bd6ae8 100644--- a/frontend/src/metabase/databases/components/DatabaseForm/DatabaseForm.tsx+++ b/frontend/src/metabase/databases/components/DatabaseForm/DatabaseForm.tsx@@ -62,7 +62,6 @@ export const DatabaseForm = ({config = {},}: DatabaseFormProps): JSX.Element => {const isAdvanced = config.isAdvanced || false;- const engineFieldState = config.engine?.fieldState;const engines = useSelector(getEngines);const initialEngineKey = useMemo(() => {@@ -119,7 +118,6 @@ export const DatabaseForm = ({// casting won't be needed after migrating all usages of engineKeyengineKey={engineKey as EngineKey}engines={engines}- engineFieldState={engineFieldState}autofocusFieldName={autofocusFieldName}isAdvanced={isAdvanced}onEngineChange={handleEngineChange}diff --git a/frontend/src/metabase/databases/components/DatabaseForm/DatabaseFormBody.tsx b/frontend/src/metabase/databases/components/DatabaseForm/DatabaseFormBody.tsxindex 1db4bd13885..63b1973ed6d 100644--- a/frontend/src/metabase/databases/components/DatabaseForm/DatabaseFormBody.tsx+++ b/frontend/src/metabase/databases/components/DatabaseForm/DatabaseFormBody.tsx@@ -22,7 +22,6 @@ interface DatabaseFormBodyProps {engine: Engine | undefined;engineKey: EngineKey | undefined;engines: Record<string, Engine>;- engineFieldState?: "default" | "hidden" | "disabled";autofocusFieldName?: string;isAdvanced: boolean;onEngineChange: (engineKey: string | undefined) => void;@@ -35,7 +34,6 @@ export const DatabaseFormBody = ({engine,engineKey,engines,- engineFieldState = "default",autofocusFieldName,isAdvanced,onEngineChange,@@ -45,6 +43,7 @@ export const DatabaseFormBody = ({}: DatabaseFormBodyProps): JSX.Element => {const { setValues } = useFormikContext<DatabaseData>();const hasConnectionError = useHasConnectionError();+ const { engine: engineFieldConfig, name: nameFieldConfig } = config;const px = match(location).with("setup", () => "sm")@@ -56,14 +55,14 @@ export const DatabaseFormBody = ({return (<Box mah={mah} mb="md" px={px} style={{ overflowY: "auto" }}>- {engineFieldState !== "hidden" && (+ {engineFieldConfig?.fieldState !== "hidden" && (<><DatabaseEngineFieldengineKey={engineKey}engines={engines}isAdvanced={isAdvanced}onChange={onEngineChange}- disabled={engineFieldState === "disabled"}+ disabled={engineFieldConfig?.fieldState === "disabled"}showSampleDatabase={showSampleDatabase}/><DatabaseEngineWarning@@ -78,7 +77,7 @@ export const DatabaseFormBody = ({location={location}setValues={setValues}/>- {engine && (+ {engine && nameFieldConfig?.fieldState !== "hidden" && (<DatabaseNameFieldengine={engine}config={config}diff --git a/frontend/src/metabase/databases/types.ts b/frontend/src/metabase/databases/types.tsindex bf9f4d32b39..2cbefd96700 100644--- a/frontend/src/metabase/databases/types.ts+++ b/frontend/src/metabase/databases/types.ts@@ -26,17 +26,16 @@ export interface EngineFieldProps {export type FormLocation = "admin" | "setup" | "embedding_setup" | "full-page";-export type EngineFieldState = "default" | "hidden" | "disabled";+export type FieldState = "default" | "hidden" | "disabled";export interface DatabaseFormConfig {/** present the form with advanced configuration options */isAdvanced?: boolean;engine?: {- /** present the engine field as normal, disabled, or hidden */- fieldState?: EngineFieldState | undefined;+ fieldState?: FieldState;};name?: {- /** present the name field as a slug */+ fieldState?: FieldState;isSlug?: boolean;};}diff --git a/frontend/src/metabase/databases/utils/schema.ts b/frontend/src/metabase/databases/utils/schema.tsindex d5abfeea58e..5bce2ad9006 100644--- a/frontend/src/metabase/databases/utils/schema.ts+++ b/frontend/src/metabase/databases/utils/schema.ts@@ -29,7 +29,6 @@ export const getValidationSchema = (const definedFields = filterFieldsInAdvancedMode(flattenedFields, isAdvanced);const fields = definedFields.filter(isDetailField);const entries = fields.map((field) => [field.name, getFieldSchema(field)]);-return Yup.object({id: Yup.number(),engine: Yup.string().default(engineKey).required(Errors.required),diff --git a/frontend/src/metabase/lib/urls/admin.ts b/frontend/src/metabase/lib/urls/admin.tsindex 945941f61af..c923eb50c2c 100644--- a/frontend/src/metabase/lib/urls/admin.ts+++ b/frontend/src/metabase/lib/urls/admin.ts@@ -64,6 +64,10 @@ export function editDatabase(databaseId: DatabaseId) {return `/admin/databases/${databaseId}/edit`;}+export function editDatabaseWritableConnection(databaseId: DatabaseId) {+ return `/admin/databases/${databaseId}/write-data`;+}+type DataModelParams = {databaseId?: DatabaseId;schemaName?: SchemaName | null;diff --git a/frontend/src/metabase/plugins/components/PluginPlaceholder/PluginPlaceholder.tsx b/frontend/src/metabase/plugins/components/PluginPlaceholder/PluginPlaceholder.tsxindex cbe7124ed27..7abb0ce3c67 100644--- a/frontend/src/metabase/plugins/components/PluginPlaceholder/PluginPlaceholder.tsx+++ b/frontend/src/metabase/plugins/components/PluginPlaceholder/PluginPlaceholder.tsx@@ -6,11 +6,11 @@ interface Props {[key: string]: any;}-export function PluginPlaceholder(_props: Props): JSX.Element | null {+export function PluginPlaceholder<T = Props>(_props: T): JSX.Element | null {return null;}-export function NotFoundPlaceholder(_props: Props): JSX.Element | null {+export function NotFoundPlaceholder<T = Props>(_props: T): JSX.Element | null {return (<EmptyStateillustrationElement={<img src={NoResults} />}diff --git a/frontend/src/metabase/plugins/index.ts b/frontend/src/metabase/plugins/index.tsindex 2384bccc96e..fb95eceec47 100644--- a/frontend/src/metabase/plugins/index.ts+++ b/frontend/src/metabase/plugins/index.ts@@ -15,6 +15,8 @@ export {PLUGIN_LDAP_FORM_FIELDS,PLUGIN_IS_PASSWORD_USER,PLUGIN_ADMIN_USER_FORM_FIELDS,+ type AuthSettingsPageProps,+ type AuthSettingsPageTab,} from "./oss/auth";export {PLUGIN_CACHING,@@ -109,7 +111,10 @@ export {} from "./oss/permissions";export { PLUGIN_REMOTE_SYNC } from "./oss/remote-sync";export { PLUGIN_RESOURCE_DOWNLOADS } from "./oss/resource-downloads";-export { PLUGIN_SEMANTIC_SEARCH } from "./oss/semantic-search";+export {+ PLUGIN_SEMANTIC_SEARCH,+ type SearchSettingsWidgetProps,+} from "./oss/semantic-search";export { PLUGIN_ADMIN_SETTINGS } from "./oss/settings";export { PLUGIN_SMTP_OVERRIDE } from "./oss/smtp-override";export {diff --git a/frontend/src/metabase/plugins/oss/auth.ts b/frontend/src/metabase/plugins/oss/auth.tsindex 621f13f9d51..71fc7249de9 100644--- a/frontend/src/metabase/plugins/oss/auth.ts+++ b/frontend/src/metabase/plugins/oss/auth.ts@@ -6,9 +6,18 @@ import type { User } from "metabase-types/api";import type { GetAuthProviders } from "../types";+export type AuthSettingsPageTab =+ | "authentication"+ | "user-provisioning"+ | "api-keys";++export type AuthSettingsPageProps = {+ tab?: AuthSettingsPageTab;+};+const getDefaultPluginAuthProviders = () => ({isEnabled: () => false,- AuthSettingsPage: PluginPlaceholder,+ AuthSettingsPage: PluginPlaceholder<AuthSettingsPageProps>,UserProvisioningSettings: NotFoundPlaceholder,SettingsSAMLForm: NotFoundPlaceholder,SettingsJWTForm: NotFoundPlaceholder,diff --git a/frontend/src/metabase/plugins/oss/permissions.ts b/frontend/src/metabase/plugins/oss/permissions.tsindex 91abbcc24bf..52d258d18ab 100644--- a/frontend/src/metabase/plugins/oss/permissions.ts+++ b/frontend/src/metabase/plugins/oss/permissions.ts@@ -1,4 +1,4 @@-import type { ReactNode } from "react";+import type { ComponentType, ReactNode } from "react";import {type DataPermission,@@ -11,6 +11,7 @@ import {import { getUserIsAdmin } from "metabase/selectors/user";import type Database from "metabase-lib/v1/metadata/Database";import type {+ Database as DatabaseType,Dataset,Group,GroupPermissions,@@ -19,6 +20,7 @@ import type {} from "metabase-types/api";import type { State } from "metabase-types/store";+import { PluginPlaceholder } from "../components/PluginPlaceholder";import type { PluginGroupManagersType } from "../types";const getDefaultAdminPermissionsDatabaseRoutes = () => [];@@ -112,6 +114,10 @@ const getDefaultAdminUserMenuRoutes = (): (() => React.ReactNode)[] => [];export const PLUGIN_ADMIN_USER_MENU_ITEMS = getDefaultAdminUserMenuItems();export const PLUGIN_ADMIN_USER_MENU_ROUTES = getDefaultAdminUserMenuRoutes();+export type WritableConnectionInfoSectionProps = {+ database: DatabaseType;+};+const getDefaultAdvancedPermissions = () => ({addDatabasePermissionOptions: (permissions: any[], _database: Database) =>permissions,@@ -127,6 +133,9 @@ const getDefaultAdvancedPermissions = () => ({isRestrictivePermission: (_value: string) => false,shouldShowViewDataColumn: false,defaultViewDataPermission: DataPermissionValue.UNRESTRICTED,+ getWritableConnectionInfoRoutes: (_IsAdmin: ComponentType): ReactNode => null,+ WritableConnectionInfoSection:+ PluginPlaceholder<WritableConnectionInfoSectionProps>,});export const PLUGIN_ADVANCED_PERMISSIONS = getDefaultAdvancedPermissions();diff --git a/frontend/src/metabase/plugins/oss/semantic-search.ts b/frontend/src/metabase/plugins/oss/semantic-search.tsindex ad79a76dc0b..604b4765ad4 100644--- a/frontend/src/metabase/plugins/oss/semantic-search.ts+++ b/frontend/src/metabase/plugins/oss/semantic-search.ts@@ -1,7 +1,11 @@import { PluginPlaceholder } from "metabase/plugins/components/PluginPlaceholder";+export type SearchSettingsWidgetProps = {+ statusPollingInterval?: number;+};+const getDefaultPluginSemanticSearch = () => ({- SearchSettingsWidget: PluginPlaceholder,+ SearchSettingsWidget: PluginPlaceholder<SearchSettingsWidgetProps>,});export const PLUGIN_SEMANTIC_SEARCH = getDefaultPluginSemanticSearch();diff --git a/modules/drivers/athena/src/metabase/driver/athena.clj b/modules/drivers/athena/src/metabase/driver/athena.cljindex 8bcf0e4b600..ecc2246c2a0 100644--- a/modules/drivers/athena/src/metabase/driver/athena.clj+++ b/modules/drivers/athena/src/metabase/driver/athena.clj@@ -10,6 +10,7 @@[metabase.driver :as driver][metabase.driver-api.core :as driver-api][metabase.driver.athena.schema-parser :as athena.schema-parser]+ [metabase.driver.connection :as driver.conn][metabase.driver.sql-jdbc.common :as sql-jdbc.common][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]@@ -47,7 +48,7 @@(defmethod driver/database-supports? [:athena :schemas][_driver _feature db]- (not (seq (:dbname (:details db)))))+ (not (seq (:dbname (driver.conn/effective-details db)))));;; +----------------------------------------------------------------------------------------------------------------+;;; | metabase.driver.sql-jdbc method impls |@@ -459,18 +460,19 @@;; Because describe-table-fields might fail, we catch the error here and return an empty set of columns(defmethod driver/describe-table :athena- [driver {{:keys [catalog dbname]} :details, :as database} table]- (sql-jdbc.execute/do-with-connection-with-options- driver- database- nil- (fn [^Connection conn]- (let [metadata (.getMetaData conn)]- (assoc (select-keys table [:name :schema])- :fields (try- (describe-table-fields metadata database driver table catalog dbname)- (catch Throwable _- (set nil))))))))+ [driver database table]+ (let [{:keys [catalog dbname]} (driver.conn/effective-details database)]+ (sql-jdbc.execute/do-with-connection-with-options+ driver+ database+ nil+ (fn [^Connection conn]+ (let [metadata (.getMetaData conn)]+ (assoc (select-keys table [:name :schema])+ :fields (try+ (describe-table-fields metadata database driver table catalog dbname)+ (catch Throwable _+ (set nil)))))))))(defn- get-tables[^DatabaseMetaData metadata, ^String schema-or-nil, ^String db-name-or-nil]@@ -526,14 +528,14 @@remarks)}))))))(defmethod driver/describe-database* :athena- [driver {details :details, :as database}]+ [driver database](sql-jdbc.execute/do-with-connection-with-optionsdriverdatabasenil(fn [^Connection conn](let [metadata (.getMetaData conn)]- {:tables (fast-active-tables driver metadata details)}))))+ {:tables (fast-active-tables driver metadata (driver.conn/effective-details database))}))))(defmethod sql.qp/format-honeysql :athena[driver honeysql-form]diff --git a/modules/drivers/athena/test/metabase/driver/athena_test.clj b/modules/drivers/athena/test/metabase/driver/athena_test.cljindex ae1378678a4..81a9e3a9412 100644--- a/modules/drivers/athena/test/metabase/driver/athena_test.clj+++ b/modules/drivers/athena/test/metabase/driver/athena_test.clj@@ -445,7 +445,8 @@[true? {:dbname nil}][true? {:dbname ""}][false? {:dbname "db_name"}]]]- (is (schemas-supported? (driver/database-supports? :athena :schemas {:details details})))))+ (is (schemas-supported? (driver/database-supports? :athena :schemas {:lib/type :metadata/database+ :details details})))))(deftest ^:parallel athena-describe-database(mt/test-driver :athenadiff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex 248f264b20a..7df652219ff 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -1,5 +1,5 @@(ns metabase.driver.bigquery-cloud-sdk- (:refer-clojure :exclude [mapv some empty? not-empty get-in])+ (:refer-clojure :exclude [mapv some empty? not-empty])(:require[clojure.core.async :as a][clojure.set :as set]@@ -11,6 +11,7 @@[metabase.driver.bigquery-cloud-sdk.params :as bigquery.params][metabase.driver.bigquery-cloud-sdk.query-processor :as bigquery.qp][metabase.driver.common.table-rows-sample :as table-rows-sample]+ [metabase.driver.connection :as driver.conn][metabase.driver.settings :as driver.settings][metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc :as driver.sql-jdbc]@@ -25,7 +26,7 @@[metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [mapv some empty? not-empty get-in]]+ [metabase.util.performance :refer [mapv some empty? not-empty]]^{:clj-kondo/ignore [:discouraged-namespace]}[toucan2.core :as t2])(:import@@ -186,8 +187,10 @@(.getTable client dataset-id table-id empty-table-options)))(mu/defn- get-table :- (driver-api/instance-of-class Table)- (^Table [{{:keys [project-id]} :details, :as database} dataset-id table-id]- (get-table (database-details->client (:details database)) project-id dataset-id table-id))+ (^Table [database dataset-id table-id]+ (let [details (driver.conn/effective-details database)+ project-id (:project-id details)]+ (get-table (database-details->client details) project-id dataset-id table-id)))(^Table [^BigQuery client :- (driver-api/instance-of-class BigQuery)project-id :- [:maybe driver-api/schema.common.non-blank-string]@@ -196,8 +199,9 @@(get-table* client project-id dataset-id table-id)))(defmethod driver/table-exists? :bigquery-cloud-sdk- [_ {:keys [details] :as _database} {table-id :name, dataset-id :schema :as _table}]- (let [client (database-details->client details)+ [_ database {table-id :name, dataset-id :schema :as _table}]+ (let [details (driver.conn/effective-details database)+ client (database-details->client details)project-id (get-project-id details)](boolean(get-table* client project-id dataset-id table-id))))@@ -225,7 +229,8 @@(defn- describe-database-tables[driver database]- (let [project-id (get-project-id (:details database))+ (let [details (driver.conn/effective-details database)+ project-id (get-project-id details)query-dataset (fn [dataset-id](query-honeysqldriver@@ -249,7 +254,7 @@;; Maybe this is something we can do once we can parse sql(= "BASE TABLE" table-type)require-partition-filter))})]- (->> (list-datasets (:details database) :logging-schema-exclusions? true)+ (->> (list-datasets details :logging-schema-exclusions? true)(eduction (mapcat (fn [dataset-id] (eduction (map #(table-info dataset-id %)) (query-dataset dataset-id))))))))(defmethod driver/describe-database* :bigquery-cloud-sdk@@ -475,8 +480,9 @@(defmethod driver/describe-fields :bigquery-cloud-sdk[driver database & {:keys [schema-names table-names]}]- (let [project-id (get-project-id (:details database))- dataset-ids (or schema-names (list-datasets (:details database)))]+ (let [details (driver.conn/effective-details database)+ project-id (get-project-id details)+ dataset-ids (or schema-names (list-datasets details))];; The contract of [[driver/describe-fields]] requires results ordered by:;; `table-schema`, `table-name`, `database-position`@@ -636,7 +642,7 @@(throw-invalid-query t sql parameters)))(defn- effective-query-timezone-id [database]- (if (get-in database [:details :use-jvm-timezone])+ (if (:use-jvm-timezone (driver.conn/effective-details database))(driver-api/system-timezone-id)"UTC"))@@ -732,22 +738,23 @@;; - Waiting for the cancel-chan to get either a cancel message or to be closed.;; - Running the BigQuery execution in another thread, since it's blocking.(let [^BigQuery client (database-details->client database-details)- result-promise (promise)- request (build-bigquery-request sql parameters)- job (.create client (JobInfo/of request) (u/varargs BigQuery$JobOption))- job-id (.getJobId job)- query-future (future- ;; ensure the classloader is available within the future.- (driver-api/the-classloader)- (try- (*page-callback*)- (let [result-options (if *page-size* [(BigQuery$QueryResultsOption/pageSize *page-size*)] [])- result (.getQueryResults job (u/varargs BigQuery$QueryResultsOption result-options))]- (if result- (deliver result-promise [:ready result])- (throw (ex-info "Null response from query" {}))))- (catch Throwable t- (deliver result-promise [:error t]))))]+ result-promise (promise)+ request (build-bigquery-request sql parameters)+ _ (driver.conn/track-connection-acquisition! database-details)+ job (.create client (JobInfo/of request) (u/varargs BigQuery$JobOption))+ job-id (.getJobId job)+ query-future (future+ ;; ensure the classloader is available within the future.+ (driver-api/the-classloader)+ (try+ (*page-callback*)+ (let [result-options (if *page-size* [(BigQuery$QueryResultsOption/pageSize *page-size*)] [])+ result (.getQueryResults job (u/varargs BigQuery$QueryResultsOption result-options))]+ (if result+ (deliver result-promise [:ready result])+ (throw (ex-info "Null response from query" {}))))+ (catch Throwable t+ (deliver result-promise [:error t]))))];; This `go` is responsible for cancelling the *initial* job execution.;; Future pages may still not be fetched and so the reducer needs to check `cancel-chan` as well.@@ -779,8 +786,7 @@{:pre [(map? database) (map? (:details database))]};; automatically retry the query if it times out or otherwise fails. This is on top of the auto-retry added by;; `execute`- ;; Apply any swapped connection details (e.g., for workspace isolation)- (let [details (driver/maybe-swap-details (:id database) (:details database))+ (let [details (driver.conn/effective-details database)thunk (fn [](execute-bigqueryrespond@@ -801,7 +807,7 @@(let [database (driver-api/database (driver-api/metadata-provider))](binding [bigquery.common/*bigquery-timezone-id* (effective-query-timezone-id database)](log/tracef "Running BigQuery query in %s timezone" bigquery.common/*bigquery-timezone-id*)- (let [sql (if (get-in database [:details :include-user-id-and-hash] true)+ (let [sql (if (:include-user-id-and-hash (driver.conn/effective-details database) true)(str "-- " (driver-api/query->remark :bigquery-cloud-sdk outer-query) "\n" sql)sql)](*process-native* respond database sql params (driver-api/canceled-chan))))))@@ -880,27 +886,33 @@;; duplicated tables with nil schema. Happily only in the "dataset-id" schema and not all schemas. But just;; leave them with nil schemas and they will get deactivated in sync.(catch Exception _e))- (let [updated-db (-> (assoc-in database [:details :dataset-filters-type] "inclusion")- (assoc-in [:details :dataset-filters-patterns] dataset-id)- (m/dissoc-in [:details :dataset-id]))]- (t2/update! :model/Database db-id {:details (:details updated-db)})- updated-db)))+ (let [updated-details (-> (driver.conn/default-details database)+ (assoc :dataset-filters-type "inclusion")+ (assoc :dataset-filters-patterns dataset-id)+ (dissoc :dataset-id))]+ (t2/update! :model/Database db-id {:details updated-details})+ (assoc database :details updated-details))));; TODO: THIS METHOD SHOULD NOT BE UPDATING THE APP-DB (which it does in [convert-dataset-id-to-filters!]);; Issue: https://github.com/metabase/metabase/issues/39392+;; NOTE: This normalize is a legacy migration (OAuth -> service-account, dataset-id -> filters).+;; write-data-details is a new feature and will never contain these legacy fields, so we only+;; normalize the primary details here. The convert-dataset-id-to-filters! function has DB write+;; side effects that are specific to the primary details.(defmethod driver/normalize-db-details :bigquery-cloud-sdk- [_driver {:keys [details] :as database}]- (when-not (empty? (filter some? ((juxt :auth-code :client-id :client-secret) details)))- (log/errorf (str "Database ID %d, which was migrated from the legacy :bigquery driver to :bigquery-cloud-sdk, has"- " one or more OAuth style authentication scheme parameters saved to db-details, which cannot"- " be automatically migrated to the newer driver (since it *requires* service-account-json instead);"- " this database must therefore be updated by an administrator (by adding a service-account-json)"- " before sync and queries will work again")- (u/the-id database)))- (if-let [dataset-id (get details :dataset-id)]- (when-not (str/blank? dataset-id)- (convert-dataset-id-to-filters! database dataset-id))- database))+ [_driver database]+ (let [details (driver.conn/default-details database)]+ (when-not (empty? (filter some? ((juxt :auth-code :client-id :client-secret) details)))+ (log/errorf (str "Database ID %d, which was migrated from the legacy :bigquery driver to :bigquery-cloud-sdk, has"+ " one or more OAuth style authentication scheme parameters saved to db-details, which cannot"+ " be automatically migrated to the newer driver (since it *requires* service-account-json instead);"+ " this database must therefore be updated by an administrator (by adding a service-account-json)"+ " before sync and queries will work again")+ (u/the-id database)))+ (if-let [dataset-id (get details :dataset-id)]+ (when-not (str/blank? dataset-id)+ (convert-dataset-id-to-filters! database dataset-id))+ database)))(defmethod driver/prettify-native-form :bigquery-cloud-sdk[_ native-form]@@ -937,13 +949,17 @@(defmethod driver/create-table! :bigquery-cloud-sdk[driver database-id table-name column-definitions & {:keys [primary-key]}]- (let [sql (#'driver.sql-jdbc/create-table!-sql driver table-name column-definitions :primary-key primary-key)]- (driver/execute-raw-queries! driver (t2/select-one :model/Database database-id) [sql])))+ (let [sql (#'driver.sql-jdbc/create-table!-sql driver table-name column-definitions :primary-key primary-key)+ database (t2/select-one :model/Database database-id)+ conn-spec (driver/connection-spec driver database)]+ (driver/execute-raw-queries! driver conn-spec [sql])))(defmethod driver/drop-table! :bigquery-cloud-sdk[driver database-id table-name]- (let [sql (driver/compile-drop-table driver table-name)]- (driver/execute-raw-queries! driver (t2/select-one :model/Database database-id) [sql])))+ (let [sql (driver/compile-drop-table driver table-name)+ database (t2/select-one :model/Database database-id)+ conn-spec (driver/connection-spec driver database)]+ (driver/execute-raw-queries! driver conn-spec [sql])))(defn- convert-value-for-insertion[base-type value]@@ -992,9 +1008,10 @@(defmethod driver/insert-from-source! [:bigquery-cloud-sdk :rows][_driver db-id {table-name :name :keys [columns]} {:keys [data]}](let [col-names (map :name columns)- {:keys [details]} (t2/select-one :model/Database db-id)+ database (t2/select-one :model/Database db-id)+ details (driver.conn/effective-details database)- client (database-details->client details)+ client (database-details->client details)project-id (get-project-id details)dataset-id (namespace table-name)@@ -1008,18 +1025,22 @@(doseq [^java.util.Map row chunk](.addRow insert-request-builder (InsertAllRequest$RowToInsert/of row)))(let [insert-request (.build insert-request-builder)- response (.insertAll client insert-request)]+ response (.insertAll client insert-request)](when (.hasErrors response)(let [errors (.getInsertErrors response)](throw (ex-info "BigQuery insert failed"{:errors (into [] (map str errors))})))))))))(defmethod driver/execute-raw-queries! :bigquery-cloud-sdk- [_driver connection-details queries]- ;; connection-details is either database details directly (from transforms)- ;; or a database map with :details key (from other contexts)- (let [details (get connection-details :details connection-details)- client (database-details->client details)]+ [driver conn-spec queries]+ ;; conn-spec should be flat details (from driver/connection-spec)+ ;; Defensive: multiple callsites used to pass in a database map. If the map has+ ;; :details key, we will call driver/connection-spec on it.+ (let [details (if (:details conn-spec)+ (driver/connection-spec driver conn-spec)+ conn-spec)+ client (database-details->client details)]+ (driver.conn/track-connection-acquisition! details)(try(doall(for [query queries]@@ -1041,14 +1062,14 @@(let [qualified-name (if schema(keyword schema name)(keyword name))- drop-sql (first (driver/compile-drop-table driver qualified-name))]- (driver/execute-raw-queries! driver database [drop-sql])+ drop-sql (first (driver/compile-drop-table driver qualified-name))+ conn-spec (driver/connection-spec driver database)]+ (driver/execute-raw-queries! driver conn-spec [drop-sql])nil))(defmethod driver/connection-spec :bigquery-cloud-sdk[_driver database]- ;; Return the database details directly since we don't use a JDBC spec for bigquery- (driver/maybe-swap-details (:id database) (:details database)))+ (driver.conn/effective-details database))(defmethod driver.sql/default-schema :bigquery-cloud-sdk[_]@@ -1059,9 +1080,8 @@;; Check if dataset exists using the BigQuery API before trying to create.;; This is important for workspace isolation where the impersonated SA has;; access to an existing isolated dataset but cannot create new datasets.- (let [details (get conn-spec :details conn-spec)- client (database-details->client details)- project-id (get-project-id details)+ (let [client (database-details->client conn-spec) ;; for bigquery, connection spec *is* the details+ project-id (get-project-id conn-spec)dataset-id (DatasetId/of project-id schema)](when-not (.getDataset client dataset-id (u/varargs BigQuery$DatasetOption));; Dataset doesn't exist, try to create it@@ -1073,7 +1093,7 @@(driver-api/with-metadata-provider db-id(->> (driver-api/metadata-provider)driver-api/database- :details+ driver.conn/effective-detailslist-datasets(some #{schema}))))@@ -1366,7 +1386,7 @@(defmethod driver/init-workspace-isolation! :bigquery-cloud-sdk[_driver database workspace]- (let [details (:details database)+ (let [details (driver.conn/effective-details database)client (ws-database-details->client details)iam-client (ws-database-details->iam-client details)project-id (get-project-id details)@@ -1420,7 +1440,7 @@[_driver database workspace tables];; For BigQuery, the workspace contains the service account email in database_details(let [ws-sa-email (-> workspace :database_details :impersonate-service-account)- details (:details database)+ details (driver.conn/effective-details database)client (ws-database-details->client details)project-id (get-project-id details)]@@ -1468,7 +1488,7 @@(defmethod driver/destroy-workspace-isolation! :bigquery-cloud-sdk[_driver database workspace]- (let [details (:details database)+ (let [details (driver.conn/effective-details database)client (ws-database-details->client details)iam-client (ws-database-details->iam-client details)project-id (get-project-id details)diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.cljindex 346f1800948..748a3968c1e 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj@@ -1,7 +1,9 @@(ns metabase.driver.bigquery-cloud-sdk.common"Common utility functions and utilities for the bigquery-cloud-sdk driver and related namespaces."(:require+ [metabase.driver.connection :as driver.conn][metabase.util :as u]+ [metabase.util.log :as log][metabase.util.malli :as mu]^{:clj-kondo/ignore [:discouraged-namespace]}[toucan2.core :as t2])@@ -57,8 +59,21 @@Returns the calculated project-id (see [[database-details->credential-project-id]]) String from the credentials."{:added "0.42.0"}- ^String [{:keys [details] :as database} :- [:map [:details RequiredDetails]]]- (let [creds-proj-id (database-details->credential-project-id details)]+ ^String [database :- [:map [:details RequiredDetails]]]+ ;; :project-id-from-credentials is a database-level cache managed by this driver. We store and read it from+ ;; `:details` regardless of connection type. This is valid so long as read and write service accounts share a+ ;; project ID. See also: [[metabase.driver.bigquery-cloud-sdk.query-processor/project-id-for-current-query]]+ (let [details (driver.conn/default-details database)+ creds-proj-id (database-details->credential-project-id details)]+ (when (driver.conn/details-for-exact-type database :write-data)+ (let [write-proj-id (driver.conn/with-write-connection+ (database-details->credential-project-id+ (driver.conn/effective-details database)))]+ (when (not= creds-proj-id write-proj-id)+ (log/warnf (str "Database %d: read and write service accounts belong to different GCP projects "+ "(%s vs %s). The cached project-id-from-credentials uses the read SA's project; "+ "query qualification may be incorrect for write connections.")+ (u/the-id database) creds-proj-id write-proj-id))))(t2/update! :model/Database(u/the-id database){:details (assoc details :project-id-from-credentials creds-proj-id)})diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.cljindex 1a9105e6f7e..6880b320750 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj@@ -10,6 +10,7 @@[metabase.driver.bigquery-cloud-sdk.common :as bigquery.common][metabase.driver.common :as driver.common]^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters]+ [metabase.driver.connection :as driver.conn][metabase.driver.sql.parameters.substitution :as sql.params.substitution][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.query-processor.util :as sql.qp.u]@@ -59,12 +60,16 @@project ID associated with the service account credentials."[](when (driver-api/initialized?)- (when-let [{:keys [details], driver :engine, :as database} (driver-api/database (driver-api/metadata-provider))]+ (when-let [{driver :engine, :as database} (driver-api/database (driver-api/metadata-provider))];; this is mostly here to catch tests that do something dumb like try to run a BigQuery tests with a MBQL query;; targeting the H2 test database(when driver(assert (isa? driver/hierarchy driver :bigquery-cloud-sdk) "Sanity check: Database is not a BigQuery database"))- (let [project-id-override (:project-id details)+ ;; :project-id-from-credentials is a database-level cache managed by this driver. We store and read it from+ ;; `:details` regardless of connection type. This is valid so long as read and write service accounts share a+ ;; project ID. If they don't, [[bigquery.common/populate-project-id-from-credentials!]] will log a warning.+ (let [details (driver.conn/default-details database)+ project-id-override (:project-id details)project-id-creds (:project-id-from-credentials details)ret-fn (fn [proj-id-1 proj-id-2](when (and (some? proj-id-1) (not= proj-id-1 proj-id-2))diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.cljindex 96ae85203a9..ecc5fe9cb87 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj@@ -1379,3 +1379,4 @@(is (= ["INSERT INTO `PRODUCTS_COPY` SELECT * FROM products" nil](driver/compile-insert :bigquery-cloud-sdk {:query {:query "SELECT * FROM products"}:output-table :PRODUCTS_COPY}))))))+diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_introspection.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_introspection.cljindex 5cfad28a909..b825540f05a 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_introspection.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_introspection.clj@@ -1,16 +1,17 @@(ns metabase.driver.clickhouse-introspection- (:refer-clojure :exclude [empty? get-in])+ (:refer-clojure :exclude [empty?])(:require[clojure.java.jdbc :as jdbc][clojure.string :as str][metabase.driver :as driver][metabase.driver-api.core :as driver-api]+ [metabase.driver.connection :as driver.conn][metabase.driver.ddl.interface :as ddl.i][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.sql-jdbc.sync :as sql-jdbc.sync][metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table][metabase.util :as u]- [metabase.util.performance :refer [empty? get-in]])+ [metabase.util.performance :refer [empty?]])(:import(java.sql Connection DatabaseMetaData)))@@ -117,9 +118,10 @@(defn- get-all-tables-in-all-dbs[driver db]- (let [db-filters-patterns (set (map (comp #(ddl.i/format-name driver %) str/trim)- (remove empty? (str/split (or (get-in db [:details :db-filters-patterns]) "") #","))))- db-filters-type (get-in db [:details :db-filters-type])]+ (let [details (driver.conn/effective-details db)+ db-filters-patterns (set (map (comp #(ddl.i/format-name driver %) str/trim)+ (remove empty? (str/split (or (:db-filters-patterns details) "") #","))))+ db-filters-type (:db-filters-type details)](sql-jdbc.execute/do-with-connection-with-optionsdriver db nil(fn [^Connection conn]@@ -144,8 +146,9 @@;; but the actual sync from the UI uses :dbname(defn- get-db-name[db]- (or (get-in db [:details :dbname])- (get-in db [:details :db])))+ (let [details (driver.conn/effective-details db)]+ (or (:dbname details)+ (:db details))))(defn- get-tables-in-db[driver db]@@ -161,12 +164,12 @@(set)))(defmethod driver/describe-database* :clickhouse- [driver {{:keys [enable-multiple-db]}- :details :as db}]- {:tables- (if (boolean enable-multiple-db)- (get-all-tables-in-all-dbs driver db)- (get-tables-in-db driver db))})+ [driver db]+ (let [{:keys [enable-multiple-db]} (driver.conn/effective-details db)+ tables (if (boolean enable-multiple-db)+ (get-all-tables-in-all-dbs driver db)+ (get-tables-in-db driver db))]+ {:tables tables}))(defn- ^:private is-db-required?[field]diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.cljindex 4550a256944..1677b19ceaa 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj@@ -1,10 +1,11 @@-"Provides the info about the ClickHouse version. Extracted from the main clickhouse.clj file,- as both Driver and QP overrides require access to it, avoiding circular dependencies."(ns metabase.driver.clickhouse-version+ "Provides the info about the ClickHouse version. Extracted from the main clickhouse.clj file,+ as both Driver and QP overrides require access to it, avoiding circular dependencies."(:require[clojure.core.memoize :as memoize][metabase.driver :as driver][metabase.driver-api.core :as driver-api]+ [metabase.driver.connection :as driver.conn][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.util :as driver.u]))@@ -21,15 +22,15 @@(def ^:private ^{:arglists '([database])} get-clickhouse-version(memoize/ttl- ^{:clojure.core.memoize/args-fn (fn [[database]] (:details database))}+ ^{:clojure.core.memoize/args-fn (fn [[database]] (driver.conn/effective-details database))}(fn [database](sql-jdbc.execute/do-with-connection-with-options:clickhouse(sql-jdbc.conn/db->pooled-connection-spec database)nil(fn [^java.sql.Connection conn]- (with-open [ver-stmt (.createStatement conn)- ver-rset (.executeQuery ver-stmt clickhouse-version-query)+ (with-open [ver-stmt (.createStatement conn)+ ver-rset (.executeQuery ver-stmt clickhouse-version-query)cloud-stmt (.createStatement conn)cloud-rset (.executeQuery cloud-stmt "SELECT value='1' FROM system.settings WHERE name='cloud_mode'")](cond-> nildiff --git a/modules/drivers/databricks/src/metabase/driver/databricks.clj b/modules/drivers/databricks/src/metabase/driver/databricks.cljindex 45f9a6f9c40..8916bc62c25 100644--- a/modules/drivers/databricks/src/metabase/driver/databricks.clj+++ b/modules/drivers/databricks/src/metabase/driver/databricks.clj@@ -1,5 +1,5 @@(ns metabase.driver.databricks- (:refer-clojure :exclude [not-empty get-in])+ (:refer-clojure :exclude [not-empty])(:require[clojure.java.jdbc :as jdbc][clojure.string :as str]@@ -7,6 +7,7 @@[java-time.api :as t][metabase.driver :as driver][metabase.driver-api.core :as driver-api]+ [metabase.driver.connection :as driver.conn][metabase.driver.hive-like :as driver.hive-like][metabase.driver.sql-jdbc :as sql-jdbc][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]@@ -19,7 +20,7 @@[metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.log :as log]- [metabase.util.performance :refer [not-empty get-in]]+ [metabase.util.performance :refer [not-empty]][ring.util.codec :as codec])(:import[java.sql@@ -78,9 +79,10 @@(defmethod driver/adjust-schema-qualification :databricks[_driver database schema]- (let [multi-level? (get-in database [:details :multi-level-schema])- catalog (get-in database [:details :catalog])- prefix (str catalog ".")]+ (let [details (driver.conn/effective-details database)+ multi-level? (:multi-level-schema details)+ catalog (:catalog details)+ prefix (str catalog ".")](cond(and multi-level? (not (str/includes? schema ".")))(str prefix schema)@@ -134,7 +136,7 @@(into#{}(filter (comp included? :schema))- (sql-jdbc.execute/reducible-query database (get-tables-sql driver (:details database)))))}+ (sql-jdbc.execute/reducible-query database (get-tables-sql driver (driver.conn/effective-details database)))))}(catch Throwable e(throw (ex-info (format "Error in %s describe-database: %s" driver (ex-message e)){}diff --git a/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj b/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.cljindex ad8b346ec8b..42c6519e14b 100644--- a/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj+++ b/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj@@ -7,6 +7,7 @@[metabase.driver :as driver][metabase.driver-api.core :as driver-api][metabase.driver.common :as driver.common]+ [metabase.driver.connection :as driver.conn][metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]@@ -188,7 +189,7 @@(defmethod driver/dbms-version :druid-jdbc[_driver database]- (let [{:keys [host port]} (:details database)]+ (let [{:keys [host port]} (driver.conn/effective-details database)](try (let [version (-> (http/get (format "%s:%s/status" host port)):bodyjson/decodediff --git a/modules/drivers/druid/src/metabase/driver/druid/execute.clj b/modules/drivers/druid/src/metabase/driver/druid/execute.cljindex 9c2a13587be..ecc551aa201 100644--- a/modules/drivers/druid/src/metabase/driver/druid/execute.clj+++ b/modules/drivers/druid/src/metabase/driver/druid/execute.clj@@ -4,6 +4,7 @@[java-time.api :as t][medley.core :as m][metabase.driver-api.core :as driver-api]+ [metabase.driver.connection :as driver.conn][metabase.driver.druid.query-processor :as druid.qp][metabase.util :as u][metabase.util.date-2 :as u.date]@@ -150,12 +151,14 @@:as mbql-query}respond]{:pre [query]}- (let [details (:details (driver-api/database (driver-api/metadata-provider)))+ (let [database (-> (driver-api/metadata-provider) driver-api/database)+ details (driver.conn/effective-details database)query (if (string? query)(json/decode+kw query)query)query-type (or query-type(keyword (namespace ::druid.qp/query) (name (:queryType query))))+ _ (driver.conn/track-connection-acquisition! details)results (try(execute* details query)(catch Throwable ediff --git a/modules/drivers/druid/src/metabase/driver/druid/sync.clj b/modules/drivers/druid/src/metabase/driver/druid/sync.cljindex c48338a36c2..8d6297a8525 100644--- a/modules/drivers/druid/src/metabase/driver/druid/sync.clj+++ b/modules/drivers/druid/src/metabase/driver/druid/sync.clj@@ -3,6 +3,7 @@(:require[medley.core :as m][metabase.driver-api.core :as driver-api]+ [metabase.driver.connection :as driver.conn][metabase.driver.druid.client :as druid.client][metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh][metabase.util.performance :refer [select-keys]]))@@ -28,7 +29,7 @@"Impl of `driver/describe-table` for Druid."[database table]{:pre [(map? database) (map? table)]}- (ssh/with-ssh-tunnel [details-with-tunnel (:details database)]+ (ssh/with-ssh-tunnel [details-with-tunnel (driver.conn/effective-details database)](let [{:keys [columns aggregators]} (first (do-segment-metadata-query details-with-tunnel (:name table)))metric-column-names (set (keys aggregators))]{:schema nil@@ -53,21 +54,23 @@"Impl of `driver/describe-database` for Druid."[database]{:pre [(map? (:details database))]}- (ssh/with-ssh-tunnel [details-with-tunnel (:details database)]- (let [druid-datasources (druid.client/GET (druid.client/details->url details-with-tunnel "/druid/v2/datasources")- :auth-enabled (-> database :details :auth-enabled)- :auth-username (-> database :details :auth-username)- :auth-token-value (driver-api/secret-value-as-string :druid (:details database) "auth-token"))]- {:tables (set (for [table-name druid-datasources]- {:schema nil, :name table-name}))})))+ (let [details (driver.conn/effective-details database)]+ (ssh/with-ssh-tunnel [details-with-tunnel details]+ (let [druid-datasources (druid.client/GET (druid.client/details->url details-with-tunnel "/druid/v2/datasources")+ :auth-enabled (:auth-enabled details)+ :auth-username (:auth-username details)+ :auth-token-value (driver-api/secret-value-as-string :druid details "auth-token"))]+ {:tables (set (for [table-name druid-datasources]+ {:schema nil, :name table-name}))}))))(defn dbms-version"Impl of `driver/dbms-version` for Druid."[database]{:pre [(map? (:details database))]}- (ssh/with-ssh-tunnel [details-with-tunnel (:details database)]- (-> (druid.client/GET (druid.client/details->url details-with-tunnel "/status")- :auth-enabled (-> database :details :auth-enabled)- :auth-username (-> database :details :auth-username)- :auth-token-value (driver-api/secret-value-as-string :druid (:details database) "auth-token"))- (select-keys [:version]))))+ (let [details (driver.conn/effective-details database)]+ (ssh/with-ssh-tunnel [details-with-tunnel details]+ (-> (druid.client/GET (druid.client/details->url details-with-tunnel "/status")+ :auth-enabled (:auth-enabled details)+ :auth-username (:auth-username details)+ :auth-token-value (driver-api/secret-value-as-string :druid details "auth-token"))+ (select-keys [:version])))))diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.cljindex 14feb25369b..a03f17c8ab8 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj@@ -7,6 +7,7 @@[metabase.driver :as driver][metabase.driver-api.core :as driver-api][metabase.driver.common.table-rows-sample :as table-rows-sample]+ [metabase.driver.connection :as driver.conn][metabase.driver.mongo.connection :as mongo.connection][metabase.driver.mongo.conversion :as mongo.conversion][metabase.driver.mongo.database :as mongo.db]@@ -599,7 +600,7 @@(defmethod driver/connection-spec :mongo[_driver database]- (driver/maybe-swap-details (:id database) (:details database)))+ (driver.conn/effective-details database))(defmulti ^:private type->database-type"Internal type->database-type multimethod for MongoDB that dispatches on type."diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/connection.clj b/modules/drivers/mongo/src/metabase/driver/mongo/connection.cljindex 8923fcd5bed..01d044ea04f 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/connection.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/connection.clj@@ -4,6 +4,7 @@(:require[clojure.string :as str][metabase.driver-api.core :as driver-api]+ [metabase.driver.connection :as driver.conn][metabase.driver.mongo.database :as mongo.db][metabase.driver.mongo.util :as mongo.util][metabase.driver.settings :as driver.settings]@@ -95,6 +96,7 @@(let [db-details (mongo.db/details-normalized database)](ssh/with-ssh-tunnel [details-with-tunnel db-details](let [client (mongo.util/mongo-client (db-details->mongo-client-settings details-with-tunnel))]+ (driver.conn/track-connection-acquisition! db-details)(log/debug (u/format-color 'cyan "Opened new MongoClient."))(try(binding [*mongo-client* client]diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/database.clj b/modules/drivers/mongo/src/metabase/driver/mongo/database.cljindex b28ac309fd6..d0327dc8c51 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/database.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/database.clj@@ -3,6 +3,7 @@(:refer-clojure :exclude [empty? not-empty])(:require[metabase.driver-api.core :as driver-api]+ [metabase.driver.connection :as driver.conn][metabase.util.i18n :refer [tru]][metabase.util.performance :refer [empty? not-empty]])(:import@@ -38,13 +39,26 @@[database](let [db-details(cond- (integer? database) (driver-api/with-metadata-provider database- (:details (driver-api/database (driver-api/metadata-provider))))- (string? database) {:dbname database}- (:dbname (:details database)) (:details database) ; entire Database obj- (:dbname database) database ; connection details map only- (:conn-uri database) database ; connection URI has all the parameters- (:conn-uri (:details database)) (:details database)+ (integer? database)+ (driver-api/with-metadata-provider database+ (driver.conn/effective-details (driver-api/database (driver-api/metadata-provider))))++ (string? database)+ {:dbname database}++ ;; Full Database object - use effective-details to respect connection type+ (:dbname (:details database))+ (driver.conn/effective-details database)++ (:dbname database)+ database ; connection details map only++ (:conn-uri database)+ database ; connection URI has all the parameters++ (:conn-uri (:details database))+ (driver.conn/effective-details database)+:else(throw (ex-info (tru "Unable to to get database details."){:database database})))]diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/database_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/database_test.cljindex 1b3a2cfb3e7..06f38d040ea 100644--- a/modules/drivers/mongo/test/metabase/driver/mongo/database_test.clj+++ b/modules/drivers/mongo/test/metabase/driver/mongo/database_test.clj@@ -1,6 +1,7 @@(ns metabase.driver.mongo.database-test(:require[clojure.test :refer :all]+ [metabase.driver.connection :as driver.conn][metabase.driver.mongo.database :as mongo.db]))(deftest ^:parallel fqdn?-test@@ -17,3 +18,37 @@(testing "`dbname` is encoded in conn-uri"(is (= "some_db"(mongo.db/db-name {:conn-uri "mongodb://localhost/some_db"})))))++(deftest ^:parallel details-normalized-respects-connection-type-test+ (testing "Mongo's details-normalized uses effective-details for full database objects"+ (let [read-details {:dbname "test-db" :host "read-host"}+ write-details {:dbname "test-db" :host "write-host"}+ database {:details read-details+ :write_data_details write-details}]+ (testing "default connection type uses :details"+ (let [result (mongo.db/details-normalized database)]+ (is (= "read-host" (:host result)))))+ (testing "write connection type uses :write_data_details"+ (driver.conn/with-write-connection+ (let [result (mongo.db/details-normalized database)]+ (is (= "write-host" (:host result))))))))+ (testing "Mongo's details-normalized falls back to :details when no write details"+ (let [read-details {:dbname "test-db" :host "read-host"}+ database {:details read-details}]+ (driver.conn/with-write-connection+ (let [result (mongo.db/details-normalized database)]+ (is (= "read-host" (:host result))))))))++(deftest ^:parallel details-normalized-conn-uri-respects-connection-type-test+ (testing "Mongo's details-normalized uses effective-details for conn-uri databases"+ (let [read-details {:conn-uri "mongodb://read-host:27017/test-db"}+ write-details {:conn-uri "mongodb://write-host:27017/test-db"}+ database {:details read-details+ :write_data_details write-details}]+ (testing "default connection type uses :details"+ (let [result (mongo.db/details-normalized database)]+ (is (= "mongodb://read-host:27017/test-db" (:conn-uri result)))))+ (testing "write connection type uses :write_data_details"+ (driver.conn/with-write-connection+ (let [result (mongo.db/details-normalized database)]+ (is (= "mongodb://write-host:27017/test-db" (:conn-uri result)))))))))diff --git a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj b/modules/drivers/oracle/test/metabase/driver/oracle_test.cljindex cc0136168fd..1020c7f8e61 100644--- a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj+++ b/modules/drivers/oracle/test/metabase/driver/oracle_test.clj@@ -158,6 +158,7 @@{:name "tunnel-private-key-passphrase"}{:name "advanced-options"}{:name "destination-database"}+ {:name "write-data-connection"}{:name "auto_run_queries"}{:name "let-user-control-scheduling"}{:name "schedules.metadata_sync"}diff --git a/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj b/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.cljindex fa341ba7926..0cd48617ca0 100644--- a/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj+++ b/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj@@ -11,6 +11,7 @@[java-time.api :as t][metabase.driver :as driver][metabase.driver-api.core :as driver-api]+ [metabase.driver.connection :as driver.conn][metabase.driver.sql-jdbc.common :as sql-jdbc.common][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]@@ -599,35 +600,37 @@(jdbc/reducible-query {:connection conn} sql))))(defmethod driver/describe-database* :presto-jdbc- [driver {{:keys [catalog schema] :as _details} :details :as database}]- (sql-jdbc.execute/do-with-connection-with-options- driver- database- nil- (fn [^Connection conn]- (let [schemas (if schema #{(describe-schema driver conn catalog schema)}- (all-schemas driver conn catalog))]- {:tables (reduce set/union #{} schemas)}))))+ [driver database]+ (let [{:keys [catalog schema]} (driver.conn/effective-details database)]+ (sql-jdbc.execute/do-with-connection-with-options+ driver+ database+ nil+ (fn [^Connection conn]+ (let [schemas (if schema #{(describe-schema driver conn catalog schema)}+ (all-schemas driver conn catalog))]+ {:tables (reduce set/union #{} schemas)})))))(defmethod driver/describe-table :presto-jdbc- [driver {{:keys [catalog] :as _details} :details :as database} {schema :schema, table-name :name}]- (sql-jdbc.execute/do-with-connection-with-options- driver- database- nil- (fn [^Connection conn]- (let [sql (describe-table-sql driver catalog schema table-name)]- (log/tracef "Running statement in describe-table: %s" sql)- {:schema schema- :name table-name- :fields (into- #{}- (map-indexed (fn [idx {:keys [column type] :as _col}]- {:name column- :database-type type- :base-type (presto-type->base-type type)- :database-position idx}))- (jdbc/reducible-query {:connection conn} sql))}))))+ [driver database {schema :schema, table-name :name}]+ (let [{:keys [catalog]} (driver.conn/effective-details database)]+ (sql-jdbc.execute/do-with-connection-with-options+ driver+ database+ nil+ (fn [^Connection conn]+ (let [sql (describe-table-sql driver catalog schema table-name)]+ (log/tracef "Running statement in describe-table: %s" sql)+ {:schema schema+ :name table-name+ :fields (into+ #{}+ (map-indexed (fn [idx {:keys [column type] :as _col}]+ {:name column+ :database-type type+ :base-type (presto-type->base-type type)+ :database-position idx}))+ (jdbc/reducible-query {:connection conn} sql))})))));;; The Presto JDBC driver DOES NOT support the `.getImportedKeys` method so just return `nil` here so the `:sql-jdbc`;;; implementation doesn't try to use it.diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.cljindex 344a126a022..23494fd0db9 100644--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj@@ -1,6 +1,6 @@(ns metabase.driver.snowflake"Snowflake Driver."- (:refer-clojure :exclude [select-keys not-empty get-in mapv])+ (:refer-clojure :exclude [select-keys not-empty mapv])(:require[buddy.core.codecs :as codecs][clojure.java.jdbc :as jdbc]@@ -12,6 +12,7 @@[metabase.driver :as driver][metabase.driver-api.core :as driver-api][metabase.driver.common :as driver.common]+ [metabase.driver.connection :as driver.conn][metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc :as sql-jdbc][metabase.driver.sql-jdbc.common :as sql-jdbc.common]@@ -34,7 +35,7 @@[metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [get-in mapv not-empty select-keys]]+ [metabase.util.performance :refer [mapv not-empty select-keys]][ring.util.codec :as codec])(:import(java.io File)@@ -549,11 +550,11 @@tests (and Snowflake itself) expected `details.db`. This has since been fixed, but for legacy support we'll stillaccept either. Throw an Exception if neither key can be found."{:arglists '([database])}- [{details :details}]- ;; ignore any blank keys- (or (m/find-first (every-pred string? (complement str/blank?))- ((juxt :db :dbname) details))- (throw (Exception. (tru "Invalid Snowflake connection details: missing DB name.")))))+ [database]+ (let [details (driver.conn/effective-details database)]+ (or (m/find-first (every-pred string? (complement str/blank?))+ ((juxt :db :dbname) details))+ (throw (Exception. (tru "Invalid Snowflake connection details: missing DB name."))))))(defn- query-db-name [];; the store is always initialized when running QP queries; for some stuff like the test extensions DDL statements@@ -856,20 +857,26 @@(jdbc/query spec (format "SHOW SCHEMAS IN DATABASE \"%s\";" db))true))))+(defn- normalize-details+ "Normalize a Snowflake details map: merge regionid into account, infer use-password. Given nil, returns nil."+ [details]+ (cond-> details+ (not (str/blank? (:regionid details)))+ (-> (update :account #(str/join "." [% (:regionid details)]))+ (dissoc :regionid))++ (and (not (contains? details :use-password))+ (:password details)+ (nil? (:private-key-id details))+ (nil? (:private-key-path details))+ (nil? (:private-key-value details)))+ (assoc :use-password true)))+(defmethod driver/normalize-db-details :snowflake[_ database]- (cond-> database- (not (str/blank? (-> database :details :regionid)))- (-> (update-in [:details :account] #(str/join "." [% (-> database :details :regionid)]))- (m/dissoc-in [:details :regionid]))-- (and- (not (contains? (:details database) :use-password))- (get-in database [:details :password])- (nil? (get-in database [:details :private-key-id]))- (nil? (get-in database [:details :private-key-path]))- (nil? (get-in database [:details :private-key-value])))- (assoc-in [:details :use-password] true)))+ (-> database+ (m/update-existing :details normalize-details)+ (m/update-existing :write_data_details normalize-details)));;; If you try to read a Snowflake `timestamptz` as a String with `.getString` it always comes back in;;; `America/Los_Angeles` for some reason I cannot figure out. Let's just read them out as UTC, which is what they're@@ -919,7 +926,7 @@(defmethod driver.sql/default-database-role :snowflake[_ database]- (-> database :details :role))+ (:role (driver.conn/effective-details database)))(defmethod sql-jdbc/impl-query-canceled? :snowflake [_ e](= (sql-jdbc/get-sql-state e) "57014"))@@ -1002,9 +1009,10 @@(defmethod driver/init-workspace-isolation! :snowflake[_driver database workspace]- (let [schema-name (driver.u/workspace-isolation-namespace-name workspace)- db-name (-> database :details :db)- warehouse (-> database :details :warehouse)+ (let [details (driver.conn/effective-details database)+ schema-name (driver.u/workspace-isolation-namespace-name workspace)+ db-name (:db details)+ warehouse (:warehouse details)role-name (isolation-role-name workspace)read-user {:user (driver.u/workspace-isolation-user-name workspace):password (driver.u/random-workspace-password)}@@ -1032,8 +1040,9 @@(defmethod driver/destroy-workspace-isolation! :snowflake[_driver database workspace]- (let [schema-name (driver.u/workspace-isolation-namespace-name workspace)- db-name (-> database :details :db)+ (let [details (driver.conn/effective-details database)+ schema-name (driver.u/workspace-isolation-namespace-name workspace)+ db-name (:db details)role-name (isolation-role-name workspace)username (driver.u/workspace-isolation-user-name workspace)conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))]@@ -1049,7 +1058,7 @@(defmethod driver/grant-workspace-read-access! :snowflake[_driver database workspace tables](let [conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))- db-name (-> database :details :db)+ db-name (:db (driver.conn/effective-details database))role-name (-> workspace :database_details :role)](when-not db-name(throw (ex-info "Snowflake database configuration is missing required 'db' (database name) setting"diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.cljindex 6fb3ba3f8b2..58473e656e2 100644--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj@@ -931,6 +931,25 @@(is (=? {:password "abc" :private-key-id int? :use-password complement} (:details db2)))(is (= {:password "abc" :use-password false} (:details db3)))))))+(deftest ^:parallel normalize-write-data-details-test+ (mt/test-driver :snowflake+ (testing "write-data-details should be normalized the same way as details"+ (testing "regionid normalization"+ (mt/with-temp [:model/Database db {:name "Snowflake DB with write details"+ :engine :snowflake+ :details {:account "my-instance"}+ :write_data_details {:account "write-instance"+ :regionid "eu-west-2"}}]+ (is (= {:account "write-instance.eu-west-2"}+ (:write_data_details db)))))+ (testing "use-password inference"+ (mt/with-temp [:model/Database db {:name "Snowflake DB with write pw"+ :engine :snowflake+ :details {:account "my-instance"}+ :write_data_details {:password "secret"}}]+ (is (= {:password "secret" :use-password true}+ (:write_data_details db))))))))+(deftest ^:parallel set-role-statement-test(testing "set-role-statement should return a USE ROLE command, with the role quoted if it contains special characters";; No special charactersdiff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.cljindex e4fbc104555..ec8c4734ea0 100644--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj@@ -11,6 +11,7 @@[java-time.api :as t][metabase.driver :as driver][metabase.driver-api.core :as driver-api]+ [metabase.driver.connection :as driver.conn][metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc :as sql-jdbc][metabase.driver.sql-jdbc.common :as sql-jdbc.common]@@ -1010,8 +1011,7 @@;; Use a "role" (sqlserver user) if it exists. Do not fall back to the user;; field automatically, as it represents the login user which may not be a;; valid database user for impersonation (see issue #60665).- (let [{:keys [role]} (:details database)]- role))+ (:role (driver.conn/effective-details database)))(defmethod driver.sql/set-role-statement :sqlserver[_driver role]diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.cljindex 6f5ae8bd5b5..0c80ef82185 100644--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj@@ -709,19 +709,23 @@(deftest ^:parallel default-database-role-test(testing "SQL Server default database role handling"(testing "returns role when explicitly configured"- (let [database {:details {:user "login_user" :role "db_user"}}]+ (let [database {:lib/type :metadata/database+ :details {:user "login_user" :role "db_user"}}](is (= "db_user" (driver.sql/default-database-role :sqlserver database)))))(testing "returns nil when no role is configured"- (let [database {:details {:user "login_user"}}]+ (let [database {:lib/type :metadata/database+ :details {:user "login_user"}}](is (nil? (driver.sql/default-database-role :sqlserver database)))))(testing "returns nil even when user is 'sa'"- (let [database {:details {:user "sa"}}]+ (let [database {:lib/type :metadata/database+ :details {:user "sa"}}](is (nil? (driver.sql/default-database-role :sqlserver database)))))(testing "ignores user field and only uses role field"- (let [database {:details {:user "login_user" :role "impersonation_user"}}]+ (let [database {:lib/type :metadata/database+ :details {:user "login_user" :role "impersonation_user"}}](is (= "impersonation_user" (driver.sql/default-database-role :sqlserver database)))))))(deftest ^:parallel wtf-testdiff --git a/modules/drivers/starburst/src/metabase/driver/starburst.clj b/modules/drivers/starburst/src/metabase/driver/starburst.cljindex adbb9258a64..33eabbfcdad 100644--- a/modules/drivers/starburst/src/metabase/driver/starburst.clj+++ b/modules/drivers/starburst/src/metabase/driver/starburst.clj@@ -1,6 +1,6 @@(ns metabase.driver.starburst"starburst driver."- (:refer-clojure :exclude [select-keys get-in])+ (:refer-clojure :exclude [select-keys])(:require[clojure.java.jdbc :as jdbc][clojure.set :as set]@@ -10,6 +10,7 @@[java-time.api :as t][metabase.driver :as driver][metabase.driver-api.core :as driver-api]+ [metabase.driver.connection :as driver.conn][metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc.common :as sql-jdbc.common][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]@@ -26,7 +27,7 @@[metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :refer [trs]][metabase.util.log :as log]- [metabase.util.performance :refer [select-keys get-in]])+ [metabase.util.performance :refer [select-keys]])(:import(com.mchange.v2.c3p0 C3P0ProxyConnection)(io.trino.jdbc TrinoConnection)@@ -489,37 +490,39 @@(jdbc/reducible-result-set rs {})))))(defmethod driver/describe-database* :starburst- [driver {{:keys [catalog schema] :as _details} :details :as database}]- (sql-jdbc.execute/do-with-connection-with-options- driver- database- nil- (fn [^Connection conn]- (let [schemas (if schema- #{(describe-schema driver conn catalog schema)}- (all-schemas driver conn catalog))]- {:tables (reduce set/union #{} schemas)}))))+ [driver database]+ (let [{:keys [catalog schema]} (driver.conn/effective-details database)]+ (sql-jdbc.execute/do-with-connection-with-options+ driver+ database+ nil+ (fn [^Connection conn]+ (let [schemas (if schema+ #{(describe-schema driver conn catalog schema)}+ (all-schemas driver conn catalog))]+ {:tables (reduce set/union #{} schemas)})))))(defmethod driver/describe-table :starburst- [driver {{:keys [catalog] :as _details} :details :as database} {schema :schema, table-name :name}]- (sql-jdbc.execute/do-with-connection-with-options- driver- database- nil- (fn [^Connection conn]- (with-open [stmt (.createStatement conn)]- (let [sql (describe-table-sql driver catalog schema table-name)- rs (sql-jdbc.execute/execute-statement! driver stmt sql)]- {:schema schema- :name table-name- :fields (into- #{}- (map-indexed (fn [idx {:keys [column type] :as _col}]- {:name column- :database-type type- :base-type (starburst-type->base-type type)- :database-position idx}))- (jdbc/reducible-result-set rs {}))})))))+ [driver database {schema :schema, table-name :name}]+ (let [{:keys [catalog]} (driver.conn/effective-details database)]+ (sql-jdbc.execute/do-with-connection-with-options+ driver+ database+ nil+ (fn [^Connection conn]+ (with-open [stmt (.createStatement conn)]+ (let [sql (describe-table-sql driver catalog schema table-name)+ rs (sql-jdbc.execute/execute-statement! driver stmt sql)]+ {:schema schema+ :name table-name+ :fields (into+ #{}+ (map-indexed (fn [idx {:keys [column type] :as _col}]+ {:name column+ :database-type type+ :base-type (starburst-type->base-type type)+ :database-position idx}))+ (jdbc/reducible-result-set rs {}))}))))))(defmethod driver/db-default-timezone :starburst[driver database]@@ -993,7 +996,7 @@(defmethod driver.sql/default-database-role :starburst[_driver database]- (get-in database [:details :user]))+ (:user (driver.conn/effective-details database)))(defmethod driver/set-role! :starburst[_driver ^Connection conn role]diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yamlindex c743d48d805..95e1c9938d8 100644--- a/resources/migrations/059_update_migrations.yaml+++ b/resources/migrations/059_update_migrations.yaml@@ -3467,6 +3467,30 @@ databaseChangeLog:path: instance_analytics_views/tasks/v4/h2-tasks.sqlrelativeToChangelogFile: true+ - changeSet:+ id: v59.2026-02-16T18:00:00+ author: galdre+ comment: Add write_data_details column to metabase_database for PRO-86 separate write connections+ preConditions:+ - onFail: MARK_RAN+ - not:+ - columnExists:+ tableName: metabase_database+ columnName: write_data_details+ changes:+ - addColumn:+ tableName: metabase_database+ columns:+ - column:+ name: write_data_details+ type: ${text.type}+ remarks: Encrypted JSON containing connection details for write operations (transforms, etc.)+ constraints:+ nullable: true+ rollback:+ - dropColumn:+ tableName: metabase_database+ columnName: write_data_details# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex de28d960b96..d21ea10788e 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -136,12 +136,12 @@(defn- conn-pool-bean-diag-info [acc ^ObjectName jmx-bean];; We should not be using specific driver implementations- (let [pool-var (requiring-resolve 'metabase.driver.sql-jdbc.connection/database-id->connection-pool)]+ (let [pool-var (requiring-resolve 'metabase.driver.sql-jdbc.connection/pool-cache-key->connection-pool)];; Using this `locking` is non-obvious but absolutely required to avoid the deadlock inside c3p0 implementation. The;; act of JMX attribute reading first locks a DynamicPooledDataSourceManagerMBean object, and then a;; PoolBackedDataSource object. Conversely, the act of creating a pool (with;; com.mchange.v2.c3p0.DataSources/pooledDataSource) first locks PoolBackedDataSource and then- ;; DynamicPooledDataSourceManagerMBean. We have to lock a common monitor (which `database-id->connection-pool` is)+ ;; DynamicPooledDataSourceManagerMBean. We have to lock a common monitor (which `pool-cache-key->connection-pool` is);; to prevent the deadlock. Hopefully.;; Issue against c3p0: https://github.com/swaldman/c3p0/issues/95(locking @pool-var@@ -265,7 +265,7 @@{:description "Number of times this instance converted a card's MBQL 4 to 5 and `clean` made a real change"})(prometheus/gauge :metabase-database/status{:description "Does a given database using driver pass a health check."- :labels [:driver :healthy :reason]})+ :labels [:driver :healthy :reason :connection-type]})(prometheus/counter :metabase-query-processor/query{:description "Did a query run by a specific driver succeed or fail":labels [:driver :status]})@@ -501,6 +501,13 @@(prometheus/counter :metabase-token-check/attempt{:description "Total number of token checks. Includes a status label.":labels [:status]})+ ;; Write-connection telemetry+ (prometheus/counter :metabase-db-connection/write-op+ {:description "JDBC connection pool acquisitions by connection type (default or write-data)."+ :labels [:connection-type]})+ (prometheus/counter :metabase-db-connection/type-resolved+ {:description "Write-data details resolved by effective-details (driver-agnostic). Only incremented when write-data-details are genuinely used, not on fallback or workspace swap."+ :labels [:connection-type]});; SQL parsing metrics(prometheus/counter :metabase-sql-parsing/context-timeouts{:description "Number of Python/GraalVM SQL parsing execution timeouts."})diff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex 155a90d7672..218d572c8ed 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -21,7 +21,6 @@[metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.json :as json]- [metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.performance :refer [mapv empty?]]@@ -63,91 +62,6 @@[driver & body]`(do-with-driver ~driver (fn [] ~@body)))-;;; +----------------------------------------------------------------------------------------------------------------+-;;; | Connection Details Swapping |-;;; +----------------------------------------------------------------------------------------------------------------+--(def ^:private ^:dynamic *swapped-connection-details*- "A dynamic var that holds a map of database-id -> swapped-details-map for temporarily swapping connection details.- When a connection spec is created for a database, if its ID is present in this map, the swap map will be- merged into the connection `:details` before they are used to create a connection.-- This provides a mechanism for temporarily using different connection details (e.g., using alternative credentials- for workspaces) without mutating the database record.-- The swap map is merged into the database `:details` map. The swap is applied before any connection-specific- processing (like hash calculation for connection pooling), so different swaps will result in different- connection pools.-- Different drivers may apply this swap at different points in their connection lifecycle, but the semantics- are consistent: swapped details are used for the duration of the dynamic scope.-- See [[with-swapped-connection-details]] for usage."- nil)--(defn- apply-detail-swaps- "Merges the `swap-map` into `details`. Supports nested maps via deep merge."- [details swap-map]- (reduce-kv- (fn [acc k v]- (if (and (map? v) (map? (get acc k)))- (assoc acc k (apply-detail-swaps (get acc k) v))- (assoc acc k v)))- details- swap-map))--(defn has-connection-swap?- "Returns true if there is an active connection detail swap for `database-id`."- [database-id]- (contains? *swapped-connection-details* database-id))--(defn maybe-swap-details- "Returns the database details with any swaps applied from [[*swapped-connection-details*]].- If no swap exists for `database-id`, returns `details` unchanged.-- Drivers should call this function when creating connections to apply any active swaps.- For JDBC drivers, this is called in [[metabase.driver.sql-jdbc.connection/db->pooled-connection-spec]].- For other drivers (e.g., MongoDB), this should be called in their connection creation logic."- [database-id details]- (if-let [swap-map (get *swapped-connection-details* database-id)]- (do- (log/debugf "Applying swapped connection details for database %d, swap keys: %s"- database-id (keys swap-map))- (apply-detail-swaps details swap-map))- details))--(defn do-with-swapped-connection-details- "Implementation for [[with-swapped-connection-details]]."- [database-id swap-map thunk]- (when (contains? *swapped-connection-details* database-id)- (throw (ex-info "Nested connection detail swaps are not supported for the same database"- {:database-id database-id})))- (log/debugf "Entering swapped connection details scope for database %d, swap keys: %s"- database-id (keys swap-map))- (binding [*swapped-connection-details* (assoc *swapped-connection-details* database-id swap-map)]- (thunk)))--(defmacro with-swapped-connection-details- "Temporarily swap the connection details for a specific database within the dynamic scope of `body`.-- The `swap-map` is a map of detail keys to swap values. These will be merged into the database's- connection `:details` map. Nested maps are deep-merged.-- Any code that creates a connection for `database-id` within this scope will use the modified details.-- **Important:** Nested swaps for the same database are not supported and will throw an exception.- Different databases can have concurrent swaps.-- Example:-- ;; Swap connection to use alternate credentials- (driver/with-swapped-connection-details 1 {:user \"workspace-user\" :password \"workspace-pass\"}- ;; All connections created in this scope use the swapped credentials- (qp/process-query query))"- {:style/indent 2}- [database-id swap-map & body]- `(do-with-swapped-connection-details ~database-id ~swap-map (fn [] ~@body)))-;;; +----------------------------------------------------------------------------------------------------------------+;;; | Driver Registration / Hierarchy / Multimethod Dispatch |;;; +----------------------------------------------------------------------------------------------------------------+@@ -1235,7 +1149,12 @@regular Liquibase queries. This multimethod will be called from a `:post-select` handler within the database model.The full `database` model object is passed as the 2nd parameter, and the multimethod implementation is expected toupdate the value for `:details`. The default implementation is essentially `identity` (i.e returns `database`- unchanged). This multimethod will only be called if `:details` is actually present in the `database` map."+ unchanged). This multimethod will only be called if `:details` is actually present in the `database` map.++ Implementations should normalize both `:details` and `:write-data-details` (if present), since+ `:write-data-details` is merged on top of `:details` by [[metabase.driver.connection/effective-details]].+ Un-normalized fields in `:write-data-details` can leak through to the merged result. See+ [[metabase.driver.connection]] for more information."{:added "0.41.0" :arglists '([driver database])}dispatch-on-initialized-driver:hierarchy #'hierarchy)diff --git a/src/metabase/driver/common.clj b/src/metabase/driver/common.cljindex 0b6f4e226f7..12d82d09a91 100644--- a/src/metabase/driver/common.clj+++ b/src/metabase/driver/common.clj@@ -1,16 +1,15 @@(ns metabase.driver.common"Shared definitions and helper functions for use across different drivers."- (:refer-clojure :exclude [get-in])#_{:clj-kondo/ignore [:metabase/modules]}(:require[clojure.string :as str][metabase.driver :as driver]+ [metabase.driver.connection :as driver.conn][metabase.premium-features.core :as premium-features][metabase.settings.core :as setting][metabase.util.i18n :refer [deferred-tru]][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [get-in]][metabase.warehouses.core :as warehouses]))(set! *warn-on-reflection* true)@@ -130,6 +129,12 @@:type :hidden:default false})+(def write-data-connection-option+ "Map representing the 'is this a writable connection' option"+ {:name "write-data-connection"+ :type :hidden+ :default false})+(def advanced-options-start"Map representing the start of the advanced option section in a DB connection form. Fields in this section shouldhave their visibility controlled using the `visible-if` property."@@ -147,7 +152,8 @@:description (deferred-tru(str "We execute the underlying query when you explore data using Summarize or Filter. ""This is on by default but you can turn it off if performance is slow."))- :visible-if {"advanced-options" true}})+ :visible-if {"advanced-options" true+ "write-data-connection" false}})(def let-user-control-scheduling"Map representing the `let-user-control-scheduling` option in a DB connection form."@@ -155,7 +161,8 @@:type :boolean:display-name (deferred-tru "Choose when syncs and scans happen"):description (deferred-tru "By default, Metabase does a lightweight hourly sync and an intensive daily scan of field values. If you have a large database, turn this on to make changes.")- :visible-if {"advanced-options" true}})+ :visible-if {"advanced-options" true+ "write-data-connection" false}})(def metadata-sync-schedule"Map representing the `schedules.metadata_sync` option in a DB connection form, which should be only visible if@@ -184,7 +191,8 @@{:name "json-unfolding":display-name (deferred-tru "Allow unfolding of JSON columns"):type :boolean- :visible-if {"advanced-options" true}+ :visible-if {"advanced-options" true+ "write-data-connection" false}:description (deferred-tru(str "This enables unfolding JSON columns into their component fields. ""Disable unfolding if performance is slow. If enabled, you can still disable unfolding for "@@ -199,7 +207,8 @@:description (deferred-tru(str "This enables Metabase to scan for additional field values during syncs allowing smarter ""behavior, like improved auto-binning on your bar charts."))- :visible-if {"advanced-options" true}})+ :visible-if {"advanced-options" true+ "write-data-connection" false}})(def multi-level-schema"Map representing the `multi-level-schema` option for databases. Stores schemas with multiple levels of hierarchy."@@ -208,8 +217,14 @@:default false})(def default-advanced-options- "Vector containing the three most common options present in the advanced option section of the DB connection form."- [destination-database-option auto-run-queries let-user-control-scheduling metadata-sync-schedule cache-field-values-schedule refingerprint])+ "Vector containing the most common options present in the advanced option section of the DB connection form."+ [destination-database-option+ write-data-connection-option+ auto-run-queries+ let-user-control-scheduling+ metadata-sync-schedule+ cache-field-values-schedule+ refingerprint])(def default-options"Default options listed above, keyed by name. These keys can be listed in the plugin manifest to specify connection@@ -406,7 +421,7 @@;; This allows adding support for nested-field-columns for drivers in the future and;; have json-unfolding enabled by default, without;; needing a migration to add the `json-unfolding=true` key to the database details.- (let [json-unfolding (get-in database [:details :json-unfolding])]+ (let [json-unfolding (:json-unfolding (driver.conn/effective-details database))](if (nil? json-unfolding)truejson-unfolding)))diff --git a/src/metabase/driver/connection.clj b/src/metabase/driver/connection.cljnew file mode 100644index 00000000000..022dccfef2e--- /dev/null+++ b/src/metabase/driver/connection.clj@@ -0,0 +1,174 @@+(ns metabase.driver.connection+ "Centralized access to database connection details.++ Direct access to `(:details database)` is an anti-pattern. It couples callers to+ the raw data layout, which means any change to how details are resolved — connection+ routing, write credentials, workspace isolation, security boundaries — requires+ finding and updating every call site. This namespace provides the indirection that+ makes those changes possible.++ Primary API: [[effective-details]], [[with-write-connection]], [[default-details]]."+ (:require+ [metabase.analytics.prometheus :as prometheus]+ [metabase.driver.connection.workspaces :as driver.w]+ [metabase.driver.util :as driver.u]+ [metabase.util :as u]+ [metabase.util.log :as log]+ [metabase.util.malli.registry :as mr]))++;; Database `:details` is a single encrypted JSON column that stores everything a driver+;; needs to know about a connection. The problem is that "everything a driver needs to know"+;; has grown to become a big blob:+;;+;; - Credentials: `password`, `secret_key`, `service-account-json`, `private-key`,+;; `tunnel-private-key`, `access-token`, and friends. These are what you'd expect+;; to find in an encrypted column.+;;+;; - Connection identifiers: `host`, `port`, `db`, `user`, `account`, `warehouse`.+;; These identify where and as whom to connect. Not credentials per se, but together+;; with the above they form a complete connection string.+;;+;; - Configuration and preferences: `auto_run_queries`, `let-user-control-scheduling`,+;; `schedules.metadata_sync`, `refingerprint`, `json-unfolding`, `schema-filters`.+;; These control Metabase behavior. They do not participate in authentication or+;; connection establishment in any way.+;;+;; All three categories live in the same encrypted blob, which means `t2/select :model/Database`+;; decrypts AWS secret keys just so the sync scheduler can check whether the user wants to+;; control their own sync timing. Every code path that touches any detail — even purely+;; cosmetic ones — carries the full credential payload through memory.+;;+;; Across 19 drivers it seems there are roughly 80 distinct field names that could live in+;; `:details`, with inconsistent naming conventions (snake_case, kebab-case, etc. together).+;; Some fields are ambiguous: `additional-options`, `kerberos-keytab-path`, etc. The existing+;; `sensitive-fields` function in `driver.util` identifies credential-type fields well, but+;; there's no corresponding classification for "this field doesn't need to be encrypted at all."+;;+;; This namespace centralizes all reads of `:details` (and `:write-data-details`) behind+;; functions that can apply connection-type routing, workspace isolation, and — eventually —+;; a proper separation of credentials from configuration. The immediate goal is to make+;; direct `(:details database)` access a greppable code smell. The longer-term goal is to+;; make it possible to store non-sensitive configuration outside the encrypted column+;; without a codebase-wide refactor. The immediate need for this namespace was to make it+;; possible to access and *use* configuration in a controllable and auditable way as we+;; are going from one master-connection to a database to two + any number of dynamic+;; workspaces connections.+;;+;; Please use/adapt/augment/improve this namespace and avoid all such patterns:+;; - (:details database)+;; - (let [{:keys [details]} database])+;; - (let [{{:keys [user]} :details} database])++(mr/def ::connection-type+ [:enum :default :write-data])++(def ^:dynamic *connection-type*+ "Which connection details [[effective-details]] should resolve.++ - `:default` — primary `:details`+ - `:write-data` — `:write-data-details` merged over `:details` (if configured)++ Bind via [[with-write-connection]], not directly."+ :default)++(defmacro with-write-connection+ "Establishes a write-connection context for body.++ [[effective-details]] calls within this scope resolve to take `:write-data-details`+ into account (if configured) instead of only primary `:details`."+ [& body]+ `(binding [*connection-type* :write-data]+ ~@body))++(def ^:dynamic ^:private *suppress-resolution-telemetry*+ false)++(defmacro without-resolution-telemetry+ "Suppresses [[effective-details]] from incrementing the `:metabase-db-connection/type-resolved`+ Prometheus counter. Use for infrastructure calls (e.g., health checks) that resolve+ write-data-details but should not inflate feature-usage metrics."+ [& body]+ `(binding [*suppress-resolution-telemetry* true]+ ~@body))++(defn effective-details+ "Returns the connection details map appropriate for the current context.++ Accepts a database (Toucan2 instance or lib/metadata). Returns nil for nil input.++ By default, returns the primary `:details`. Within a [[with-write-connection]] scope,+ takes `:write-data-details` into account (if configured). Within a+ [[driver.w/with-swapped-connection-details]] scope, applies workspace isolation+ overrides on top."+ [database]+ (when-let [database (some-> database driver.u/ensure-lib-database)]+ (let [write? (= *connection-type* :write-data)+ write-details (when write? (:write-data-details database))+ base (merge (:details database) write-details)]+ ;; Track when write-data-details are genuinely used (not fallback, not workspace-swapped).+ ;; Default resolutions are not tracked here — see :metabase-db-connection/write-op for+ ;; pool-level connection acquisition metrics.+ (when (and write-details+ (not *suppress-resolution-telemetry*)+ (not (driver.w/has-connection-swap? (:id database))))+ (try (prometheus/inc! :metabase-db-connection/type-resolved {:connection-type "write-data"})+ (catch Exception _ nil)))+ (-> (driver.w/maybe-swap-details (:id database) base)+ (assoc ::effective-connection-type (if write-details :write-data :default))+ (assoc ::database-id (u/id database))))))++(defn details-for-exact-type+ "Returns the details map for exactly the given connection-type, with no fallback or merging.++ Unlike [[effective-details]], `:write-data` returns only `:write-data-details` (possibly nil),+ not a merge with `:details`. Use when you need to inspect or update a specific details map+ without resorting to raw key access."+ [database connection-type]+ (let [database (driver.u/ensure-lib-database database)]+ (case connection-type+ :default (:details database)+ :write-data (:write-data-details database))))++(defn write-connection-requested?+ "True if currently executing within a [[with-write-connection]] scope."+ []+ (= *connection-type* :write-data))++(defn effective-connection-type+ "Returns the connection type actually in effect for the given database. Return value+ matches malli schema [[::connection-type]].++ Returns `:write-data` only when a write connection is both *requested*+ ([[with-write-connection]]) and *configured* (the database has `:write-data-details`).+ Otherwise returns `:default`, avoiding duplicate resource allocation (e.g. connection+ pools) when the requested type would resolve identically to `:default`."+ [database]+ (if (and (= *connection-type* :write-data)+ (some? (:write-data-details (driver.u/ensure-lib-database database))))+ :write-data+ :default))++(defn track-connection-acquisition!+ "Increments a Prometheus counter tracking connection acquisitions by connection type+ and logs the connection type + database ID at DEBUG.++ Accepts a map of connection details, expected to be the output of [[effective-details]]++ Call at the point where a driver actually obtains a connection (e.g., pool checkout).+ Non-JDBC drivers that manage their own connections should call this explicitly."+ [connection-details]+ (if-let [conn-type (::effective-connection-type connection-details)]+ (do+ (log/debugf "Acquiring %s connection for db %s" conn-type (::database-id connection-details))+ (try (prometheus/inc! :metabase-db-connection/write-op {:connection-type (name conn-type)})+ (catch Exception _ nil)))+ (log/warnf "%s was unable to determine connection type" `track-connection-acquisition!)))++(defn default-details+ "Returns primary `:details`, ignoring [[*connection-type*]].++ For operations that must always use the base configuration regardless of context —+ configuration migrations, admin-initiated writes, credential caching. For normal+ driver operations, prefer [[effective-details]]."+ [database]+ (:details database))diff --git a/src/metabase/driver/connection/workspaces.clj b/src/metabase/driver/connection/workspaces.cljnew file mode 100644index 00000000000..8f397050a3c--- /dev/null+++ b/src/metabase/driver/connection/workspaces.clj@@ -0,0 +1,84 @@+(ns metabase.driver.connection.workspaces+ (:require+ [metabase.util.log :as log]))++(def ^:private ^:dynamic *swapped-connection-details*+ "A dynamic var that holds a map of database-id -> swapped-details-map for temporarily swapping connection details.+ When a connection spec is created for a database, if its ID is present in this map, the swap map will be+ merged into the connection `:details` before they are used to create a connection.++ This provides a mechanism for temporarily using different connection details (e.g., using alternative credentials+ for workspaces) without mutating the database record.++ The swap map is merged into the database `:details` map. The swap is applied before any connection-specific+ processing (like hash calculation for connection pooling), so different swaps will result in different+ connection pools.++ Different drivers may apply this swap at different points in their connection lifecycle, but the semantics+ are consistent: swapped details are used for the duration of the dynamic scope.++ See [[with-swapped-connection-details]] for usage."+ nil)++(defn- apply-detail-swaps+ "Merges the `swap-map` into `details`. Supports nested maps via deep merge."+ [details swap-map]+ (reduce-kv+ (fn [acc k v]+ (if (and (map? v) (map? (get acc k)))+ (assoc acc k (apply-detail-swaps (get acc k) v))+ (assoc acc k v)))+ details+ swap-map))++(defn has-connection-swap?+ "Returns true if there is an active connection detail swap for `database-id`."+ [database-id]+ (contains? *swapped-connection-details* database-id))++(defn maybe-swap-details+ "Returns the database details with any swaps applied from [[*swapped-connection-details*]].+ If no swap exists for `database-id`, returns `details` unchanged.++ Drivers should call this function when creating connections to apply any active swaps.+ For JDBC drivers, this is called in [[metabase.driver.sql-jdbc.connection/db->pooled-connection-spec]].+ For other drivers (e.g., MongoDB), this should be called in their connection creation logic."+ [database-id details]+ (if-let [swap-map (get *swapped-connection-details* database-id)]+ (do+ (log/debugf "Applying swapped connection details for database %d, swap keys: %s"+ database-id (keys swap-map))+ (apply-detail-swaps details swap-map))+ details))++(defn do-with-swapped-connection-details+ "Implementation for [[with-swapped-connection-details]]."+ [database-id swap-map thunk]+ (when (contains? *swapped-connection-details* database-id)+ (throw (ex-info "Nested connection detail swaps are not supported for the same database"+ {:database-id database-id})))+ (log/debugf "Entering swapped connection details scope for database %d, swap keys: %s"+ database-id (keys swap-map))+ (binding [*swapped-connection-details* (assoc *swapped-connection-details* database-id swap-map)]+ (thunk)))++(defmacro with-swapped-connection-details+ "Temporarily swap the connection details for a specific database within the dynamic scope of `body`.++ The `swap-map` is a map of detail keys to swap values. These will be merged into the database's+ connection `:details` map. Nested maps are deep-merged.++ Any code that creates a connection for `database-id` within this scope will use the modified details.++ **Important:** Nested swaps for the same database are not supported and will throw an exception.+ Different databases can have concurrent swaps.++ Example:++ ;; Swap connection to use alternate credentials+ (driver/with-swapped-connection-details 1 {:user \"workspace-user\" :password \"workspace-pass\"}+ ;; All connections created in this scope use the swapped credentials+ (qp/process-query query))"+ {:style/indent 2}+ [database-id swap-map & body]+ `(do-with-swapped-connection-details ~database-id ~swap-map (fn [] ~@body)))diff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.cljindex 08ea620ca1f..3c1fba0c5c8 100644--- a/src/metabase/driver/h2.clj+++ b/src/metabase/driver/h2.clj@@ -1,5 +1,5 @@(ns metabase.driver.h2- (:refer-clojure :exclude [some every? get-in])+ (:refer-clojure :exclude [some every?])(:require[clojure.java.jdbc :as jdbc][clojure.math.combinatorics :as math.combo]@@ -9,6 +9,7 @@[metabase.driver :as driver][metabase.driver-api.core :as driver-api][metabase.driver.common :as driver.common]+ [metabase.driver.connection :as driver.conn][metabase.driver.h2.actions :as h2.actions][metabase.driver.settings :as driver.settings][metabase.driver.sql :as sql]@@ -28,7 +29,7 @@[metabase.util.i18n :refer [deferred-tru tru]][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [every? get-in some]])+ [metabase.util.performance :refer [every? some]])(:import(java.sql Clob Connection ResultSet ResultSetMetaData SQLException Statement)(java.time OffsetTime)@@ -172,7 +173,7 @@;; connection string. We don't allow SQL execution on H2 databases for the default admin account for security;; reasons(when (= (keyword query-type) :native)- (let [{:keys [details]} (driver-api/database (driver-api/metadata-provider))+ (let [details (-> (driver-api/metadata-provider) driver-api/database driver.conn/effective-details)user (db-details->user details)](when (and config/is-prod? ;; we elevated permissions in workspace tests(or (str/blank? user)@@ -705,7 +706,7 @@username (driver.u/workspace-isolation-user-name workspace)password (driver.u/random-workspace-password);; H2 embeds credentials in the :db connection string, so we need to build a new one- original-db (get-in database [:details :db])+ original-db (:db (driver.conn/effective-details database))new-db (replace-credentials original-db username password)](jdbc/with-db-transaction [t-conn (sql-jdbc.conn/db->pooled-connection-spec (:id database))](with-open [^Statement stmt (.createStatement ^Connection (:connection t-conn))]diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.cljindex b0a69e73c15..418a6815fed 100644--- a/src/metabase/driver/mysql.clj+++ b/src/metabase/driver/mysql.clj@@ -12,6 +12,7 @@[metabase.driver :as driver][metabase.driver-api.core :as driver-api][metabase.driver.common :as driver.common]+ [metabase.driver.connection :as driver.conn][metabase.driver.mysql.actions :as mysql.actions][metabase.driver.mysql.ddl :as mysql.ddl][metabase.driver.sql :as driver.sql]@@ -1187,7 +1188,7 @@(defmethod driver.sql/default-database-role :mysql[_driver database]- (-> database :details :role))+ (-> database driver.conn/effective-details :role))(defmethod driver.sql/set-role-statement :mysql[_driver role]@@ -1209,7 +1210,7 @@(let [db (try(driver-api/database (driver-api/metadata-provider))(catch Throwable _ nil))- tiny-int-1-is-bit? (not (some-> db :details :additional-options (str/includes? "tinyInt1isBit=false")))+ tiny-int-1-is-bit? (not (some-> db driver.conn/effective-details :additional-options (str/includes? "tinyInt1isBit=false")))db-type-name (.getColumnTypeName rsmeta column-index)precision (try(.getPrecision rsmeta column-index)diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.cljindex 66ec7de2e0c..49159e0fa31 100644--- a/src/metabase/driver/sql.clj+++ b/src/metabase/driver/sql.clj@@ -121,7 +121,7 @@(defn- create-table-and-insert-data![driver transform-details conn-spec]- (let [create-query (driver/compile-transform driver transform-details)+ (let [create-query (driver/compile-transform driver transform-details)rows-affected (last (driver/execute-raw-queries! driver conn-spec [create-query]))]rows-affected))@@ -131,9 +131,9 @@old-temp (driver.u/temp-table-name driver output-table)](try(let [new-temp-details (assoc transform-details :output-table new-temp)- rows-affected (create-table-and-insert-data! driver new-temp-details conn-spec)]+ rows-affected (create-table-and-insert-data! driver new-temp-details conn-spec)](driver/rename-tables! driver (:id database) {output-table old-temp- new-temp output-table})+ new-temp output-table})(driver/drop-table! driver (:id database) old-temp){:rows-affected rows-affected})(catch Exception e@@ -146,7 +146,7 @@(let [tmp-table (driver.u/temp-table-name driver output-table)](try(let [tmp-table-details (assoc transform-details :output-table tmp-table)- rows-affected (create-table-and-insert-data! driver tmp-table-details conn-spec)]+ rows-affected (create-table-and-insert-data! driver tmp-table-details conn-spec)](driver/drop-table! driver (:id database) output-table)(driver/rename-table! driver (:id database) tmp-table output-table){:rows-affected rows-affected})@@ -169,7 +169,7 @@[driver {:keys [conn-spec output-table database] :as transform-details} _opts](let [table-exists? (driver/table-exists? driver database{:schema (namespace output-table)- :name (name output-table)})]+ :name (name output-table)})](cond(or (not table-exists?)(driver/database-supports? driver :create-or-replace-table database))diff --git a/src/metabase/driver/sql_jdbc/connection.clj b/src/metabase/driver/sql_jdbc/connection.cljindex 27a2432f9a4..119c4e772b7 100644--- a/src/metabase/driver/sql_jdbc/connection.clj+++ b/src/metabase/driver/sql_jdbc/connection.clj@@ -6,6 +6,8 @@[clojure.java.jdbc :as jdbc][metabase.driver :as driver][metabase.driver-api.core :as driver-api]+ [metabase.driver.connection :as driver.conn]+ [metabase.driver.connection.workspaces :as driver.w][metabase.driver.settings :as driver.settings][metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh][metabase.driver.util :as driver.u]@@ -90,19 +92,19 @@[driver database]{;; only fetch one new connection at a time, rather than batching fetches (default = 3 at a time). This is done in;; interest of minimizing memory consumption- "acquireIncrement" 1+ "acquireIncrement" 1;; Never retry instead of the default of retrying 30 times (#51176);; While a couple queries may fail during a reboot, this should allow quicker recovery and less spinning on outdated;; credentials;; However, keep 1 retry for the tests to reduce flakiness.- "acquireRetryAttempts" (if driver-api/is-test? 1 0)+ "acquireRetryAttempts" (if driver-api/is-test? 1 0);; [From dox] Seconds a Connection can remain pooled but unused before being discarded.- "maxIdleTime" (* 3 60 60) ; 3 hours+ "maxIdleTime" (* 3 60 60) ; 3 hours;; In the case of serverless databases, we don't want to periodically;; wake them up to keep a connection open (#58373).- "minPoolSize" 0- "initialPoolSize" 0- "maxPoolSize" (driver.settings/jdbc-data-warehouse-max-connection-pool-size)+ "minPoolSize" 0+ "initialPoolSize" 0+ "maxPoolSize" (driver.settings/jdbc-data-warehouse-max-connection-pool-size);; [From dox] If true, an operation will be performed at every connection checkout to verify that the connection is;; valid. [...] ;; Testing Connections in checkout is the simplest and most reliable form of Connection testing,;; but for better performance, consider verifying connections periodically using `idleConnectionTestPeriod`. [...]@@ -118,7 +120,7 @@;; request. IRL the Metabase server and data warehouse are likely to be located in closer geographical proximity to;; one another than my trans-contintental tests. Thus in the majority of cases the overhead should be next to;; nothing, and in the worst case close to imperceptible.- "testConnectionOnCheckout" true+ "testConnectionOnCheckout" true;; [From dox] Number of seconds that Connections in excess of minPoolSize should be permitted to remain idle in the;; pool before being culled. Intended for applications that wish to aggressively minimize the number of open;; Connections, shrinking the pool back towards minPoolSize if, following a spike, the load level diminishes and@@ -126,7 +128,7 @@;; if the parameter is to have any effect.;;;; Kill idle connections above the minPoolSize after 5 minutes.- "maxIdleTimeExcessConnections" (* 5 60)+ "maxIdleTimeExcessConnections" (* 5 60);; [From dox] Seconds. If set, if an application checks out but then fails to check-in [i.e. close()] a Connection;; within the specified period of time, the pool will unceremoniously destroy() the Connection. This permits;; applications with occasional Connection leaks to survive, rather than eventually exhausting the Connection@@ -137,7 +139,7 @@;; This should be the same as the query timeout. This theoretically shouldn't happen since the QP should kill;; things after a certain timeout but it's better to be safe than sorry -- it seems like in practice some;; connections disappear into the ether- "unreturnedConnectionTimeout" (driver.settings/jdbc-data-warehouse-unreturned-connection-timeout-seconds)+ "unreturnedConnectionTimeout" (driver.settings/jdbc-data-warehouse-unreturned-connection-timeout-seconds);; [From dox] If true, and if unreturnedConnectionTimeout is set to a positive value, then the pool will capture;; the stack trace (via an Exception) of all Connection checkouts, and the stack traces will be printed when;; unreturned checked-out Connections timeout. This is intended to debug applications with Connection leaks, that@@ -159,10 +161,10 @@"see stacktraces in the logs.")));; Set the data source name so that the c3p0 JMX bean has a useful identifier, which incorporates the DB ID, driver,;; and name from the details- "dataSourceName" (format "db-%d-%s-%s"- (u/the-id database)- (name driver)- (data-source-name driver (:details database)))})+ "dataSourceName" (format "db-%d-%s-%s"+ (u/the-id database)+ (name driver)+ (data-source-name driver (driver.conn/effective-details database)))})(defn- connection-pool-spec"Like [[connection-pool/connection-pool-spec]] but also handles situations when the unpooled spec is a `:datasource`."@@ -171,7 +173,7 @@{:datasource (DataSources/pooledDataSource datasource (driver-api/map->properties pool-properties))}(driver-api/connection-pool-spec spec pool-properties)))-(defn ^:private default-ssh-tunnel-target-port [driver]+(defn ^:private default-ssh-tunnel-target-port [driver](when-let [port-info (some#(when (= "port" (:name %)) %)(driver/connection-properties driver))]@@ -184,11 +186,15 @@(select-keys spec-or-details [:tunnel-enabled :tunnel-session :tunnel-tracker :tunnel-entrance-port :tunnel-entrance-host]))(defn- create-pool!- "Create a new C3P0 `ComboPooledDataSource` for connecting to the given `database`."- [{:keys [id details], driver :engine, :as database}]+ "Create a new C3P0 `ComboPooledDataSource` for connecting to the given `database`.+ Uses [[driver.conn/effective-details]] to select the appropriate connection details+ based on the current [[driver.conn/*connection-type*]]."+ [{:keys [id], driver :engine, :as database}]{:pre [(map? database)]}- (log/debug (u/format-color :cyan "Creating new connection pool for %s database %s ..." driver id))- (let [details-with-tunnel (driver/incorporate-ssh-tunnel-details ;; If the tunnel is disabled this returned unchanged+ (log/debug (u/format-color :cyan "Creating new connection pool for %s database %s (connection-type: %s) ..."+ driver id driver.conn/*connection-type*))+ (let [details (driver.conn/effective-details database)+ details-with-tunnel (driver/incorporate-ssh-tunnel-details ;; If the tunnel is disabled this returned unchangeddriver(update details :port #(or % (default-ssh-tunnel-target-port driver))))details-with-auth (driver.u/fetch-and-incorporate-auth-provider-details@@ -216,8 +222,8 @@15)(defonce ^:private ^{:doc "A map of our currently open connection pools for canonical (non-swapped) connections,- keyed by database-id. Each database has at most one canonical pool."}- database-id->connection-pool+ keyed by pool cache key `[database-id, connection-type]`. Each database+connection-type pair has at most one canonical pool."}+ pool-cache-key->connection-pool(atom {}))(defonce ^:private ^Cache ^{:doc "A Guava cache of swapped connection pools with TTL-based expiration,@@ -237,53 +243,70 @@nil))))(build)))-(defonce ^:private ^{:doc "A map of DB details hash values for the canonical (non-swapped) details, keyed by Database `:id`.+(defonce ^:private ^{:doc "A map of DB details hash values for the canonical (non-swapped) details,+ keyed by pool cache key `[database-id, connection-type]`.This is used to detect when database details have been updated in the application database."}- database-id->jdbc-spec-hash+ pool-cache-key->jdbc-spec-hash(atom {}))+(defn- pool-cache-key+ "Returns the cache key for connection pools: `[database-id, connection-type]`.+ Uses [[driver.conn/effective-connection-type]] so that a requested write connection+ without configured `:write-data-details` resolves to `:default`, reusing the+ existing pool instead of creating a duplicate."+ [database]+ [(u/the-id database) (driver.conn/effective-connection-type database)])+(mu/defn- jdbc-spec-hash- "Computes a hash value for the JDBC connection spec based on `database`'s `:details` map, for the purpose of- determining if details changed and therefore the existing connection pool needs to be invalidated."- [{driver :engine, :keys [details], :as database} :- [:maybe :map]]+ "Computes a hash value for the JDBC connection spec based on the effective connection details, for the purpose of+ determining if details changed and therefore the existing connection pool needs to be invalidated.+ Uses [[driver.conn/effective-details]] to select the appropriate details based on [[driver.conn/*connection-type*]]."+ [{driver :engine, :as database} :- [:maybe :map]](when (some? database)- (hash (connection-details->spec driver details))))+ (hash (connection-details->spec driver (driver.conn/effective-details database)))))(defn- set-canonical-pool!- "Atomically update the canonical connection pool for a database.+ "Atomically update the canonical connection pool for a database and connection type.Use this function instead of modifying connection pool atoms directly because it properly closes down old pools in a- thread-safe way, ensuring no more than one canonical pool is ever open for a specific database.+ thread-safe way, ensuring no more than one canonical pool is ever open for a specific database and connection type.This function is only for canonical (non-swapped) pools. Swapped pools are managed by the Guava cache."- [database-id details-hash pool-spec-or-nil]- {:pre [(integer? database-id) (some? details-hash)]}- (let [[old-pool-map] (if pool-spec-or-nil- (swap-vals! database-id->connection-pool assoc database-id pool-spec-or-nil)- (swap-vals! database-id->connection-pool dissoc database-id))]+ [cache-key details-hash pool-spec-or-nil]+ {:pre [(vector? cache-key) (some? details-hash)]}+ (let [[database-id _connection-type] cache-key+ [old-pool-map] (if pool-spec-or-nil+ (swap-vals! pool-cache-key->connection-pool assoc cache-key pool-spec-or-nil)+ (swap-vals! pool-cache-key->connection-pool dissoc cache-key))];; if we replaced a different pool with the new pool that is different from the old one, destroy the old pool- (when-let [old-pool-spec (get old-pool-map database-id)]+ (when-let [old-pool-spec (get old-pool-map cache-key)](when-not (identical? old-pool-spec pool-spec-or-nil)(destroy-pool! database-id old-pool-spec))));; Update canonical hash cache- (swap! database-id->jdbc-spec-hash assoc database-id details-hash)+ (swap! pool-cache-key->jdbc-spec-hash assoc cache-key details-hash)nil)(defn invalidate-pool-for-db!- "Invalidates all connection pools for the given database (canonical and swapped) by closing them and removing from cache."+ "Invalidates all connection pools for the given database (canonical -- all connection types -- and swapped) by closing+ them and removing from cache."[database]- (let [db-id (u/the-id database)- has-canonical? (contains? @database-id->connection-pool db-id)+ (let [db-id (u/the-id database)+ canonical-keys [[db-id :default] [db-id :write-data]]+ pool-map @pool-cache-key->connection-pool+ canonical-count (count (filter pool-map canonical-keys));; Find all swapped pool keys for this database (keys are [db-id, details-hash] tuples)- swapped-keys (filter (fn [[cached-db-id _details-hash]]- (= cached-db-id db-id))- (keys (.asMap ^Cache swapped-connection-pools)))]- (log/debugf "Invalidating connection pools for database %d (canonical: %s, swapped count: %d)"- db-id has-canonical? (count swapped-keys))- ;; Clear canonical pool- (when-let [pool-spec (get @database-id->connection-pool db-id)]+ swapped-keys (filter (fn [[cached-db-id _details-hash]]+ (= cached-db-id db-id))+ (keys (.asMap ^Cache swapped-connection-pools)))]+ (log/debugf "Invalidating connection pools for database %d (canonical count: %d, swapped count: %d)"+ db-id canonical-count (count swapped-keys))+ ;; Clear canonical pools for both connection types+ (doseq [cache-key canonical-keys+ :let [[old-map] (swap-vals! pool-cache-key->connection-pool dissoc cache-key)+ pool-spec (get old-map cache-key)]+ :when pool-spec](destroy-pool! db-id pool-spec)- (swap! database-id->connection-pool dissoc db-id))+ (swap! pool-cache-key->jdbc-spec-hash dissoc cache-key));; Clear all swapped pools for this DB (removal listener will call destroy-pool!)(doseq [cache-key swapped-keys](.invalidate ^Cache swapped-connection-pools cache-key))))@@ -381,26 +404,30 @@(defn- canonical-pool-hash-changed?"Check if the canonical pool's hash differs from the expected hash.- Handles stale DatabaseInstance by re-fetching from app DB."- [database-id expected-hash]- (let [curr-hash (get @database-id->jdbc-spec-hash database-id)]+ Handles stale DatabaseInstance by re-fetching from app DB.+ `cache-key` is a `[database-id, connection-type]` tuple."+ [cache-key expected-hash]+ (let [database-id (first cache-key)+ curr-hash (get @pool-cache-key->jdbc-spec-hash cache-key)](when (and (some? curr-hash) (not= curr-hash expected-hash));; the hash didn't match, but it's possible that a stale instance of `DatabaseInstance`;; was passed in (ex: from a long-running sync operation); fetch the latest one from;; our app DB, and see if it STILL doesn't match- (not= curr-hash (-> (t2/select-one [:model/Database :id :engine :details] :id database-id)+ (not= curr-hash (-> (t2/select-one [:model/Database :id :engine :details :write_data_details] :id database-id)jdbc-spec-hash)))))(defn- get-canonical-pool- "Get a canonical pool if it exists and is valid, otherwise return nil."- [database-id details-hash log-invalidation?]- (let [pool-spec (get @database-id->connection-pool database-id ::not-found)]+ "Get a canonical pool if it exists and is valid, otherwise return nil.+ `cache-key` is a `[database-id, connection-type]` tuple."+ [cache-key details-hash log-invalidation?]+ (let [database-id (first cache-key)+ pool-spec (get @pool-cache-key->connection-pool cache-key ::not-found)](cond(= ::not-found pool-spec)nil;; Check if the hash has changed (details were updated in DB)- (canonical-pool-hash-changed? database-id details-hash)+ (canonical-pool-hash-changed? cache-key details-hash)(do(when log-invalidation?(log-pool-invalidation! database-id :hash-changed))@@ -412,7 +439,8 @@(defn db->pooled-connection-spec"Return a JDBC connection spec that includes a c3p0 `ComboPooledDataSource`. These connection pools are cached so we- don't create multiple ones for the same DB.+ don't create multiple ones for the same DB and connection type. The connection type is determined by+ [[driver.conn/*connection-type*]] - use [[driver.conn/with-write-connection]] to get a write connection pool.When [[metabase.driver/with-swapped-connection-details]] is active for a database, the database details aremodified before creating the connection pool. Swapped pools are stored in a separate Guava cache with TTL-based@@ -425,21 +453,23 @@(u/id db-or-id-or-spec)(let [database-id (u/the-id db-or-id-or-spec);; we need the Database instance no matter what (in order to calculate details hash)- db-original (or (when (driver-api/instance-of? :model/Database db-or-id-or-spec)+ db (or (when (driver-api/instance-of? :model/Database db-or-id-or-spec)(driver-api/instance->metadata db-or-id-or-spec :metadata/database))(when (= (:lib/type db-or-id-or-spec) :metadata/database)db-or-id-or-spec)(driver-api/with-metadata-provider database-id(driver-api/database (driver-api/metadata-provider))))- ;; Apply connection detail swaps if present- has-swap? (driver/has-connection-swap? database-id)- db (update db-original :details #(driver/maybe-swap-details database-id %))- ;; Calculate hash from final (possibly swapped) details+ cache-key (pool-cache-key db)+ ;; Check for workspace detail swaps (for pool routing: canonical atom vs Guava TTL cache).+ ;; The actual swap is applied inside effective-details, not here.+ has-swap? (driver.w/has-connection-swap? database-id)+ ;; Calculate hash from effective details (includes write-connection merge + workspace swap)details-hash (jdbc-spec-hash db)]+ (driver.conn/track-connection-acquisition! (driver.conn/effective-details db))(cond;; for the audit db, we pass the datasource for the app-db. This lets us use fewer db;; connections with *application-db* and 1 less connection pool. Note: This data-source is- ;; not in [[database-id->connection-pool]].+ ;; not in [[pool-cache-key->connection-pool]].(or (:is-audit db) (get-in db [:details :is-audit-dev])){:datasource (driver-api/data-source)}@@ -451,17 +481,17 @@:else(or;; we have an existing valid pool for this database, so use it- (get-canonical-pool database-id details-hash true)+ (get-canonical-pool cache-key details-hash true);; We don't want to end up with a bunch of simultaneous threads creating pools only to have them destroyed;; the very next instant. This will cause their queries to fail. Thus we should do the usual locking here;; and make sure only one thread will be creating a pool at a given instant.- (locking database-id->connection-pool+ (locking pool-cache-key->connection-pool(or;; check if another thread created the pool while we were waiting to acquire the lock- (get-canonical-pool database-id details-hash false)+ (get-canonical-pool cache-key details-hash false);; create a new pool and add it to our cache, then return it(u/prog1 (create-pool! db)- (set-canonical-pool! database-id details-hash <>)))))))+ (set-canonical-pool! cache-key details-hash <>)))))));; already a `clojure.java.jdbc` spec map(map? db-or-id-or-spec)diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.cljindex c39a0e003b2..d535e7fd2d9 100644--- a/src/metabase/driver/sql_jdbc/execute.clj+++ b/src/metabase/driver/sql_jdbc/execute.clj@@ -14,6 +14,7 @@[medley.core :as m][metabase.driver :as driver][metabase.driver-api.core :as driver-api]+ [metabase.driver.connection :as driver.conn][metabase.driver.settings :as driver.settings][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute.diagnostic :as sql-jdbc.execute.diagnostic]@@ -773,7 +774,7 @@[driver {{sql :query, params :params} :native, :as outer-query} _context respond]{:pre [(string? sql) (seq sql)]}(let [database (driver-api/database (driver-api/metadata-provider))- sql (if (get-in database [:details :include-user-id-and-hash] true)+ sql (if (get-in (driver.conn/effective-details database) [:include-user-id-and-hash] true)(->> (driver-api/query->remark driver outer-query)(inject-remark driver sql))sql)diff --git a/src/metabase/driver/sql_jdbc/sync/describe_table.clj b/src/metabase/driver/sql_jdbc/sync/describe_table.cljindex 9225400f05c..12b4dd89f2a 100644--- a/src/metabase/driver/sql_jdbc/sync/describe_table.clj+++ b/src/metabase/driver/sql_jdbc/sync/describe_table.clj@@ -10,6 +10,7 @@[metabase.driver :as driver][metabase.driver-api.core :as driver-api][metabase.driver.common.table-rows-sample :as table-rows-sample]+ [metabase.driver.connection :as driver.conn][metabase.driver.settings :as driver.settings][metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]@@ -370,7 +371,7 @@(if (or (and schema-names (empty? schema-names))(and table-names (empty? table-names)))[]- (let [sql (describe-fields-sql driver (assoc args :details (:details db)))]+ (let [sql (describe-fields-sql driver (assoc args :details (driver.conn/effective-details db)))](try(log/debugf "`describe-fields` sql query:\n```\n%s\n```\n`describe-fields` args:\n```\n%s\n```"(driver/prettify-native-form driver (first sql))@@ -400,7 +401,7 @@[](eduction(map (fn [col] (select-keys col [:table-schema :table-name :field-name])))- (sql-jdbc.execute/reducible-query db (describe-indexes-sql driver (assoc args :details (:details db)))))))+ (sql-jdbc.execute/reducible-query db (describe-indexes-sql driver (assoc args :details (driver.conn/effective-details db)))))))(defn- describe-table-fks*[_driver ^Connection conn {^String schema :schema, ^String table-name :name} & [^String db-name-or-nil]]@@ -438,7 +439,7 @@(if (or (and schema-names (empty? schema-names))(and table-names (empty? table-names)))[]- (sql-jdbc.execute/reducible-query db (describe-fks-sql driver (assoc args :details (:details db))))))+ (sql-jdbc.execute/reducible-query db (describe-fks-sql driver (assoc args :details (driver.conn/effective-details db))))))(defn describe-table-indexes"Default implementation of [[metabase.driver/describe-table-indexes]] for SQL JDBC drivers. Uses JDBC DatabaseMetaData."diff --git a/src/metabase/driver/sync.clj b/src/metabase/driver/sync.cljindex 271d34ce58d..d2ef36e2386 100644--- a/src/metabase/driver/sync.clj+++ b/src/metabase/driver/sync.clj@@ -2,6 +2,7 @@"General functions and utilities for sync operations across multiple drivers."(:require[clojure.string :as str]+ [metabase.driver.connection :as driver.conn][metabase.driver.util :as driver.u])(:import(java.util.regex Pattern)))@@ -60,9 +61,10 @@([database](let [{prop-name :name} (driver.u/find-schema-filters-prop (driver.u/database->driver database))](db-details->schema-filter-patterns prop-name database)))- ([prop-nm {db-details :details :as _database}]+ ([prop-nm database](when prop-nm- (let [schema-filter-type (get db-details (keyword (str prop-nm "-type")))+ (let [db-details (driver.conn/effective-details database)+ schema-filter-type (get db-details (keyword (str prop-nm "-type")))schema-filter-patterns (get db-details (keyword (str prop-nm "-patterns")))](case schema-filter-type"exclusion" [nil schema-filter-patterns]diff --git a/src/metabase/driver/util.clj b/src/metabase/driver/util.cljindex d621710d5a9..6a2dcc1a6e1 100644--- a/src/metabase/driver/util.clj+++ b/src/metabase/driver/util.clj@@ -223,8 +223,10 @@[driver feature (mdb/unique-identifier) (:id database) (:updated-at database)])))));;; this can get called in post-select which doesn't always have ID-(mu/defn- ensure-lib-database :- [:map- [:lib/type [:= :metadata/database]]]+(mu/defn ensure-lib-database :- [:map+ [:lib/type [:= :metadata/database]]]+ "Ensures the database is in MLv2 metadata format (SnakeHatingMap with kebab-case keys).+ If passed a Toucan2 instance, converts it. If already MLv2 metadata, returns as-is."[database :- [:or[:map[:lib/type [:= :metadata/database]]]@@ -729,6 +731,21 @@(into default-sensitive-fields (map (comp keyword :name) password-fields)))default-sensitive-fields))+(defn fields-hidden-for-write-data-connection+ "Returns the set of field names (strings) that should NOT appear in `write_data_details` for the given `driver`.+ These are fields whose resolved `visible-if` includes `\"write-data-connection\" false`, meaning they are hidden+ when the write-data-connection form marker is true."+ [driver]+ (when-some [conn-prop-fn (get-method driver/connection-properties driver)]+ (let [all-props (conn-prop-fn driver)+ resolved (connection-props-server->client driver all-props)+ props-by-name (collect-all-props-by-name resolved)]+ (into #{}+ (keep (fn [[field-name {:keys [visible-if]}]]+ (when (false? (get visible-if "write-data-connection"))+ field-name)))+ props-by-name))))+(defn fetch-and-incorporate-auth-provider-details"Incorporates auth-provider responses with db-details.diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.cljindex fb816492fbe..1e4182d15bd 100644--- a/src/metabase/lib_be/metadata/jvm.clj+++ b/src/metabase/lib_be/metadata/jvm.clj@@ -79,7 +79,7 @@#_resolved-query clojure.lang.IPersistentMap][query-type model parsed-args honeysql](merge (next-method query-type model parsed-args honeysql)- {:select [:id :engine :name :dbms_version :settings :is_audit :details :timezone :router_database_id]}))+ {:select [:id :engine :name :dbms_version :settings :is_audit :details :write_data_details :timezone :router_database_id]}))(t2/define-after-select :metadata/database[database]diff --git a/src/metabase/secrets/models/secret.clj b/src/metabase/secrets/models/secret.cljindex 89fcdf2bc29..790d11c6e66 100644--- a/src/metabase/secrets/models/secret.clj+++ b/src/metabase/secrets/models/secret.clj@@ -277,18 +277,19 @@(defn delete-orphaned-secrets!"Delete Secret instances from the app DB, that will become orphaned when `database` is deleted. For now, this will- simply delete any Secret whose ID appears in the details blob, since every Secret instance that is currently created- is exclusively associated with a single Database.+ simply delete any Secret whose ID appears in the details blobs (both `:details` and `:write_data_details`), since+ every Secret instance that is currently created is exclusively associated with a single Database.In the future, if/when we allow arbitrary association of secret instances to database instances, this will need tochange and become more complicated (likely by consulting a many-to-many join table)."- [{:keys [id details] :as database}]+ [{:keys [id details write_data_details] :as database}](when-let [possible-secret-prop-names (seq (keys (secret-conn-props-by-name (driver.u/database->driver database))))](doseq [secret-id (reduce (fn [acc prop-name]- (if-let [secret-id (get details (->id-kw prop-name))]- (conj acc secret-id)- acc))- []+ (let [id-kw (->id-kw prop-name)]+ (into acc+ (keep #(get % id-kw))+ [details write_data_details])))+ #{}possible-secret-prop-names)](log/infof "Deleting secret ID %s from app DB because the owning database (%s) is being deleted" secret-id id)(t2/delete! :model/Secret :id secret-id))))@@ -325,17 +326,16 @@"To satisfy clients we need to return the keys they send us in details.This is a transformation on `:model/Database` `to-json`- Fetches the stored secret and fills in `-path` `-options` `-value` for each secret property"+ Fetches the stored secret and fills in `-path` `-options` `-value` for each secret property.+ Operates on both `:details` and `:write_data_details`."[database]- (let [driver (driver.u/database->driver database)]- (m/update-existing- database- :details- (fn [details]- (reduce-over-details-secret-values- driver- details- hydrate-redacted-secret)))))+ (let [driver (driver.u/database->driver database)+ hydrate (fn [details]+ (reduce-over-details-secret-values driver details hydrate-redacted-secret))]+ ;; Very low-level operation here, so not using driver.conn/* utils:+ (-> database+ (m/update-existing :details hydrate)+ (m/update-existing :write_data_details hydrate))))(defn clean-secret-properties-from-details"Ensures that all possible secret property values are removed from `:details`.@@ -350,21 +350,52 @@(apply dissoc db-details (vals (->possible-secret-property-names conn-prop-nm))))))(defn clean-secret-properties-from-database- "Ensures that all possible secret property values are removed from `:details`.+ "Ensures that all possible secret property values are removed from `:details` and `:write_data_details`.This is a transformation on `:model/Database` `results-transform`."[database]- (m/update-existing- database- :details- #(clean-secret-properties-from-details- %- (driver.u/database->driver database))))+ (let [clean #(clean-secret-properties-from-details % (driver.u/database->driver database))]+ ;; Very low-level operation here, so not using driver.conn/* utils:+ (-> database+ (m/update-existing :details clean)+ (m/update-existing :write_data_details clean))))++(defn- handle-secrets-for-details-key+ "Process secret-type connection properties in `details-key` of `database`, converting raw secret+ values into Secret records and storing only the secret ID. Returns the updated database."+ [database details-key]+ (if-let [details (get database details-key)]+ (let [original-details (get (t2/original database) details-key)+ updated-details (reduce-over-details-secret-values+ (driver.u/database->driver database)+ details+ (fn [db-details conn-prop-nm conn-prop]+ (let [kws (->possible-secret-property-names conn-prop-nm)+ id-kw (keyword (str conn-prop-nm "-id"))+ secret-id (get original-details id-kw)+ secret (secret-map-from-details db-details conn-prop)+ cleared-details (apply dissoc db-details (vals kws))]+ (if secret+ (if (:value secret)+ (let [{:keys [id]} (upsert-secret-value!+ secret-id+ (format "%s for %s" (:display-name conn-prop) (:name database))+ (:secret-kind conn-prop)+ (:source secret)+ (:value secret))]+ (assoc cleared-details id-kw id))+ (do+ (t2/delete! :model/Secret :id secret-id)+ (dissoc cleared-details id-kw)))+ ;; Don't throw out a secret even if the client didn't send it back+ (m/assoc-some cleared-details id-kw secret-id)))))]+ (assoc database details-key updated-details))+ database))(defn handle-incoming-client-secrets!- "Converts incoming secret values in `:details` into Secrets.+ "Converts incoming secret values in `:details` and `:write_data_details` into Secrets.This is a transformation on `:model/Database` `before-insert` and `before-update`.- Only the Secret id should be stored in `:details`. All other secret props should be cleared.+ Only the Secret id should be stored in the details maps. All other secret props should be cleared.A secret prop like `{:type :secret, :secret-kind :pem-cert :name :private-key}` can get expanded by[[driver.u/connection-props-server->client]] into these connection properties:@@ -378,29 +409,7 @@In this case, we want to look for a `:private-key-id` in the `original-details` which would indicatethat we have stored a Secret before. We upsert the secret-value (which could be `-path` or `-value`).We clear out all the possible secret-keys `-value`, `-options`, `-path` and store the `-id`."- [{:keys [details] :as database}]- (let [{original-details :details} (t2/original database)- updated-details (reduce-over-details-secret-values- (driver.u/database->driver database)- details- (fn [db-details conn-prop-nm conn-prop]- (let [kws (->possible-secret-property-names conn-prop-nm)- id-kw (keyword (str conn-prop-nm "-id"))- secret-id (get original-details id-kw)- secret (secret-map-from-details db-details conn-prop)- cleared-details (apply dissoc db-details (vals kws))]- (if secret- (if (:value secret)- (let [{:keys [id]} (upsert-secret-value!- secret-id- (format "%s for %s" (:display-name conn-prop) (:name database))- (:secret-kind conn-prop)- (:source secret)- (:value secret))]- (assoc cleared-details id-kw id))- (do- (t2/delete! :model/Secret :id secret-id)- (dissoc cleared-details id-kw)))- ;; Don't throw out a secret even if the client didn't sent it back- (m/assoc-some cleared-details id-kw secret-id)))))]- (assoc database :details updated-details)))+ [database]+ (-> database+ (handle-secrets-for-details-key :details)+ (handle-secrets-for-details-key :write_data_details)))diff --git a/src/metabase/transforms/api/transform.clj b/src/metabase/transforms/api/transform.cljindex aed523ae748..2eb6f0b8021 100644--- a/src/metabase/transforms/api/transform.clj+++ b/src/metabase/transforms/api/transform.clj@@ -424,6 +424,15 @@:limit (request/limit)))(update :data #(map transforms.util/localize-run-timestamps %))))+(api.macros/defendpoint :get "/run/:run-id" :- TransformRunResponse+ "Get a transform run by ID."+ [{:keys [run-id]} :- [:map+ [:run-id ms/PositiveInt]]]+ (api/check-data-analyst)+ (let [run (api/check-404 (t2/select-one :model/TransformRun :id run-id))]+ (-> (t2/hydrate run [:transform :collection :transform_tag_ids])+ transforms.util/localize-run-timestamps)))+(defn update-transform!"Update a transform. Validates features, database support, cycles, and target conflicts.Returns the updated transform with hydrated associations."diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.cljindex ebd2b65b457..e53845b5814 100644--- a/src/metabase/transforms/query_impl.clj+++ b/src/metabase/transforms/query_impl.clj@@ -2,6 +2,7 @@(:require[metabase.database-routing.core :as database-routing][metabase.driver :as driver]+ [metabase.driver.connection :as driver.conn][metabase.driver.util :as driver.u][metabase.lib.schema.common :as schema.common][metabase.query-processor.compile :as qp.compile]@@ -54,40 +55,45 @@([transform] (run-mbql-transform! transform nil))([{:keys [id source target owner_user_id creator_id] :as transform} {:keys [run-method start-promise user-id]}](try- (let [db (get-in source [:query :database])+ (let [db (get-in source [:query :database]){driver :engine :as database} (t2/select-one :model/Database db);; important to test routing before calling compile-source (whose qp middleware will also throw)- _ (throw-if-db-routing-enabled transform driver database)- transform-details {:db-id db- :database database- :transform-id id- :transform-type (keyword (:type target))- :conn-spec (driver/connection-spec driver database)- :query (transforms.util/compile-source transform)- :output-schema (:schema target)- :output-table (transforms.util/qualified-table-name driver target)}- opts (transform-opts transform-details)- features (transforms.util/required-database-features transform)- ;; For manual runs, use the triggering user; for cron, use owner/creator- run-user-id (if (and (= run-method :manual) user-id)- user-id- (or owner_user_id creator_id))]- (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)- (throw (ex-info "The database does not support the requested transform target type."- {:driver driver, :database database, :features features})))- ;; mark the execution as started and notify any observers- (let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method run-user-id)]- (when start-promise- (deliver start-promise [:started run-id]))- (log/info "Executing transform" id "with target" (pr-str target))- (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]- (transforms.util/run-cancelable-transform!- run-id driver transform-details- (fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))- (transforms.util/handle-transform-complete!- :run-id run-id- :transform transform- :db database)))+ _ (throw-if-db-routing-enabled transform driver database)]+ (driver.conn/with-write-connection+ (let [conn-spec (driver/connection-spec driver database)+ transform-details {:db-id db+ :database database+ :transform-id id+ :transform-type (keyword (:type target))+ :conn-spec conn-spec+ :query (transforms.util/compile-source transform)+ :output-schema (:schema target)+ :output-table (transforms.util/qualified-table-name driver target)}+ opts (transform-opts transform-details)+ features (transforms.util/required-database-features transform)+ ;; For manual runs, use the triggering user; for cron, use owner/creator+ run-user-id (if (and (= run-method :manual) user-id)+ user-id+ (or owner_user_id creator_id))]+ (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)+ (throw (ex-info "The database does not support the requested transform target type."+ {:driver driver, :database database, :features features})))+ ;; mark the execution as started and notify any observers+ (let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method run-user-id)]+ (when start-promise+ (deliver start-promise [:started run-id]))+ (log/info "Executing transform" id "with target" (pr-str target)+ (when (driver.conn/write-connection-requested?)+ " using write connection"))+ (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]+ (transforms.util/run-cancelable-transform!+ run-id driver transform-details+ (fn [_cancel-chan]+ (driver/run-transform! driver transform-details opts))))+ (transforms.util/handle-transform-complete!+ :run-id run-id+ :transform transform+ :db database)))))(catch Throwable t(if (= :already-running (:error (ex-data t)))(log/warnf "Transform %d is already running" id)diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 19e12b3ac5c..212f616498a 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -8,6 +8,7 @@[metabase.app-db.core :as mdb][metabase.audit-app.core :as audit][metabase.driver :as driver]+ [metabase.driver.connection :as driver.conn][metabase.driver.impl :as driver.impl][metabase.driver.settings :as driver.settings][metabase.driver.util :as driver.u]@@ -50,6 +51,7 @@(t2/deftransforms :model/Database{:details mi/transform-encrypted-json+ :write_data_details mi/transform-encrypted-json:engine mi/transform-keyword:metadata_sync_schedule mi/transform-cron-string:cache_field_values_schedule mi/transform-cron-string@@ -220,45 +222,68 @@(defn maybe-test-and-migrate-details!"When a driver has db-details to test and migrate:we loop through them until we find one that works and update the database with the working details."- [{:keys [engine details] :as database}]- (if-let [details-to-test (seq (driver/db-details-to-test-and-migrate (keyword engine) details))]- (do- (log/infof "Attempting to connect to %d possible legacy details" (count details-to-test))- (loop [[test-details & tail] details-to-test]- (if test-details- (if (driver.u/can-connect-with-details? engine (assoc test-details :engine engine))- (let [keys-remaining (-> test-details keys set)- [_ removed _] (data/diff keys-remaining (-> details keys set))]- (log/infof "Successfully connected, migrating to: %s" (pr-str {:keys keys-remaining :keys-removed removed}))- (t2/update! :model/Database (:id database) {:details test-details})- test-details)- (recur tail))- ;; if we go through the list and we can't fine a working detail to test, keep original value- details)))- details))+ [{:keys [engine] :as database}]+ (let [details (driver.conn/default-details database)+ test-and-migrate! (fn [details-to-test]+ (log/infof "Attempting to connect to %d possible legacy details" (count details-to-test))+ (loop [[test-details & tail] details-to-test]+ (if test-details+ (if (driver.u/can-connect-with-details? engine (assoc test-details :engine engine))+ (let [keys-remaining (-> test-details keys set)+ [_ removed _] (data/diff keys-remaining (-> details keys set))]+ (log/infof "Successfully connected, migrating to: %s" (pr-str {:keys keys-remaining :keys-removed removed}))+ (t2/update! :model/Database (:id database) {:details test-details})+ test-details)+ (recur tail))+ ;; if we go through the list and we can't fine a working detail to test, keep original value+ details)))]+ (if-let [details-to-test (seq (driver/db-details-to-test-and-migrate (keyword engine) details))]+ (test-and-migrate! details-to-test)+ details)))++(defn- check-connection!+ "Checks connectivity for a set of database details. Returns true on success, false on failure.+ Reports analytics with the given connection-type label."+ [database driver engine details-map connection-type]+ (try+ (log/info (u/format-color :cyan "Health check [%s]: checking %s {:id %d}"+ connection-type (:name database) (:id database)))+ (u/with-timeout (driver.settings/db-connection-timeout-ms)+ (or (driver/can-connect? driver details-map)+ (throw (Exception. (format "Failed to connect to Database (%s)" connection-type)))))+ (log/info (u/format-color :green "Health check [%s]: success %s {:id %d}"+ connection-type (:name database) (:id database)))+ (analytics/inc! :metabase-database/status {:driver engine :healthy true :connection-type connection-type})+ true+ (catch Throwable e+ (let [humanized-message (some->> (u/all-ex-messages e)+ (driver/humanize-connection-error-message driver))+ reason (if (keyword? humanized-message) "user-input" "exception")]+ (log/error e (u/format-color :red "Health check [%s]: failure with error %s {:id %d :reason %s :message %s}"+ connection-type+ (:name database)+ (:id database)+ reason+ humanized-message))+ (analytics/inc! :metabase-database/status {:driver engine :healthy false :reason reason :connection-type connection-type}))+ false)))(defn health-check-database!"Checks database health off-thread.- - checks connectivity- - cleans-up ambiguous legacy, db-details"+ - checks connectivity for the default connection+ - checks connectivity for the write connection (if configured)+ - cleans-up ambiguous legacy db-details"[{:keys [engine] :as database}](when-not (or (:is_audit database) (:is_sample database))(log/info (u/format-color :cyan "Health check: queueing %s {:id %d}" (:name database) (:id database)))(quick-task/submit-task!(fn []- (let [details (maybe-test-and-migrate-details! database)- engine (name engine)- driver (keyword engine)- details-map (assoc details :engine engine)]- (try- (log/info (u/format-color :cyan "Health check: checking %s {:id %d}" (:name database) (:id database)))- (u/with-timeout (driver.settings/db-connection-timeout-ms)- (or (driver/can-connect? driver details-map)- (throw (Exception. "Failed to connect to Database"))))- (log/info (u/format-color :green "Health check: success %s {:id %d}" (:name database) (:id database)))- (analytics/inc! :metabase-database/status {:driver engine :healthy true})-- ;; Detect and update provider name+ (let [details (maybe-test-and-migrate-details! database)+ database (assoc database :details details)+ engine-str (name engine)+ driver (keyword engine-str)+ details-map (assoc details :engine engine-str)]+ (when (check-connection! database driver engine-str details-map "default")(let [provider (provider-detection/detect-provider-from-database database)](when (not= provider (:provider_name database))(try@@ -267,18 +292,12 @@(:provider_name database) provider))(t2/update! :model/Database (:id database) {:provider_name provider})(catch Throwable provider-e- (log/warnf provider-e "Error during provider detection for database {:id %d}" (:id database))))))-- (catch Throwable e- (let [humanized-message (some->> (u/all-ex-messages e)- (driver/humanize-connection-error-message driver))- reason (if (keyword? humanized-message) "user-input" "exception")]- (log/error e (u/format-color :red "Health check: failure with error %s {:id %d :reason %s :message %s}"- (:name database)- (:id database)- reason- humanized-message))- (analytics/inc! :metabase-database/status {:driver engine :healthy false :reason reason})))))))))+ (log/warnf provider-e "Error during provider detection for database {:id %d}" (:id database)))))))+ (when (:write_data_details database)+ (let [write-details (driver.conn/without-resolution-telemetry+ (driver.conn/with-write-connection+ (driver.conn/effective-details database)))]+ (check-connection! database driver engine-str (assoc write-details :engine engine-str) "write-data"))))))))(defn check-health!"Health checks databases connected to metabase asynchronously using a thread pool."@@ -348,7 +367,9 @@(binding [*normalizing-details* true](driver/normalize-db-detailsdriver- (m/update-existing-in db [:details :auth-provider] keyword))))]+ (-> db+ (m/update-existing-in [:details :auth-provider] keyword)+ (m/update-existing-in [:write_data_details :auth-provider] keyword)))))](cond-> database;; TODO - this is only really needed for API responses. This should be a `hydrate` thing instead!(and driver@@ -453,7 +474,8 @@(some some? [(:cache_field_values_schedule changes) (:metadata_sync_schedule changes)]))infer-db-schedules- (some? (:details changes))+ (or (some? (:details changes))+ (some? (:write_data_details changes)))secret/handle-incoming-client-secrets!(:uploads_enabled changes)@@ -548,46 +570,48 @@driver.u/default-sensitive-fields))(methodical/defmethod mi/to-json :model/Database- "When encoding a Database as JSON remove the `details` for any User without write perms for the DB.- Users with write perms can see the `details` but remove anything resembling a password. No one gets to see this in- an API response!+ "When encoding a Database as JSON remove the `details` and `write_data_details` for any User without write perms+ for the DB. Users with write perms can see the details but remove anything resembling a password. No one gets to+ see this in an API response!Also remove settings that the User doesn't have read perms for."[db json-generator]- (next-method- (let [db (if (not (mi/can-write? db))- (do (log/debug "Fully redacting database details during json encoding.")- (dissoc db :details))- (do (log/debug "Redacting sensitive fields within database details during json encoding.")- (-> db- (secret/to-json-hydrate-redacted-secrets)- (update :details (fn [details]- (reduce- #(m/update-existing %1 %2 (fn [v] (when v secret/protected-password)))- details- (sensitive-fields-for-db db)))))))]- (update db :settings- (fn [settings]- (when (map? settings)- (u/prog1- (m/filter-keys- (fn [setting-name]- (try- (setting/can-read-setting? setting-name- (setting/current-user-readable-visibilities))- (catch Throwable e+ (letfn [(redact-sensitive-fields [details]+ (reduce+ #(m/update-existing %1 %2 (fn [v] (when v secret/protected-password)))+ details+ (sensitive-fields-for-db db)))]+ (next-method+ (let [db (if (not (mi/can-write? db))+ (do (log/debug "Fully redacting database details during json encoding.")+ (dissoc db :details :write_data_details))+ (do (log/debug "Redacting sensitive fields within database details during json encoding.")+ (-> db+ (secret/to-json-hydrate-redacted-secrets)+ (update :details redact-sensitive-fields)+ (m/update-existing :write_data_details redact-sensitive-fields))))]+ (update db :settings+ (fn [settings]+ (when (map? settings)+ (u/prog1+ (m/filter-keys+ (fn [setting-name]+ (try+ (setting/can-read-setting? setting-name+ (setting/current-user-readable-visibilities))+ (catch Throwable e;; there is an known issue with exception is ignored when render API response (#32822);; If you see this error, you probably need to define a setting for `setting-name`.;; But ideally, we should resolve the above issue, and remove this try/catch- (log/errorf e "Error checking the readability of %s setting. The setting will be hidden in API response."- setting-name)+ (log/errorf e "Error checking the readability of %s setting. The setting will be hidden in API response."+ setting-name);; let's be conservative and hide it by defaults, if you want to see it,;; you need to define it :)- false)))- settings)- (when (not= <> settings)- (log/debug "Redacting non-user-readable database settings during json encoding.")))))))- json-generator))+ false)))+ settings)+ (when (not= <> settings)+ (log/debug "Redacting non-user-readable database settings during json encoding.")))))))+ json-generator)));;; ------------------------------------------ Workspace Permissions Cache --------------------------------------------@@ -634,16 +658,23 @@details::serdes/skip)):import identity}+ :write_data_details {:export-with-context+ (fn [current _ details]+ (if (and include-database-secrets+ (not (:is_attached_dwh current)))+ details+ ::serdes/skip))+ :import identity}:creator_id (serdes/fk :model/User):router_database_id (serdes/fk :model/Database):initial_sync_status {:export identity :import (constantly "complete")}}})(defmethod serdes/extract-query "Database"[model-name {:keys [where]}]- (t2/reducible-select (keyword "model" model-name) {:where- [:and- (or where true)- [:= :router_database_id nil]]}))+ (t2/reducible-select (keyword "model" model-name)+ {:where [:and+ (or where true)+ [:= :router_database_id nil]]}))(defmethod serdes/entity-id "Database"[_ {:keys [name]}]@@ -681,7 +712,7 @@:updated-at true}:search-terms {:name search.spec/explode-camel-case:description true}- :where [:= :router_database_id nil]+ :where [:= :router_database_id nil]:render-terms {:initial-sync-status true}})(defenterprise hydrate-router-user-attributediff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.cljindex 390efddaa12..289eb5c3872 100644--- a/src/metabase/warehouses_rest/api.clj+++ b/src/metabase/warehouses_rest/api.clj@@ -12,6 +12,7 @@[metabase.config.core :as config][metabase.database-routing.core :as database-routing][metabase.driver :as driver]+ [metabase.driver.connection :as driver.conn][metabase.driver.settings :as driver.settings][metabase.driver.util :as driver.u][metabase.events.core :as events]@@ -938,17 +939,43 @@;;; --------------------------------------------- PUT /api/database/:id ----------------------------------------------(defn- upsert-sensitive-fields- "Replace any sensitive values not overridden in the PUT with the original values"- [database details]- (when details- (merge (:details database)- (reduce- (fn [details k]- (if (= secret/protected-password (get details k))- (m/update-existing details k (constantly (get-in database [:details k])))- details))- details- (database/sensitive-fields-for-db database)))))+ "Replace any sensitive values not overridden in the PUT with the original values.+ `details-key` is the key in the database map to use (e.g., :details or :write_data_details)."+ ([database details]+ (upsert-sensitive-fields database details :details))+ ([database details details-key]+ (when details+ (merge (get database details-key)+ (reduce+ (fn [details k]+ (if (= secret/protected-password (get details k))+ (m/update-existing details k (constantly (get-in database [details-key k])))+ details))+ details+ (database/sensitive-fields-for-db database))))))++(defn- validate-write-data-details!+ "Validates write_data_details guardrails. Throws 400 on violation."+ [existing-database write-data-details]+ (api/check-400 (not (:router_database_id existing-database))+ (tru "Cannot configure a write connection on a destination database"))+ (api/check-400 (not (t2/exists? :model/Database :router_database_id (:id existing-database)))+ (tru "Cannot configure a write connection on a router database"))+ (when-not (:write-data-connection write-data-details)+ (throw (ex-info (tru "write-data-connection must be set in write_data_details")+ {:status-code 400})))+ (when (:destination-database write-data-details)+ (throw (ex-info (tru "destination-database must be false in write_data_details")+ {:status-code 400})))+ (let [engine (keyword (:engine existing-database))]+ (when-let [hidden-fields (not-empty (driver.u/fields-hidden-for-write-data-connection engine))]+ (let [hidden-kws (into #{} (map keyword) hidden-fields)+ disallowed-keys (filterv #(contains? hidden-kws %) (keys write-data-details))]+ (when (seq disallowed-keys)+ (throw (ex-info (tru "write_data_details must not contain fields hidden for write connections: {0}"+ (str/join ", " (sort (map name disallowed-keys))))+ {:status-code 400+ :disallowed-keys disallowed-keys})))))));; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to;; use our API + we will need it when we make auto-TypeScript-signature generation happen@@ -959,80 +986,102 @@[{:keys [id]} :- [:map[:id ms/PositiveInt]]_query-params- {:keys [name engine details is_full_sync is_on_demand description caveats points_of_interest schedules- auto_run_queries refingerprint cache_ttl settings provider_name]} :- [:map- [:name {:optional true} [:maybe ms/NonBlankString]]- [:engine {:optional true} [:maybe DBEngineString]]- [:refingerprint {:optional true} [:maybe :boolean]]- [:details {:optional true} [:maybe ms/Map]]- [:schedules {:optional true} [:maybe sync.schedules/ExpandedSchedulesMap]]- [:description {:optional true} [:maybe :string]]- [:caveats {:optional true} [:maybe :string]]- [:points_of_interest {:optional true} [:maybe :string]]- [:auto_run_queries {:optional true} [:maybe :boolean]]- [:cache_ttl {:optional true} [:maybe ms/PositiveInt]]- [:provider_name {:optional true} [:maybe :string]]- [:settings {:optional true} [:maybe ms/Map]]]]+ {:keys [name engine details write_data_details is_full_sync is_on_demand description caveats points_of_interest+ schedules auto_run_queries refingerprint cache_ttl settings provider_name]}+ :- [:map+ [:name {:optional true} [:maybe ms/NonBlankString]]+ [:engine {:optional true} [:maybe DBEngineString]]+ [:refingerprint {:optional true} [:maybe :boolean]]+ [:details {:optional true} [:maybe ms/Map]]+ [:write_data_details {:optional true} [:maybe ms/Map]]+ [:schedules {:optional true} [:maybe sync.schedules/ExpandedSchedulesMap]]+ [:description {:optional true} [:maybe :string]]+ [:caveats {:optional true} [:maybe :string]]+ [:points_of_interest {:optional true} [:maybe :string]]+ [:auto_run_queries {:optional true} [:maybe :boolean]]+ [:cache_ttl {:optional true} [:maybe ms/PositiveInt]]+ [:provider_name {:optional true} [:maybe :string]]+ [:settings {:optional true} [:maybe ms/Map]]]];; TODO - ensure that custom schedules and let-user-control-scheduling go in lockstep- (let [existing-database (api/write-check (t2/select-one :model/Database :id id))- incoming-details details- details (some->> details- (upsert-sensitive-fields existing-database))- ;; verify that we can connect to the database if `:details` OR `:engine` have changed.- details-changed? (some-> details (not= (:details existing-database)))- engine-changed? (some-> engine keyword (not= (:engine existing-database)))- conn-error (when (or details-changed? engine-changed?)- (warehouses/test-database-connection (or engine (:engine existing-database))- (or details (:details existing-database))))- full-sync? (some-> is_full_sync boolean)- on-demand? (boolean is_on_demand)]- (if conn-error+ (when (some? write_data_details)+ (premium-features/assert-has-feature :advanced-permissions (tru "Advanced Permissions")))+ (when (:write-data-connection details)+ (throw (ex-info (tru "write-data-connection must not be set in details")+ {:status-code 400})))+ (let [existing-database (api/write-check (t2/select-one :model/Database :id id))+ _ (when write_data_details+ (validate-write-data-details! existing-database write_data_details))+ incoming-details details+ incoming-write-data-details write_data_details+ details-with-secrets (some->> details+ (upsert-sensitive-fields existing-database))+ write-data-details-with-secrets (when write_data_details+ (upsert-sensitive-fields existing-database write_data_details :write_data_details))+ ;; verify that we can connect to the database if details OR `:engine` have changed.+ details-changed? (some-> details-with-secrets (not= (:details existing-database)))+ write-details-changed? (some-> write-data-details-with-secrets (not= (:write_data_details existing-database)))+ engine-changed? (some-> engine keyword (not= (:engine existing-database)))+ main-conn-error (when (or details-changed? engine-changed?)+ (warehouses/test-database-connection (or engine (:engine existing-database))+ (or details-with-secrets (driver.conn/default-details existing-database))))+ write-conn-error (when (or write-details-changed? engine-changed?)+ (let [would-be-database (cond-> existing-database+ details-with-secrets (assoc :details details-with-secrets)+ write-data-details-with-secrets (assoc :write_data_details write-data-details-with-secrets))]+ (driver.conn/with-write-connection+ (warehouses/test-database-connection (or engine (:engine would-be-database))+ (driver.conn/effective-details would-be-database)))))+ full-sync? (some-> is_full_sync boolean)+ on-demand? (boolean is_on_demand)]+ (if (or main-conn-error write-conn-error);; failed to connect, return error{:status 400- :body conn-error}+ ;; Write details are a merge over the top of default connection:+ :body (or main-conn-error write-conn-error)};; no error, proceed with update(let [existing-settings (:settings existing-database)- pending-settings (into {}- ;; upsert settings with a PATCH-style update. `nil` key means unset the Setting.- (remove (fn [[_k v]] (nil? v)))- (merge existing-settings settings))- updates (merge- ;; TODO - is there really a reason to let someone change the engine on an existing database?- ;; that seems like the kind of thing that will almost never work in any practical way- ;; TODO - this means one cannot unset the description. Does that matter?- (u/select-keys-when- {:name name- :engine engine- :details details- :refingerprint refingerprint- :is_full_sync full-sync?- :is_on_demand on-demand?- :description description- :caveats caveats- :points_of_interest points_of_interest- :auto_run_queries auto_run_queries- :settings (when (seq settings) pending-settings)- :provider_name provider_name}- :non-nil #{:name :engine :details :refingerprint :is_full_sync :is_on_demand- :description :caveats :points_of_interest :auto_run_queries :settings}- :present #{:provider_name})- ;; cache_field_values_schedule can be nil- (when schedules- (sync.schedules/schedule-map->cron-strings schedules)))- pending-db (merge existing-database updates)]+ pending-settings (into {}+ ;; upsert settings with a PATCH-style update. `nil` key means unset the Setting.+ (remove (fn [[_k v]] (nil? v)))+ (merge existing-settings settings))+ updates (merge+ ;; TODO - is there really a reason to let someone change the engine on an existing database?+ ;; that seems like the kind of thing that will almost never work in any practical way+ ;; TODO - this means one cannot unset the description. Does that matter?+ (u/select-keys-when+ {:name name+ :engine engine+ :details details-with-secrets+ :write_data_details write-data-details-with-secrets+ :refingerprint refingerprint+ :is_full_sync full-sync?+ :is_on_demand on-demand?+ :description description+ :caveats caveats+ :points_of_interest points_of_interest+ :auto_run_queries auto_run_queries+ :settings (when (seq settings) pending-settings)+ :provider_name provider_name}+ :non-nil #{:name :engine :details :refingerprint :is_full_sync :is_on_demand+ :description :caveats :points_of_interest :auto_run_queries :settings}+ :present #{:provider_name :write_data_details})+ ;; cache_field_values_schedule can be nil+ (when schedules+ (sync.schedules/schedule-map->cron-strings schedules)))+ pending-db (merge existing-database updates)];; pass in this predicate to break circular dependency(let [driver-supports? (fn [db feature] (driver.u/supports? (driver.u/database->driver db) feature db))];; ensure we're not trying to set anything we should not be able to.;; Note: it's also possible for existing settings to become invalid when changing things like the engine.;; We skip validation for: unchanged values and nil values (resetting to default is always allowed).(doseq [[setting-kw new-value] settings- :when (and (some? new-value)+ :when (and (some? new-value);; Allow explicit default value as well (typically this is what FE will actually do);; Should we translate this into setting it to NULL? That seems too opinionated.- (not= new-value (try (setting/default-value setting-kw)+ (not= new-value (try (setting/default-value setting-kw);; fallback to a redundant nil check- (catch Exception _)))- (not= new-value (get existing-settings setting-kw)))]+ (catch Exception _)))+ (not= new-value (get existing-settings setting-kw)))](try(setting/validate-settable-for-db! setting-kw pending-db driver-supports?)(catch Exception e@@ -1047,15 +1096,16 @@;; the details in db and existing-database have been normalized so they are the same here;; we need to pass through details-changed? which is calculated before detail normalization;; to ensure the pool is invalidated and [[driver-api/secret-value-as-file!]] memoization is cleared- (events/publish-event! :event/database-update {:object db- :user-id api/*current-user-id*- :previous-object existing-database+ (events/publish-event! :event/database-update {:object db+ :user-id api/*current-user-id*+ :previous-object existing-database:details-changed? details-changed?})(-> db;; return the DB with the expanded schedules back in placeadd-expanded-schedules;; return the DB with the passed in details in place- (m/update-existing :details #(merge incoming-details %))))))))+ (m/update-existing :details #(merge incoming-details %))+ (m/update-existing :write_data_details #(merge incoming-write-data-details %))))))));;; -------------------------------------------- DELETE /api/database/:id --------------------------------------------@@ -1099,7 +1149,7 @@;; it's okay to allow testing H2 connections during sync. We only want to disallow you from testing them for the;; purposes of creating a new H2 database.(binding [driver.settings/*allow-testing-h2-connections* true]- (driver.u/can-connect-with-details? (:engine db) (:details db) :throw-exceptions))+ (driver.u/can-connect-with-details? (:engine db) (driver.conn/default-details db) :throw-exceptions))nil(catch Throwable ee))]@@ -1486,11 +1536,15 @@#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}(api.macros/defendpoint :get "/:id/healthcheck""Reports whether the database can currently connect"- [{:keys [id]} :- [:map [:id ms/PositiveInt]]]- (let [{:keys [engine details]} (t2/select-one :model/Database :id id)]+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]+ {:keys [connection-type]} :- [:map [:connection-type {:optional true} ::driver.conn/connection-type]]]+ (let [{:as database :keys [engine]} (t2/select-one :model/Database :id id)+ connection-type (or connection-type :default)+ connection-details (driver.conn/details-for-exact-type database connection-type)]+ (api/check-400 connection-details (tru "No {0} connection configured for this database" (name connection-type)));; we only want to prevent creating new H2 databases. Testing the existing database is fine.(binding [driver.settings/*allow-testing-h2-connections* true]- (if-let [err-map (warehouses/test-database-connection engine details)]+ (if-let [err-map (warehouses/test-database-connection engine connection-details)](merge err-map {:status "error"}){:status "ok"}))))diff --git a/test/metabase/driver/common_test.clj b/test/metabase/driver/common_test.cljindex 4e279cd1317..451e75a5fb5 100644--- a/test/metabase/driver/common_test.clj+++ b/test/metabase/driver/common_test.clj@@ -73,7 +73,9 @@(testing "JSON Unfolding database support details behave as they're supposed to"#_{:clj-kondo/ignore [:equals-true]}(are [details expected] (= expected- (driver.common/json-unfolding-default {:details details}))+ (driver.common/json-unfolding-default+ {:lib/type :metadata/database+ :details details})){} true{:json-unfolding nil} true{:json-unfolding true} truediff --git a/test/metabase/driver/connection/workspaces_test.clj b/test/metabase/driver/connection/workspaces_test.cljnew file mode 100644index 00000000000..4a240f2434a--- /dev/null+++ b/test/metabase/driver/connection/workspaces_test.clj@@ -0,0 +1,48 @@+(ns metabase.driver.connection.workspaces-test+ (:require+ [clojure.test :refer :all]+ [metabase.driver.connection.workspaces :as driver.w]))++(deftest ^:parallel maybe-swap-details-test+ (testing "maybe-swap-details merges swap map into details"+ (driver.w/with-swapped-connection-details 1 {:user "swap-user" :password "swap-pass"}+ (is (= {:host "localhost" :user "swap-user" :password "swap-pass"}+ (driver.w/maybe-swap-details 1 {:host "localhost" :user "original-user" :password "original-pass"})))))++ (testing "maybe-swap-details returns details unchanged when no swap exists"+ (driver.w/with-swapped-connection-details 1 {:user "swap-user"}+ (is (= {:host "localhost" :user "original-user"}+ (driver.w/maybe-swap-details 2 {:host "localhost" :user "original-user"})))))++ (testing "maybe-swap-details supports deep merge for nested maps"+ (driver.w/with-swapped-connection-details 1 {:ssl {:key-store-password "new-pass"}}+ (is (= {:host "localhost" :ssl {:enabled true :key-store-password "new-pass"}}+ (driver.w/maybe-swap-details 1 {:host "localhost" :ssl {:enabled true :key-store-password "old-pass"}})))))++ (testing "deep merge adds new keys to nested maps"+ (driver.w/with-swapped-connection-details 1 {:ssl {:new-key "new-value"}}+ (is (= {:host "localhost" :ssl {:enabled true :new-key "new-value"}}+ (driver.w/maybe-swap-details 1 {:host "localhost" :ssl {:enabled true}})))))++ (testing "deep merge replaces nested map with non-map value"+ (driver.w/with-swapped-connection-details 1 {:ssl "disabled"}+ (is (= {:host "localhost" :ssl "disabled"}+ (driver.w/maybe-swap-details 1 {:host "localhost" :ssl {:enabled true :key-store "path"}})))))++ (testing "deep merge adds nested map where none existed"+ (driver.w/with-swapped-connection-details 1 {:ssl {:enabled true}}+ (is (= {:host "localhost" :ssl {:enabled true}}+ (driver.w/maybe-swap-details 1 {:host "localhost"})))))++ (testing "deep merge works with multiple levels of nesting"+ (driver.w/with-swapped-connection-details 1 {:advanced {:ssl {:cert {:path "/new/path"}}}}+ (is (= {:host "localhost" :advanced {:timeout 30 :ssl {:enabled true :cert {:path "/new/path"}}}}+ (driver.w/maybe-swap-details 1 {:host "localhost" :advanced {:timeout 30 :ssl {:enabled true :cert {:path "/old/path"}}}})))))++ (testing "nested swaps for the same database throw an exception"+ (driver.w/with-swapped-connection-details 1 {:user "outer"}+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Nested connection detail swaps are not supported"+ (driver.w/with-swapped-connection-details 1 {:user "inner"}+ nil))))))diff --git a/test/metabase/driver/connection_test.clj b/test/metabase/driver/connection_test.cljnew file mode 100644index 00000000000..7ae1a2cc61d--- /dev/null+++ b/test/metabase/driver/connection_test.clj@@ -0,0 +1,153 @@+(ns metabase.driver.connection-test+ (:require+ [clojure.test :refer :all]+ [mb.hawk.assert-exprs.approximately-equal :as =?]+ [metabase.analytics.prometheus-test :as prometheus-test]+ [metabase.driver.connection :as driver.conn]+ [metabase.driver.connection.workspaces :as driver.w]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(comment =?/keep-me)++(deftest effective-details-default-test+ (testing "effective-details returns :details when *connection-type* is :default"+ (let [details {:host "read-host" :port 5432}+ write-data-details {:host "write-host" :port 5432}+ database {:lib/type :metadata/database+ :details details+ :write-data-details write-data-details}]+ (is (=? details+ (driver.conn/effective-details database))))))++(deftest effective-details-write-with-write-details-test+ (let [details {:host "read-host" :port 5432}+ write-data-details {:host "write-host" :port 5432}]+ (testing "effective-details returns write_data_details when :write-data and details exist"+ (testing "with kebab-case key (MLv2 metadata style)"+ (let [database {:lib/type :metadata/database+ :details details+ :write-data-details write-data-details}]+ (driver.conn/with-write-connection+ (is (=? write-data-details+ (driver.conn/effective-details database))))))+ (testing "with snake_case key (Toucan2 instance style)"+ (let [database (t2/instance :model/Database+ {:id 1+ :details details+ :write_data_details write-data-details})]+ (driver.conn/with-write-connection+ (is (=? write-data-details+ (driver.conn/effective-details database)))))))))++(deftest effective-details-write-fallback-test+ (let [details {:host "read-host" :port 5432}]+ (testing "effective-details falls back to :details when :write-data but no write details"+ (let [database {:lib/type :metadata/database+ :details details+ :write-data-details nil}]+ (driver.conn/with-write-connection+ (is (=? details+ (driver.conn/effective-details database)))))+ (testing "also when :write_data_details key is missing entirely"+ (let [database {:lib/type :metadata/database+ :details details}]+ (driver.conn/with-write-connection+ (is (=? details+ (driver.conn/effective-details database)))))))))++(deftest effective-details-nil-test+ (testing "effective-details returns nil when database is nil"+ (is (nil? (driver.conn/effective-details nil)))+ (driver.conn/with-write-connection+ (is (nil? (driver.conn/effective-details nil))))))++(deftest write-connection-requested?-test+ (testing "write-connection-requested? returns false by default"+ (is (false? (driver.conn/write-connection-requested?))))+ (testing "write-connection-requested? returns true inside with-write-connection"+ (driver.conn/with-write-connection+ (is (true? (driver.conn/write-connection-requested?))))))++(deftest effective-details-with-workspace-swap-test+ (testing "effective-details applies workspace swap in :default connection type"+ (let [database {:lib/type :metadata/database+ :id 1+ :details {:host "read-host" :user "admin" :port 5432}}]+ (driver.w/with-swapped-connection-details 1 {:user "ws-user" :password "ws-pass"}+ (is (=? {:host "read-host" :user "ws-user" :password "ws-pass" :port 5432}+ (driver.conn/effective-details database))))))++ (testing "effective-details applies workspace swap AFTER write-data merge"+ (let [database {:lib/type :metadata/database+ :id 1+ :details {:host "host" :user "admin" :port 5432}+ :write-data-details {:user "writer" :password "write-pass" :write true}}]+ (driver.w/with-swapped-connection-details 1 {:user "ws-user" :password "ws-pass"}+ (driver.conn/with-write-connection+ (is (=? {:host "host" :user "ws-user" :password "ws-pass" :port 5432 :write true}+ (driver.conn/effective-details database)))))))++ (testing "without workspace swap, effective-details is unchanged (regression)"+ (let [database {:lib/type :metadata/database+ :id 1+ :details {:host "host" :user "admin"}+ :write-data-details {:user "writer"}}]+ (is (=? {:host "host" :user "admin"}+ (driver.conn/effective-details database)))+ (driver.conn/with-write-connection+ (is (=? {:host "host" :user "writer"}+ (driver.conn/effective-details database)))))))++(deftest type-resolved-metric-test+ (testing "type-resolved counter increments when write-data-details are genuinely used"+ (mt/with-prometheus-system! [_ system]+ (let [database {:lib/type :metadata/database+ :id 1+ :details {:host "read-host" :port 5432}+ :write-data-details {:host "write-host"}}]+ (driver.conn/with-write-connection+ (driver.conn/effective-details database))+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-db-connection/type-resolved+ {:connection-type "write-data"}))))))+ (testing "type-resolved counter does NOT increment on fallback (no write-data-details)"+ (mt/with-prometheus-system! [_ system]+ (let [database {:lib/type :metadata/database+ :id 1+ :details {:host "read-host" :port 5432}}]+ (driver.conn/with-write-connection+ (driver.conn/effective-details database))+ (is (prometheus-test/approx= 0 (mt/metric-value system :metabase-db-connection/type-resolved+ {:connection-type "write-data"}))))))+ (testing "type-resolved counter does NOT increment for default connection type"+ (mt/with-prometheus-system! [_ system]+ (let [database {:lib/type :metadata/database+ :id 1+ :details {:host "read-host" :port 5432}+ :write-data-details {:host "write-host"}}]+ (driver.conn/effective-details database)+ (is (prometheus-test/approx= 0 (mt/metric-value system :metabase-db-connection/type-resolved+ {:connection-type "write-data"}))))))+ (testing "type-resolved counter does NOT increment when workspace swap is active"+ (mt/with-prometheus-system! [_ system]+ (let [database {:lib/type :metadata/database+ :id 1+ :details {:host "read-host" :port 5432}+ :write-data-details {:host "write-host"}}]+ (driver.w/with-swapped-connection-details 1 {:user "ws-user"}+ (driver.conn/with-write-connection+ (driver.conn/effective-details database)))+ (is (prometheus-test/approx= 0 (mt/metric-value system :metabase-db-connection/type-resolved+ {:connection-type "write-data"}))))))+ (testing "type-resolved counter does NOT increment inside without-resolution-telemetry"+ (mt/with-prometheus-system! [_ system]+ (let [database {:lib/type :metadata/database+ :id 1+ :details {:host "read-host" :port 5432}+ :write-data-details {:host "write-host"}}]+ (driver.conn/without-resolution-telemetry+ (driver.conn/with-write-connection+ (is (=? {:host "write-host" :port 5432}+ (driver.conn/effective-details database)))))+ (is (prometheus-test/approx= 0 (mt/metric-value system :metabase-db-connection/type-resolved+ {:connection-type "write-data"})))))))diff --git a/test/metabase/driver/sql_jdbc/connection_test.clj b/test/metabase/driver/sql_jdbc/connection_test.cljindex 9473a376946..838706fc1b5 100644--- a/test/metabase/driver/sql_jdbc/connection_test.clj+++ b/test/metabase/driver/sql_jdbc/connection_test.clj@@ -12,6 +12,8 @@[metabase.config.core :as config][metabase.core.core :as mbc][metabase.driver :as driver]+ [metabase.driver.connection :as driver.conn]+ [metabase.driver.connection.workspaces :as driver.w][metabase.driver.settings :as driver.settings][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh]@@ -34,6 +36,7 @@[toucan2.core :as t2])(:import(com.google.common.cache Cache)+ (com.mchange.v2.c3p0 DataSources)(org.h2.tools Server)))(set! *warn-on-reflection* true)@@ -64,6 +67,7 @@;; need to create a new, nonexistent h2 db(let [destroyed? (atom false)original-destroy @#'sql-jdbc.conn/destroy-pool!+ pool-cache-key @#'sql-jdbc.conn/pool-cache-keyconnection-details {:db "mem:connection_test"}spec (mdb/spec :h2 connection-details)](with-redefs [sql-jdbc.conn/destroy-pool! (fn [id destroyed-spec]@@ -77,22 +81,153 @@(next.jdbc/execute! conn ["CREATE TABLE birds (name varchar)"])(next.jdbc/execute! conn ["INSERT INTO birds values ('rasta'),('lucky')"])(mt/with-temp [:model/Database database {:engine :h2, :details connection-details}]- (testing "database id is not in our connection map initially"+ (let [cache-key (pool-cache-key database)]+ (testing "database id is not in our connection map initially";; deref'ing a var to get the atom. looks weird- (is (not (contains? @@#'sql-jdbc.conn/database-id->connection-pool- (u/id database)))))- (testing "when getting a pooled connection it is now in our connection map"- (let [stored-spec (sql-jdbc.conn/db->pooled-connection-spec database)- birds (jdbc/query stored-spec ["SELECT * FROM birds"])]- (is (seq birds))- (is (contains? @@#'sql-jdbc.conn/database-id->connection-pool- (u/id database)))))- (testing "and is no longer in our connection map after cleanup"- (driver/notify-database-updated :h2 database)- (is (not (contains? @@#'sql-jdbc.conn/database-id->connection-pool- (u/id database)))))- (testing "the pool has been destroyed"- (is @destroyed?))))))))))+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool cache-key))))+ (testing "when getting a pooled connection it is now in our connection map"+ (let [stored-spec (sql-jdbc.conn/db->pooled-connection-spec database)+ birds (jdbc/query stored-spec ["SELECT * FROM birds"])]+ (is (seq birds))+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool cache-key))))+ (testing "and is no longer in our connection map after cleanup"+ (driver/notify-database-updated :h2 database)+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool cache-key))))+ (testing "the pool has been destroyed"+ (is @destroyed?)))))))))))++(deftest connection-type-pool-separation-test+ (mt/test-driver :h2+ (testing "Different connection types get separate pools for the same database"+ (let [read-details {:db "mem:read_pool_test"}+ write-details {:db "mem:write_pool_test"}+ spec (mdb/spec :h2 read-details)]+ ;; Create an in-memory H2 db we can use for the test+ (sql-jdbc.execute/do-with-connection-with-options+ :h2+ spec+ {:write? true}+ (fn [conn]+ (next.jdbc/execute! conn ["CREATE TABLE IF NOT EXISTS test_tbl (id int)"])+ ;; Use snake_case for column name since deftransforms uses snake_case keys+ (mt/with-temp [:model/Database database {:engine :h2+ :details read-details+ :write_data_details write-details}]+ (let [db-id (u/the-id database)+ default-cache-key [db-id :default]+ write-cache-key [db-id :write-data]]+ ;; Ensure pools are cleared+ (sql-jdbc.conn/invalidate-pool-for-db! database)++ (testing "initially no pools exist"+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key)))+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))++ (testing "getting a default connection creates only the default pool"+ (sql-jdbc.conn/db->pooled-connection-spec database)+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key))+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))++ (testing "getting a write connection creates a separate write pool"+ (driver.conn/with-write-connection+ (sql-jdbc.conn/db->pooled-connection-spec database))+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key))+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key)))++ (testing "the two pools are different objects"+ (let [default-pool (get @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key)+ write-pool (get @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key)]+ (is (some? default-pool))+ (is (some? write-pool))+ (is (not (identical? default-pool write-pool)))))++ ;; Cleanup+ (sql-jdbc.conn/invalidate-pool-for-db! database)))))))))++(deftest write-connection-reuses-default-pool-when-unconfigured-test+ (mt/test-driver :h2+ (testing "When write-data-details is not configured, with-write-connection reuses the default pool"+ (mt/with-temp [:model/Database database {:engine :h2+ :details {:db "mem:no_write_details_test"}}]+ (let [db-id (u/the-id database)+ default-cache-key [db-id :default]+ write-cache-key [db-id :write-data]]+ (sql-jdbc.conn/invalidate-pool-for-db! database)+ (testing "get the default pool"+ (sql-jdbc.conn/db->pooled-connection-spec database)+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key)))+ (testing "with-write-connection reuses the default pool, no duplicate created"+ (let [default-pool (get @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key)+ write-pool (driver.conn/with-write-connection+ (sql-jdbc.conn/db->pooled-connection-spec database))]+ (is (identical? default-pool write-pool)+ "Should return the exact same pool object")+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))+ "No separate write pool entry should exist")))+ (sql-jdbc.conn/invalidate-pool-for-db! database))))))++(deftest write-pool-uses-write-details-test+ (mt/test-driver :h2+ (testing "Write connection pool uses :write-data-details when available"+ (let [read-details {:db "mem:read_details_db"}+ write-details {:db "mem:write_details_db"}]+ ;; Use snake_case for column name since deftransforms uses snake_case keys+ (mt/with-temp [:model/Database database {:engine :h2+ :details read-details+ :write_data_details write-details}]+ (let [db-id (u/the-id database)]+ ;; Ensure pools are cleared+ (sql-jdbc.conn/invalidate-pool-for-db! database)++ (testing "jdbc-spec-hash differs between default and write connection types"+ (let [default-hash (#'sql-jdbc.conn/jdbc-spec-hash database)+ write-hash (driver.conn/with-write-connection+ (#'sql-jdbc.conn/jdbc-spec-hash database))]+ (is (integer? default-hash))+ (is (integer? write-hash))+ (is (not= default-hash write-hash)+ "Hash should differ because effective-details returns different details")))++ (testing "hash cache uses composite keys"+ ;; Get both pools+ (sql-jdbc.conn/db->pooled-connection-spec database)+ (driver.conn/with-write-connection+ (sql-jdbc.conn/db->pooled-connection-spec database))++ (let [default-cached-hash (get @@#'sql-jdbc.conn/pool-cache-key->jdbc-spec-hash [db-id :default])+ write-cached-hash (get @@#'sql-jdbc.conn/pool-cache-key->jdbc-spec-hash [db-id :write-data])]+ (is (some? default-cached-hash))+ (is (some? write-cached-hash))+ (is (not= default-cached-hash write-cached-hash))))++ ;; Cleanup+ (sql-jdbc.conn/invalidate-pool-for-db! database)))))))++(deftest invalidate-pool-clears-both-connection-types-test+ (mt/test-driver :h2+ (testing "invalidate-pool-for-db! clears both default and write pools"+ (let [read-details {:db "mem:invalidate_test"}+ write-details {:db "mem:invalidate_write_test"}]+ ;; Use snake_case for column name since deftransforms uses snake_case keys+ (mt/with-temp [:model/Database database {:engine :h2+ :details read-details+ :write_data_details write-details}]+ (let [db-id (u/the-id database)+ default-cache-key [db-id :default]+ write-cache-key [db-id :write-data]]+ ;; Create both pools+ (sql-jdbc.conn/db->pooled-connection-spec database)+ (driver.conn/with-write-connection+ (sql-jdbc.conn/db->pooled-connection-spec database))++ (testing "both pools exist before invalidation"+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key))+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key)))++ (testing "invalidate-pool-for-db! removes both pools"+ (sql-jdbc.conn/invalidate-pool-for-db! database)+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key)))+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))))))))(deftest ^:parallel c3p0-datasource-name-test(mt/test-drivers (mt/driver-select {:+parent :sql-jdbc})@@ -163,7 +298,7 @@(with-redefs [sql-jdbc.conn/log-jdbc-spec-hash-change-msg! hash-change-fndriver.u/supports? supports?-fn](let [pool-spec-1 (sql-jdbc.conn/db->pooled-connection-spec db)- db-hash-1 (get @@#'sql-jdbc.conn/database-id->jdbc-spec-hash (u/the-id db))]+ db-hash-1 (get @@#'sql-jdbc.conn/pool-cache-key->jdbc-spec-hash (#'sql-jdbc.conn/pool-cache-key db))](testing "hash value calculated correctly for new pooled conn"(is (some? pool-spec-1))(is (integer? db-hash-1))@@ -176,8 +311,8 @@(t2/update! :model/Database (mt/id) {:details (:details db-perturbed)})(let [;; this call should result in the connection pool becoming invalidated, and the new hash value;; being stored based upon these updated details- pool-spec-2 (sql-jdbc.conn/db->pooled-connection-spec db-perturbed)- db-hash-2 (get @@#'sql-jdbc.conn/database-id->jdbc-spec-hash (u/the-id db))]+ pool-spec-2 (sql-jdbc.conn/db->pooled-connection-spec db-perturbed)+ db-hash-2 (get @@#'sql-jdbc.conn/pool-cache-key->jdbc-spec-hash (#'sql-jdbc.conn/pool-cache-key db))];; to throw a wrench into things, kick off a sync of the original db (unperturbed); this;; simulates a long running sync that began before the perturbed details were saved to the app DB;; the sync steps SHOULD NOT invalidate the connection pool, because doing so could cause a seesaw@@ -209,14 +344,15 @@:source nil:value (.getBytes "super secret"):creator_id (mt/user->id :crowberto)}]- (let [db {:engine :postgres- :details {:ssl true- :ssl-mode "verify-ca"- :ssl-root-cert-options "uploaded"- :ssl-root-cert-creator-id (mt/user->id :crowberto)- :ssl-root-cert-source nil- :ssl-root-cert-id (:id secret)- :ssl-root-cert-created-at "2022-07-25T15:57:51.556-05:00"}}]+ (let [db {:lib/type :metadata/database+ :engine :postgres+ :details {:ssl true+ :ssl-mode "verify-ca"+ :ssl-root-cert-options "uploaded"+ :ssl-root-cert-creator-id (mt/user->id :crowberto)+ :ssl-root-cert-source nil+ :ssl-root-cert-id (:id secret)+ :ssl-root-cert-created-at "2022-07-25T15:57:51.556-05:00"}}](is (instance? java.io.File(:sslrootcert (#'sql-jdbc.conn/connection-details->spec :postgres(:details db))))@@ -236,15 +372,15 @@;; TODO (Cam 9/30/25) -- sort of evil to delete databases like this in a test, shouldn't we do this in a;; transaction or something?(t2/delete! :model/Database {:where [:= :is_audit true]})- (let [status (mbc/ensure-audit-db-installed!)+ (let [status (mbc/ensure-audit-db-installed!)audit-db-id (t2/select-one-fn :id :model/Database {:where [:= :is_audit true]})- _ (is (= :metabase-enterprise.audit-app.audit/installed status))- _ (is (= 13371337 audit-db-id))- first-pool (sql-jdbc.conn/db->pooled-connection-spec audit-db-id)+ _ (is (= :metabase-enterprise.audit-app.audit/installed status))+ _ (is (= 13371337 audit-db-id))+ first-pool (sql-jdbc.conn/db->pooled-connection-spec audit-db-id)second-pool (sql-jdbc.conn/db->pooled-connection-spec audit-db-id)](is (= first-pool second-pool))(is (= ::audit-db-not-in-cache!- (get @#'sql-jdbc.conn/database-id->connection-pool audit-db-id ::audit-db-not-in-cache!)))))))+ (get @#'sql-jdbc.conn/pool-cache-key->connection-pool audit-db-id ::audit-db-not-in-cache!)))))))(deftest ^:parallel include-unreturned-connection-timeout-test(testing "We should be setting unreturnedConnectionTimeout; it should be the same as the query timeout (#33646)"@@ -659,7 +795,7 @@(sql-jdbc.conn/invalidate-pool-for-db! db)(let [original-spec (sql-jdbc.conn/db->pooled-connection-spec db)](testing "Swap map is merged into details when creating connection"- (driver/with-swapped-connection-details db-id {:test-swap true}+ (driver.w/with-swapped-connection-details db-id {:test-swap true}(testing "spec is swapped"(is (not= original-spec (sql-jdbc.conn/db->pooled-connection-spec db))))(testing "Pool was created with swap in swapped pools cache"@@ -680,16 +816,16 @@pool-b (atom nil)pool-a-2 (atom nil)](testing "User A swaps with their credentials"- (driver/with-swapped-connection-details db-id {:user "user-a" :password "pass-a" :log-level 100}+ (driver.w/with-swapped-connection-details db-id {:user "user-a" :password "pass-a" :log-level 100}(reset! pool-a-1 (sql-jdbc.conn/db->pooled-connection-spec db))(is (= 1 (count-swapped-pools-for-db db-id)) "First swap creates one pool")))(testing "User B swaps with different credentials"- (driver/with-swapped-connection-details db-id {:user "user-b" :password "pass-b" :log-level 99}+ (driver.w/with-swapped-connection-details db-id {:user "user-b" :password "pass-b" :log-level 99}(reset! pool-b (sql-jdbc.conn/db->pooled-connection-spec db))(is (= 2 (count-swapped-pools-for-db db-id)) "Different swap details create a second pool")(is (not (identical? @pool-a-1 @pool-b)) "Different swap details return different pool instances")))(testing "User A returns - should reuse their original pool (still in cache due to TTL)"- (driver/with-swapped-connection-details db-id {:user "user-a" :password "pass-a" :log-level 100}+ (driver.w/with-swapped-connection-details db-id {:user "user-a" :password "pass-a" :log-level 100}(reset! pool-a-2 (sql-jdbc.conn/db->pooled-connection-spec db))(is (= 2 (count-swapped-pools-for-db db-id)) "Identical swap details reuse existing pool")(is (identical? @pool-a-1 @pool-a-2) "Identical swap details return the same pool instance"))))))))@@ -700,13 +836,13 @@(let [db (mt/db)db-id (u/the-id db)](sql-jdbc.conn/invalidate-pool-for-db! db)- (driver/with-swapped-connection-details db-id {:outer-swap true}+ (driver.w/with-swapped-connection-details db-id {:outer-swap true}(sql-jdbc.conn/db->pooled-connection-spec db)(testing "Attempting nested swap for same database throws"(is (thrown-with-msg?clojure.lang.ExceptionInfo#"Nested connection detail swaps are not supported"- (driver/with-swapped-connection-details db-id {:inner-swap true}+ (driver.w/with-swapped-connection-details db-id {:inner-swap true}(sql-jdbc.conn/db->pooled-connection-spec db)))))))))(testing "Different databases can have concurrent swaps"@@ -714,29 +850,46 @@(let [db-1 (mt/db)db-1-id (u/the-id db-1)];; We can only test this with one db in most test setups, but the code path works- (driver/with-swapped-connection-details db-1-id {:swap-1 true}+ (driver.w/with-swapped-connection-details db-1-id {:swap-1 true};; This would work for a different db-id(is (some? (sql-jdbc.conn/db->pooled-connection-spec db-1))))))))(deftest invalidate-pool-clears-both-canonical-and-swapped-test- (testing "invalidate-pool-for-db! clears both canonical and swapped pools"+ (testing "invalidate-pool-for-db! clears canonical pools (default and write) and swapped pools"(mt/test-drivers (mt/normal-driver-select {:+parent :sql-jdbc})- (let [db (mt/db)- db-id (u/the-id db)]+ (let [db (mt/db)+ db-id (u/the-id db)+ default-cache-key [db-id :default]+ write-cache-key [db-id :write-data]](sql-jdbc.conn/invalidate-pool-for-db! db)+ ;; Create default canonical pool(sql-jdbc.conn/db->pooled-connection-spec db)- (is (contains? @@#'sql-jdbc.conn/database-id->connection-pool db-id))-- (driver/with-swapped-connection-details db-id {:test-swap true}+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key)+ "default canonical pool exists")++ ;; Insert a fake write pool entry to verify it gets cleared.+ ;; Needs :datasource so destroy-pool! can handle it.+ (let [fake-ds (DataSources/pooledDataSource+ (DataSources/unpooledDataSource "jdbc:h2:mem:fake_write_pool"))]+ (swap! @#'sql-jdbc.conn/pool-cache-key->connection-pool+ assoc write-cache-key {:datasource fake-ds}))+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key)+ "write canonical pool exists")++ ;; Create swapped pool (for default connection type)+ (driver.w/with-swapped-connection-details db-id {:test-swap true}(sql-jdbc.conn/db->pooled-connection-spec db))- (is (= 1 (count-swapped-pools-for-db db-id)))+ (is (= 1 (count-swapped-pools-for-db db-id))+ "swapped pool exists")- ;; Now invalidate - should clear both+ ;; Now invalidate - should clear all pools(sql-jdbc.conn/invalidate-pool-for-db! db)- (testing "Canonical pool is cleared"- (is (not (contains? @@#'sql-jdbc.conn/database-id->connection-pool db-id))))+ (testing "Default canonical pool is cleared"+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool default-cache-key))))+ (testing "Write canonical pool is cleared"+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))(testing "Swapped pool is cleared"(is (= 0 (count-swapped-pools-for-db db-id))))))))@@ -752,7 +905,7 @@(fn [db](swap! create-count inc)(original db)))]- (driver/with-swapped-connection-details db-id swap-details+ (driver.w/with-swapped-connection-details db-id swap-details;; First call creates a pool(let [pool-1 (sql-jdbc.conn/db->pooled-connection-spec db)](is (= 1 @create-count))@@ -785,7 +938,7 @@(fn [db](swap! create-count inc)(original db)))]- (driver/with-swapped-connection-details db-id swap-details+ (driver.w/with-swapped-connection-details db-id swap-details;; First call creates a pool(let [pool-1 (sql-jdbc.conn/db->pooled-connection-spec db)](is (= 1 @create-count))@@ -819,7 +972,7 @@(fn [db](swap! create-count inc)(original db)))]- (driver/with-swapped-connection-details db-id {:test-swap true}+ (driver.w/with-swapped-connection-details db-id {:test-swap true};; First call creates a pool(let [pool-1 (sql-jdbc.conn/db->pooled-connection-spec db)](is (= 1 @create-count))diff --git a/test/metabase/driver/sql_jdbc/execute_test.clj b/test/metabase/driver/sql_jdbc/execute_test.cljindex 128ffff02ca..a930f894a04 100644--- a/test/metabase/driver/sql_jdbc/execute_test.clj+++ b/test/metabase/driver/sql_jdbc/execute_test.clj@@ -3,6 +3,7 @@[clojure.test :refer :all][malli.error :as me][metabase.driver :as driver]+ [metabase.driver.connection :as driver.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.test :as mt][metabase.util.malli.registry :as mr])@@ -55,6 +56,29 @@(sql-jdbc.execute/try-ensure-open-conn! driver closed-conn)(is (= 2 @connection-count))))))))))+(deftest resilient-reconnect-preserves-connection-type-test+ (testing "resilient reconnection preserves *connection-type* binding"+ (mt/test-drivers (descendants driver/hierarchy :sql-jdbc)+ (let [test-db-id (mt/id)+ captured-connection-type (volatile! nil)+ orig-fn @#'sql-jdbc.execute/do-with-resolved-connection-data-source]+ (with-redefs [sql-jdbc.execute/do-with-resolved-connection-data-source+ (fn [driver db opts]+ (when (and (= db test-db-id) (:keep-open? opts))+ (vreset! captured-connection-type driver.conn/*connection-type*))+ (orig-fn driver db opts))]+ (let [closed-conn (doto (.getConnection ^DataSource+ (orig-fn driver/*driver* test-db-id {}))+ (.close))]+ (driver.conn/with-write-connection+ (driver/do-with-resilient-connection+ driver/*driver*+ test-db-id+ (fn [driver _]+ (sql-jdbc.execute/try-ensure-open-conn! driver closed-conn)+ (is (= :write-data @captured-connection-type)+ "Reconnection should preserve write-data connection type"))))))))))+(deftest try-ensure-open-conn-sets-non-recursive-options-test(testing "try-ensure-open-conn! sets connection options as non-recursive"#_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}@@ -166,3 +190,24 @@(is (false? (.isClosed prepared-stmt)))(.close prepared-stmt)(is (true? (.isClosed prepared-stmt)))))))))++(deftest write-op-metric-test+ (testing "write-op counter tracks default connection acquisitions"+ (mt/with-prometheus-system! [_ system]+ (mt/test-drivers (descendants driver/hierarchy :sql-jdbc)+ (sql-jdbc.execute/do-with-connection-with-options+ driver/*driver* (mt/id) nil+ (fn [_conn] nil))+ (is (pos? (mt/metric-value system :metabase-db-connection/write-op+ {:connection-type "default"}))))))+ (testing "write-op counter tracks write-data connection acquisitions"+ (mt/with-prometheus-system! [_ system]+ (mt/test-drivers (descendants driver/hierarchy :sql-jdbc)+ (let [db (mt/db)]+ (mt/with-temp-vals-in-db :model/Database (:id db) {:write_data_details (:details db)}+ (driver.conn/with-write-connection+ (sql-jdbc.execute/do-with-connection-with-options+ driver/*driver* (mt/id) nil+ (fn [_conn] nil)))+ (is (pos? (mt/metric-value system :metabase-db-connection/write-op+ {:connection-type "write-data"})))))))))diff --git a/test/metabase/driver/sync_test.clj b/test/metabase/driver/sync_test.cljindex 7d0cfc93144..69f7409fc0c 100644--- a/test/metabase/driver/sync_test.clj+++ b/test/metabase/driver/sync_test.clj@@ -34,12 +34,14 @@(deftest ^:parallel schema-filter-NPE-test(testing "Schema filter function should not NPE if you pass in a `nil` schema (#38156)"(testing "inclusion -- don't include nil schemas"- (let [db {:details {:schema-filters-type "inclusion"- :schema-filters-patterns "x"}- :engine :postgres}]+ (let [db {:lib/type :metadata/database+ :details {:schema-filters-type "inclusion"+ :schema-filters-patterns "x"}+ :engine :postgres}](is (not (driver.s/include-schema? db nil)))))(testing "exclusion -- don't exclude nil schemas"- (let [db {:details {:schema-filters-type "exclusion"- :schema-filters-patterns "x"}- :engine :postgres}]+ (let [db {:lib/type :metadata/database+ :details {:schema-filters-type "exclusion"+ :schema-filters-patterns "x"}+ :engine :postgres}](is (driver.s/include-schema? db nil))))))diff --git a/test/metabase/driver_test.clj b/test/metabase/driver_test.cljindex 1a1980a4cbf..512cba9e393 100644--- a/test/metabase/driver_test.clj+++ b/test/metabase/driver_test.clj@@ -481,47 +481,3 @@(deftest deps-flags-when-supported-driver-is-not-covered-test(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Database that supports :dependencies/native does not provide an implementation of driver/native-query-deps"(driver/native-query-deps ::mock-deps-driver nil nil))))--(deftest ^:parallel maybe-swap-details-test- (testing "maybe-swap-details merges swap map into details"- (driver/with-swapped-connection-details 1 {:user "swap-user" :password "swap-pass"}- (is (= {:host "localhost" :user "swap-user" :password "swap-pass"}- (driver/maybe-swap-details 1 {:host "localhost" :user "original-user" :password "original-pass"})))))-- (testing "maybe-swap-details returns details unchanged when no swap exists"- (driver/with-swapped-connection-details 1 {:user "swap-user"}- (is (= {:host "localhost" :user "original-user"}- (driver/maybe-swap-details 2 {:host "localhost" :user "original-user"})))))-- (testing "maybe-swap-details supports deep merge for nested maps"- (driver/with-swapped-connection-details 1 {:ssl {:key-store-password "new-pass"}}- (is (= {:host "localhost" :ssl {:enabled true :key-store-password "new-pass"}}- (driver/maybe-swap-details 1 {:host "localhost" :ssl {:enabled true :key-store-password "old-pass"}})))))-- (testing "deep merge adds new keys to nested maps"- (driver/with-swapped-connection-details 1 {:ssl {:new-key "new-value"}}- (is (= {:host "localhost" :ssl {:enabled true :new-key "new-value"}}- (driver/maybe-swap-details 1 {:host "localhost" :ssl {:enabled true}})))))-- (testing "deep merge replaces nested map with non-map value"- (driver/with-swapped-connection-details 1 {:ssl "disabled"}- (is (= {:host "localhost" :ssl "disabled"}- (driver/maybe-swap-details 1 {:host "localhost" :ssl {:enabled true :key-store "path"}})))))-- (testing "deep merge adds nested map where none existed"- (driver/with-swapped-connection-details 1 {:ssl {:enabled true}}- (is (= {:host "localhost" :ssl {:enabled true}}- (driver/maybe-swap-details 1 {:host "localhost"})))))-- (testing "deep merge works with multiple levels of nesting"- (driver/with-swapped-connection-details 1 {:advanced {:ssl {:cert {:path "/new/path"}}}}- (is (= {:host "localhost" :advanced {:timeout 30 :ssl {:enabled true :cert {:path "/new/path"}}}}- (driver/maybe-swap-details 1 {:host "localhost" :advanced {:timeout 30 :ssl {:enabled true :cert {:path "/old/path"}}}})))))-- (testing "nested swaps for the same database throw an exception"- (driver/with-swapped-connection-details 1 {:user "outer"}- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"Nested connection detail swaps are not supported"- (driver/with-swapped-connection-details 1 {:user "inner"}- nil))))))diff --git a/test/metabase/transforms/execute_test.clj b/test/metabase/transforms/execute_test.cljindex 998159bdcad..dd86df90910 100644--- a/test/metabase/transforms/execute_test.clj+++ b/test/metabase/transforms/execute_test.clj@@ -16,8 +16,8 @@[metabase.transforms.test-dataset :as transforms-dataset][metabase.transforms.test-util :as transforms.tu :refer [delete-schema! with-transform-cleanup!]][metabase.transforms.util :as transforms.util]- [toucan2.core :as t2]- [toucan2.util :as u]))+ [metabase.util :as u]+ [toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -368,8 +368,7 @@record-call! (fn [call-name] (swap! calls conj call-name))mock-query "CREATE TABLE test_schema.test_table AS (SELECT 1 AS x);"mock-rows [[1]]- transform-details {:conn-spec {}- :output-table :test_schema/test_table+ transform-details {:output-table :test_schema/test_table:database {:id 1}:transform-type :table}](with-redefs [driver/table-exists? (constantly true)@@ -429,3 +428,110 @@clojure.lang.ExceptionInfo#"Failed to run the transform \(Test transform\) because the database \(.+\) has database routing turned on. Running transforms on databases with db routing enabled is not supported."(transforms.execute/execute! transform {:run-method :cron})))))))))++;; TODO(Timothy, 02-16-26):+;; Like no-create-schema-permissions-test, this would be improved by being driver agnostic+(deftest write-connection-used-for-transforms-test+ (mt/test-driver :postgres+ (mt/dataset transforms-dataset/transforms-test+ (let [details (:details (mt/db))+ password (:password details)+ readonly-user (u/lower-case-en (mt/random-name))+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))+ schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))+ make-widget-query+ (fn [mp]+ (let [transforms-products (lib.metadata/table mp (mt/id :transforms_products))+ products-category (lib.metadata/field mp (mt/id :transforms_products :category))+ products-id (lib.metadata/field mp (mt/id :transforms_products :id))]+ (-> (lib/query mp transforms-products)+ (lib/filter (lib/= products-category "Widget"))+ (lib/order-by products-id :asc))))]+ (try+ (driver/execute-raw-queries! driver/*driver* spec+ [[(format "CREATE ROLE %s WITH LOGIN PASSWORD '%s';" readonly-user password)]+ [(format "GRANT USAGE ON SCHEMA %s TO %s;" schema readonly-user)]+ [(format "GRANT SELECT ON ALL TABLES IN SCHEMA %s TO %s;" schema readonly-user)]+ ;; Grant SELECT on future tables created by the write user,+ ;; so the readonly user can query transform output tables.+ [(format "ALTER DEFAULT PRIVILEGES FOR ROLE %s IN SCHEMA %s GRANT SELECT ON TABLES TO %s;"+ (:user details) schema readonly-user)]+ ;; PG < 15 grants CREATE on `public` to PUBLIC by default.+ ;; Revoke it so the readonly user truly cannot create tables. :thinking_face:+ [(format "REVOKE CREATE ON SCHEMA %s FROM PUBLIC;" schema)]])+ (let [readonly-details (assoc details :user readonly-user)+ write-details {:user (:user details)+ :password (:password details)}+ run-transform!+ (fn [db-opts target-name]+ (mt/with-temp [:model/Database db (merge {:engine :postgres} db-opts)]+ (mt/with-db db+ (sync/sync-database! db {:scan :schema})+ (let [query (make-widget-query (mt/metadata-provider))]+ (with-transform-cleanup! [target-table {:type "table"+ :schema schema+ :name target-name}]+ (mt/with-temp [:model/Transform transform {:name target-name+ :source {:type :query+ :query query}+ :target target-table}]+ (transforms.execute/execute! transform {:run-method :manual})+ (let [_ (transforms.tu/wait-for-table (:name target-table) 10000)+ mp (mt/metadata-provider)+ table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))]+ (->> (lib/query mp table-result)+ (qp/process-query)+ (mt/formatted-rows [int str str 2.0 str])+ (sort-by first <)))))))))]+ (testing "Negative: read-only user without write_data_details fails to run transform"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"permission denied"+ (run-transform! {:details readonly-details} "write_conn_neg"))))+ (testing "Positive: read-only user with write_data_details succeeds"+ (is (= [[1 "Widget A" "Widget" 19.99 "2024-01-01T10:00:00Z"]+ [7 "Widget B" "Widget" 24.99 "2024-01-07T10:00:00Z"]+ [9 "Widget C" "Widget" 14.99 "2024-01-09T10:00:00Z"]+ [10 "Widget D" "Widget" 34.99 "2024-01-10T10:00:00Z"]+ [15 "Widget E" "Widget" 44.99 "2024-01-15T10:00:00Z"]]+ (run-transform! {:details readonly-details+ :write_data_details write-details}+ "write_conn_pos")))))+ (finally+ (driver/execute-raw-queries! driver/*driver* spec+ [;; [(format "GRANT CREATE ON SCHEMA %s TO PUBLIC;" schema)]+ [(format "DROP OWNED BY %s;" readonly-user)]+ [(format "DROP ROLE IF EXISTS %s;" readonly-user)]])))))))++(deftest transform-creates-write-pool-test+ (mt/test-drivers (mt/normal-driver-select {:+parent :sql-jdbc, :+features [:transforms/table]})+ (mt/dataset transforms-dataset/transforms-test+ (let [db-id (mt/id)+ write-cache-key [db-id :write-data]+ schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))+ old-write-details (:write_data_details (mt/db))]+ (when-not old-write-details+ (t2/update! :model/Database db-id {:write_data_details (:details (mt/db))}))+ (try+ (sql-jdbc.conn/invalidate-pool-for-db! (mt/db))+ (testing "write pool does not exist before transform execution"+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))+ (with-transform-cleanup! [target-table {:type "table"+ :schema schema+ :name "pool_test"}]+ (let [mp (mt/metadata-provider)+ transforms-products (lib.metadata/table mp (mt/id :transforms_products))+ products-category (lib.metadata/field mp (mt/id :transforms_products :category))+ query (-> (lib/query mp transforms-products)+ (lib/filter (lib/= products-category "Widget")))]+ (mt/with-temp [:model/Transform transform {:name "pool-test"+ :source {:type :query+ :query query}+ :target target-table}]+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table (:name target-table) 10000)+ (testing "write pool is created during transform execution"+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))))+ (finally+ (t2/update! :model/Database db-id {:write_data_details old-write-details})+ (sql-jdbc.conn/invalidate-pool-for-db! (mt/db))))))))diff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.cljindex 2801e7606dd..21503cef2e9 100644--- a/test/metabase/warehouse_schema_rest/api/table_test.clj+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj@@ -425,7 +425,7 @@(is (= (merge(-> (table-defaults)(dissoc :segments :field_values :metrics :measures :updated_at)- (update :db merge (select-keys (mt/db) [:details])))+ (update :db merge (select-keys (mt/db) [:details :write_data_details])))(t2/hydrate (t2/select-one [:model/Table :id :schema :name :created_at :initial_sync_status] :id (u/the-id table)):pk_field :collection){:description "What a nice table!"diff --git a/test/metabase/warehouses/models/database_test.clj b/test/metabase/warehouses/models/database_test.cljindex b32160182ef..38e3a983682 100644--- a/test/metabase/warehouses/models/database_test.clj+++ b/test/metabase/warehouses/models/database_test.clj@@ -77,9 +77,9 @@(mt/with-prometheus-system! [_ system](mt/with-temporary-setting-values [db-connection-timeout-ms 30000](database/check-health!)- (is (== 1.0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})))+ (is (== 1.0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})))(database/check-health!)- (is (== 1.0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true}))))))))))+ (is (== 1.0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"}))))))))))(deftest health-check-database-test(mt/test-drivers (mt/normal-drivers)@@ -89,47 +89,83 @@(mt/with-prometheus-system! [_ system](mt/with-temporary-setting-values [db-connection-timeout-ms 30000](database/health-check-database! (mt/db))- (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})) "healthy")- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input"})) "unhealthy user-input")- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception"})) "unhealthy exception"))))+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})) "healthy")+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input" :connection-type "default"})) "unhealthy user-input")+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "default"})) "unhealthy exception"))))(testing "skip audit"(mt/with-prometheus-system! [_ system](database/health-check-database! (assoc (mt/db) :is_audit true))- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})) "healthy")- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input"})) "unhealthy user-input")- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception"})) "unhealthy exception")))+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})) "healthy")+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input" :connection-type "default"})) "unhealthy user-input")+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "default"})) "unhealthy exception")))(testing "skip sample"(mt/with-prometheus-system! [_ system](database/health-check-database! (assoc (mt/db) :is_sample true))- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})) "healthy")- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input"})) "unhealthy user-input")- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception"})) "unhealthy exception")))+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})) "healthy")+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input" :connection-type "default"})) "unhealthy user-input")+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "default"})) "unhealthy exception")))(testing "failures for timeout"(mt/with-prometheus-system! [_ system]- (mt/with-temporary-setting-values [db-connection-timeout-ms -1] ;; setting to -1 because 0 sometimes flakes+ (mt/with-temporary-setting-values [db-connection-timeout-ms -1](database/health-check-database! (mt/db))- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})) "healthy")- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input"})) "unhealthy user-input")- (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception"})) "unhealthy exception"))))+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})) "healthy")+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input" :connection-type "default"})) "unhealthy user-input")+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "default"})) "unhealthy exception"))))(testing "failures for bad connections"(when-let [bad-conn (tx/bad-connection-details driver/*driver*)](mt/with-prometheus-system! [_ system](database/health-check-database! (update (mt/db) :details merge bad-conn))- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})) "healthy")- (is (or (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input"}))- (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception"}))) "unhealthy user-input or exception"))))+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})) "healthy")+ (is (or (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input" :connection-type "default"}))+ (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "default"}))) "unhealthy user-input or exception"))))(testing "failures for exception"(with-redefs [driver/can-connect? (fn [& _args] (throw (Exception. "boom")))](mt/with-prometheus-system! [_ system](database/health-check-database! (mt/db))- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true})) "healthy")- (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input"})) "unhealthy user-input")- (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception"})) "unhealthy exception"))))))))+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"})) "healthy")+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "user-input" :connection-type "default"})) "unhealthy user-input")+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "default"})) "unhealthy exception"))))))))++(deftest health-check-write-connection-test+ (mt/test-drivers (mt/normal-drivers)+ (with-redefs [quick-task/submit-task! (fn [task] (task))]+ (binding [driver.settings/*allow-testing-h2-connections* true]+ (testing "database with write_data_details checks both connections"+ (mt/with-prometheus-system! [_ system]+ (mt/with-temporary-setting-values [db-connection-timeout-ms 30000]+ (database/health-check-database! (assoc (mt/db) :write_data_details (:details (mt/db))))+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"}))+ "default connection healthy")+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "write-data"}))+ "write-data connection healthy"))))++ (testing "database without write_data_details only checks default connection"+ (mt/with-prometheus-system! [_ system]+ (mt/with-temporary-setting-values [db-connection-timeout-ms 30000]+ (database/health-check-database! (mt/db))+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"}))+ "default connection healthy")+ (is (== 0 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "write-data"}))+ "no write-data connection checked"))))++ (testing "write connection failure does not prevent default check"+ (let [call-count (atom 0)]+ (with-redefs [driver/can-connect? (fn [& _args]+ (if (< (swap! call-count inc) 2)+ true+ (throw (Exception. "write connection failed"))))]+ (mt/with-prometheus-system! [_ system]+ (mt/with-temporary-setting-values [db-connection-timeout-ms 30000]+ (database/health-check-database! (assoc (mt/db) :write_data_details (:details (mt/db))))+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy true :connection-type "default"}))+ "default connection healthy")+ (is (== 1 (mt/metric-value system :metabase-database/status {:driver driver/*driver* :healthy false :reason "exception" :connection-type "write-data"}))+ "write-data connection failed"))))))))))(deftest can-read-database-setting-test(let [encode-decode (comp json/decode json/encode)@@ -570,6 +606,98 @@(is (nil? (t2/select-one :model/Secret :id secret-id))(format "Secret ID %d was not removed from the app DB" secret-id)))))))))))+(deftest write-data-details-secrets-on-insert-test+ (testing "handle-incoming-client-secrets! processes write_data_details on insert"+ (mt/with-temp [:model/Database db {:engine "secret-test-driver"+ :name "Secret Test"+ :details {:host "localhost"}+ :write_data_details {:keystore-value "write-secret"}}]+ (let [db-id (:id db)+ raw-write-details (t2/select-one-fn+ (comp json/decode+kw :write_data_details)+ (t2/table-name :model/Database) db-id)]+ (testing "keystore-value replaced with keystore-id"+ (is (contains? raw-write-details :keystore-id))+ (is (not (contains? raw-write-details :keystore-value))))+ (testing "Secret record created with correct value"+ (when-let [secret-id (:keystore-id raw-write-details)]+ (is (=? {:value (u/string-to-bytes "write-secret") :source :uploaded :version 1}+ (secret/latest-for-id secret-id)))))))))++(deftest write-data-details-secrets-on-update-test+ (testing "handle-incoming-client-secrets! processes write_data_details on update"+ (mt/with-temp [:model/Database db {:engine "secret-test-driver"+ :name "Secret Test"+ :details {:host "localhost"}}]+ (let [db-id (:id db)]+ (t2/update! :model/Database db-id {:write_data_details {:keystore-value "write-secret"}})+ (let [raw-write-details (t2/select-one-fn+ (comp json/decode+kw :write_data_details)+ (t2/table-name :model/Database) db-id)]+ (testing "keystore-value replaced with keystore-id"+ (is (contains? raw-write-details :keystore-id))+ (is (not (contains? raw-write-details :keystore-value))))+ (testing "Secret record created with correct value"+ (when-let [secret-id (:keystore-id raw-write-details)]+ (is (=? {:value (u/string-to-bytes "write-secret") :source :uploaded :version 1}+ (secret/latest-for-id secret-id))))))))))++(deftest delete-orphaned-secrets-includes-write-data-details-test+ (testing "Secret records referenced in write_data_details are deleted when database is deleted"+ (let [db-table (t2/table-name :model/Database)]+ (mt/with-temp [:model/Secret {write-secret-id :id} {:name "write-secret" :value "secret-value" :kind "keystore"}]+ (mt/with-temp [db-table {db-id :id} {:engine (name :secret-test-driver)+ :name "Secret Test"+ :created_at (t/instant)+ :updated_at (t/instant)+ :details (json/encode {:host "localhost"})+ :write_data_details (json/encode {:keystore-id write-secret-id})}]+ (t2/delete! :model/Database :id db-id)+ (is (nil? (t2/select-one :model/Secret :id write-secret-id))+ "Secret referenced in write_data_details should be deleted"))))))++(deftest write-data-details-redaction-to-json-test+ (testing "to-json-hydrate-redacted-secrets hydrates redacted values in write_data_details"+ (mt/with-temp [:model/Secret {secret-id :id} {:name "secret" :value "stored-secret-value" :kind "s" :source "uploaded"}+ :model/Database db {:engine (name :secret-test-driver)+ :name "Secret Test"+ :details {:host "localhost"}}]+ (mt/with-current-user (mt/user->id :crowberto)+ (let [db-with-write-details (assoc db :write_data_details {:keystore-id secret-id})+ json-result (-> db-with-write-details json/encode json/decode+kw)]+ (testing "keystore-value hydrated with redacted placeholder"+ (is (= secret/protected-password+ (get-in json-result [:write_data_details :keystore-value]))))+ (testing "keystore-options hydrated"+ (is (= "uploaded"+ (get-in json-result [:write_data_details :keystore-options])))))))))++(deftest clean-secret-properties-from-write-data-details-test+ (testing "results-transform cleans secret properties from write_data_details"+ (let [db-table (t2/table-name :model/Database)]+ (mt/with-temp [db-table {db-id :id} {:engine (name :secret-test-driver)+ :name "Secret Test"+ :created_at (t/instant)+ :updated_at (t/instant)+ :details (json/encode {:host "localhost"})+ :write_data_details (json/encode {:keystore-value "should-be-cleaned"+ :keystore-id 999})}]+ (let [db (t2/select-one :model/Database db-id)]+ (testing "keystore-value cleaned from write_data_details"+ (is (not (contains? (:write_data_details db) :keystore-value)))))))))++(deftest after-select-keywordizes-auth-provider-in-details-test+ (testing "after-select keywordizes :auth-provider in both :details and :write_data_details"+ (mt/with-temp [:model/Database db {:engine :h2+ :name "Auth Provider Test"+ :details {:auth-provider "aws-iam"}+ :write_data_details {:auth-provider "aws-iam"}}]+ (let [db (t2/select-one :model/Database (:id db))]+ (testing ":details :auth-provider is keywordized"+ (is (keyword? (get-in db [:details :auth-provider]))))+ (testing ":write_data_details :auth-provider is keywordized"+ (is (keyword? (get-in db [:write_data_details :auth-provider]))))))))+(deftest user-may-not-update-sample-database-test(mt/with-temp [:model/Database {:keys [id] :as _sample-database} {:engine :h2:is_sample truediff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex 4fb2308652b..5fe7ee7aed2 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -155,7 +155,7 @@(testing "DB details visibility"(testing "Regular users should not see DB details"(is (= (-> (db-details)- (dissoc :details :schedules))+ (dissoc :details :write_data_details :schedules))(-> (mt/user-http-request :rasta :get 200 (format "database/%d" (mt/id)))(dissoc :schedules :can_upload)))))(testing "Superusers should see DB details"@@ -714,7 +714,7 @@(deftest ^:parallel fetch-database-metadata-test(testing "GET /api/database/:id/metadata"- (is (= (merge (dissoc (db-details) :details :router_user_attribute)+ (is (= (merge (dissoc (db-details) :details :write_data_details :router_user_attribute){:engine "h2":name "test-data (h2)":features (map u/qualified-name (driver.u/features :h2 (mt/db)))@@ -963,7 +963,7 @@(testing "Database details/settings *should not* come back for Rasta since she's not a superuser"(let [expected-keys (-> #{:features :native_permissions :can_upload :router_user_attribute :transforms_permissions}(into (keys (t2/select-one :model/Database :id (mt/id))))- (disj :details))]+ (disj :details :write_data_details))](doseq [db (:data (mt/user-http-request :rasta :get 200 "database"))](testing (format "Database %s %d %s" (:engine db) (u/the-id db) (pr-str (:name db)))(is (= expected-keys@@ -2425,7 +2425,21 @@driver/can-connect? (constantly false)](is (= {:status "error":message "Failed to connect to Database"}- (mt/user-http-request :crowberto :get 200 (str "database/" id "/healthcheck")))))))))+ (mt/user-http-request :crowberto :get 200 (str "database/" id "/healthcheck")))))))+ (testing "connection-type passed and configured"+ (mt/with-temp [:model/Database {id :id} {:details {:host "primary"}+ :write_data_details {:host "write"}}]+ (with-redefs [driver/available? (constantly true)+ driver/can-connect? (constantly true)]+ (is (= {:status "ok"}+ (mt/user-http-request :crowberto :get 200 (str "database/" id "/healthcheck?connection-type=write-data")))))))+ (testing "connection-type passed but not configured returns 400"+ (mt/with-temp [:model/Database {id :id} {:details {:host "primary"}}]+ (with-redefs [driver/available? (constantly true)]+ (is (mt/user-http-request :crowberto :get 400 (str "database/" id "/healthcheck?connection-type=write-data"))))))+ (testing "invalid connection-type value returns 400"+ (mt/with-temp [:model/Database {id :id} {}]+ (is (mt/user-http-request :crowberto :get 400 (str "database/" id "/healthcheck?connection-type=invalid")))))))(setting/defsetting api-test-missing-premium-feature"A feature used for testing /settings-available (1)"@@ -2649,3 +2663,172 @@(is (contains? response "Manage DB")))(testing "Database with no permissions should not be included"(is (not (contains? response "No Access DB"))))))))++;;; ----------------------------------------- write_data_details tests -----------------------------------------++(deftest ^:parallel upsert-sensitive-fields-write-data-details-test+ (testing "upsert-sensitive-fields works with :write_data_details key"+ (is (= {:host "localhost"+ :port 5432+ :password "new-password"}+ (#'api.database/upsert-sensitive-fields+ {:engine :h2+ :id (mt/id)+ :details {:host "localhost" :port 5432 :password "main-pass"}+ :write_data_details {:host "localhost" :port 5432 :password "write-pass"}}+ {:host "localhost"+ :port 5432+ :password "new-password"}+ :write_data_details)))+ (testing "protected passwords are replaced from original"+ (is (= {:host "localhost"+ :port 5432+ :password "write-pass"}+ (#'api.database/upsert-sensitive-fields+ {:engine :h2+ :id (mt/id)+ :details {:host "localhost" :port 5432 :password "main-pass"}+ :write_data_details {:host "localhost" :port 5432 :password "write-pass"}}+ {:host "localhost"+ :port 5432+ :password secret/protected-password}+ :write_data_details))))))++(deftest get-database-write-data-details-test+ (testing "GET /api/database/:id"+ (testing "Superusers see write_data_details with sensitive fields redacted"+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2+ :details {:host "localhost"}+ :write_data_details {:host "write-host"+ :password "secret-write-pass"}}]+ (let [response (mt/user-http-request :crowberto :get 200 (format "database/%d" db-id))]+ (is (some? (:write_data_details response)))+ (is (= "write-host" (get-in response [:write_data_details :host])))+ (is (= secret/protected-password (get-in response [:write_data_details :password]))))))+ (testing "Regular users do not see write_data_details"+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2+ :details {:host "localhost"}+ :write_data_details {:host "write-host"}}]+ (let [response (mt/user-http-request :rasta :get 200 (format "database/%d" db-id))]+ (is (not (contains? response :write_data_details)))+ (is (not (contains? response :details))))))))++(deftest update-database-write-data-details-test+ (testing "PUT /api/database/:id with write_data_details"+ (testing "Superusers can set write_data_details"+ (mt/with-premium-features #{:advanced-permissions}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2+ :details {:host "localhost"}}]+ (with-redefs [driver/can-connect? (constantly true)]+ (let [response (mt/user-http-request :crowberto :put 200 (format "database/%d" db-id)+ {:write_data_details {:host "write-host"+ :password "write-pass"+ :write-data-connection true}})]+ (is (= "write-host" (get-in response [:write_data_details :host])))+ (is (= secret/protected-password (get-in response [:write_data_details :password])))+ (let [db (t2/select-one :model/Database :id db-id)]+ (is (= {:host "write-host" :password "write-pass" :write-data-connection true}+ (:write_data_details db)))))))))+ (testing "Superusers can clear write_data_details by setting it to nil"+ (mt/with-premium-features #{:advanced-permissions}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2+ :details {:host "localhost"}+ :write_data_details {:host "write-host"}}]+ (with-redefs [driver/can-connect? (constantly true)]+ (mt/user-http-request :crowberto :put 200 (format "database/%d" db-id)+ {:write_data_details nil})+ (let [db (t2/select-one :model/Database :id db-id)]+ (is (nil? (:write_data_details db))))))))+ (testing "Sensitive fields are preserved when protected-password is sent"+ (mt/with-premium-features #{:advanced-permissions}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2+ :details {:host "localhost"}+ :write_data_details {:host "write-host"+ :password "original-pass"}}]+ (with-redefs [driver/can-connect? (constantly true)]+ (mt/user-http-request :crowberto :put 200 (format "database/%d" db-id)+ {:write_data_details {:host "new-write-host"+ :password secret/protected-password+ :write-data-connection true}})+ (let [db (t2/select-one :model/Database :id db-id)]+ (is (= "new-write-host" (get-in db [:write_data_details :host])))+ (is (= "original-pass" (get-in db [:write_data_details :password]))))))))+ (testing "Returns 402 without :advanced-permissions feature"+ (with-redefs [premium-features/has-feature? (constantly false)]+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2+ :details {:host "localhost"}}]+ (mt/user-http-request :crowberto :put 402 (format "database/%d" db-id)+ {:write_data_details {:host "write-host"}}))))))++(deftest put-validates-write-data-details-connection-test+ (testing "PUT /api/database/:id returns 400 when write connection test fails"+ (mt/with-premium-features #{:advanced-permissions}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2+ :details {:host "localhost"}}]+ (with-redefs [driver/can-connect? (fn [_engine details]+ (if (:write-data-connection details)+ (throw (Exception. "Write connection failed"))+ true))]+ (let [response (mt/user-http-request :crowberto :put 400 (format "database/%d" db-id)+ {:write_data_details {:host "totally-bogus-host"+ :write-data-connection true}})]+ (is (= "Write connection failed" (:message response)))))))))++(deftest write-data-details-guardrails-test+ (testing "PUT /api/database/:id write_data_details guardrails"+ (testing "write-data-connection must not be truthy in details"+ (mt/with-premium-features #{:advanced-permissions}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2+ :details {:host "localhost"}}]+ (is (= "write-data-connection must not be set in details"+ (mt/user-http-request :crowberto :put 400 (format "database/%d" db-id)+ {:details {:host "localhost"+ :write-data-connection true}}))))))+ (testing "write-data-connection must be truthy in write_data_details"+ (mt/with-premium-features #{:advanced-permissions}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2+ :details {:host "localhost"}}]+ (is (= "write-data-connection must be set in write_data_details"+ (mt/user-http-request :crowberto :put 400 (format "database/%d" db-id)+ {:write_data_details {:host "write-host"}}))))))+ (testing "Destination-database must be false in write_data_details"+ (mt/with-premium-features #{:advanced-permissions}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2+ :details {:host "localhost"}}]+ (is (= "destination-database must be false in write_data_details"+ (mt/user-http-request :crowberto :put 400 (format "database/%d" db-id)+ {:write_data_details {:host "write-host"+ :write-data-connection true+ :destination-database true}}))))))+ (testing "Fields hidden for write connections must not be in write_data_details"+ (mt/with-premium-features #{:advanced-permissions}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2+ :details {:host "localhost"}}]+ (is (str/includes?+ (mt/user-http-request :crowberto :put 400 (format "database/%d" db-id)+ {:write_data_details {:host "write-host"+ :write-data-connection true+ :auto_run_queries true}})+ "write_data_details must not contain fields hidden for write connections")))))+ (testing "Cannot set write_data_details on a destination database"+ (mt/with-premium-features #{:advanced-permissions}+ (mt/with-temp [:model/Database {router-id :id} {:engine :h2+ :details {:host "localhost"}}+ :model/Database {dest-id :id} {:engine :h2+ :details {:host "localhost"}+ :router_database_id router-id}]+ (is (= "Cannot configure a write connection on a destination database"+ (mt/user-http-request :crowberto :put 400 (format "database/%d" dest-id)+ {:write_data_details {:host "write-host"+ :write-data-connection true}}))))))+ (testing "Cannot set write_data_details on a router database"+ (mt/with-premium-features #{:advanced-permissions}+ (mt/with-temp [:model/Database {router-id :id} {:engine :h2+ :details {:host "localhost"}}+ :model/Database {_dest-id :id} {:engine :h2+ :details {:host "localhost"}+ :router_database_id router-id}]+ (is (= "Cannot configure a write connection on a router database"+ (mt/user-http-request :crowberto :put 400 (format "database/%d" router-id)+ {:write_data_details {:host "write-host"+ :write-data-connection true}}))))))))