The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GDGT-3229] read ClickHouse indexes from SHOW CREATE TABLE instead of system tables (#82573)
YB12a22c03928d
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.mdindex f4fbb0c3925..f679c4d54f7 100644--- a/docs/developers-guide/driver-changelog.md+++ b/docs/developers-guide/driver-changelog.md@@ -84,6 +84,10 @@ title: Driver interface changelog- `metabase.driver/compile-create-index` `[driver schema table structured]` -- compiles a `:standalone` index intothe DDL statement(s) that create it.+ - `metabase.driver/humanize-index-error-message` `[driver message]` -- trims the message of an exception raised by+ `fetch-table-indexes` or by index DDL before it is shown to the user. Defaults to returning it unchanged;+ ClickHouse drops the `(version ...) (queryId=...)` tail its server appends.+- `metabase.driver.sql-jdbc.sync.interface/db-tables` is now a multimethod for retrieving JDBC metadatatables. SQL JDBC drivers can override this method to customize which database objects are discovered during sync.diff --git a/frontend/src/metabase-types/api/index-manager.ts b/frontend/src/metabase-types/api/index-manager.tsindex 878b1143586..408e000c818 100644--- a/frontend/src/metabase-types/api/index-manager.ts+++ b/frontend/src/metabase-types/api/index-manager.ts@@ -130,6 +130,8 @@ export type ListTableIndexesRequest = {export type ListTableIndexesResponse = {data: TableIndexEntry[];+ /** Present when the warehouse couldn't be read, so every entry's presence is unverified. */+ warehouse_error?: string | null;};export type CreateTableIndexRequest = {diff --git a/frontend/src/metabase-types/api/mocks/index-manager.ts b/frontend/src/metabase-types/api/mocks/index-manager.tsindex 0a89be036b8..e6dbd9fd555 100644--- a/frontend/src/metabase-types/api/mocks/index-manager.ts+++ b/frontend/src/metabase-types/api/mocks/index-manager.ts@@ -1,6 +1,7 @@import type {IndexField,IndexMethod,+ ListTableIndexesResponse,RequestableIndexes,TableIndexEntry,TableIndexRequest,@@ -44,6 +45,13 @@ export const createMockTableIndexEntry = (...opts,});+export const createMockListTableIndexesResponse = (+ opts?: Partial<ListTableIndexesResponse>,+): ListTableIndexesResponse => ({+ data: [],+ ...opts,+});+const INDEX_NAME_FIELD: IndexField = {name: "name","display-name": "Give your index a name",diff --git a/frontend/src/metabase/api/index-manager.ts b/frontend/src/metabase/api/index-manager.tsindex 531b05308fc..7ed02a3f867 100644--- a/frontend/src/metabase/api/index-manager.ts+++ b/frontend/src/metabase/api/index-manager.ts@@ -17,20 +17,29 @@ import {provideTableIndexTags,} from "./tags";+export type ListTableIndexesResult = {+ indexes: TableIndexEntry[];+ warehouseError: string | null;+};+export const indexManagerApi = Api.injectEndpoints({endpoints: (builder) => ({- listTableIndexes: builder.query<TableIndexEntry[], ListTableIndexesRequest>(- {- query: (params) => ({- method: "GET",- url: "/api/index",- params,- }),- transformResponse: (response: ListTableIndexesResponse) =>- response.data,- providesTags: (indexes = []) => provideTableIndexListTags(indexes),- },- ),+ listTableIndexes: builder.query<+ ListTableIndexesResult,+ ListTableIndexesRequest+ >({+ query: (params) => ({+ method: "GET",+ url: "/api/index",+ params,+ }),+ transformResponse: (response: ListTableIndexesResponse) => ({+ indexes: response.data,+ warehouseError: response.warehouse_error ?? null,+ }),+ providesTags: (result) =>+ provideTableIndexListTags(result?.indexes ?? []),+ }),getTableIndex: builder.query<TableIndexRequest, TableIndexRequestId>({query: (id) => ({method: "GET",diff --git a/frontend/src/metabase/transforms/pages/TransformIndexesPage/TransformIndexesPage.tsx b/frontend/src/metabase/transforms/pages/TransformIndexesPage/TransformIndexesPage.tsxindex 1466c4131cb..2b584c08b6a 100644--- a/frontend/src/metabase/transforms/pages/TransformIndexesPage/TransformIndexesPage.tsx+++ b/frontend/src/metabase/transforms/pages/TransformIndexesPage/TransformIndexesPage.tsx@@ -1,5 +1,5 @@import { useState } from "react";-import { t } from "ttag";+import { c, t } from "ttag";import {skipToken,@@ -13,10 +13,12 @@ import { PageContainer } from "metabase/common/data-studio/components/PageContaiimport { TitleSection } from "metabase/common/data-studio/components/TitleSection";import { useToast } from "metabase/common/hooks";import { useConfirmation } from "metabase/common/hooks/use-confirmation";+import { useSelector } from "metabase/redux";import { useParams } from "metabase/router";+import { getApplicationName } from "metabase/selectors/whitelabel";import { trackTransformIndexDeleted } from "metabase/transforms/analytics";import { useTransformPermissions } from "metabase/transforms/hooks/use-transform-permissions";-import { Center } from "metabase/ui";+import { Alert, Center, Icon } from "metabase/ui";import * as Urls from "metabase/urls";import { isNullOrUndefined } from "metabase/utils/types";import type { TableIndexEntry, Transform } from "metabase-types/api";@@ -70,15 +72,20 @@ function TransformIndexesContent({readOnly: boolean | undefined;}) {const {- data: indexes = [],+ data: indexList,isLoading,error,} = useListTableIndexesQuery({ "transform-id": transform.id });+ const indexes = indexList?.indexes ?? [];+ const warehouseError = indexList?.warehouseError ?? null;const { deleteIndex, confirmationModal } = useDeleteIndex();const targetTableExists = transform.table != null;const hasRequestableIndexes =Object.keys(transform.requestable_indexes ?? {}).length > 0;const canCreate = targetTableExists && hasRequestableIndexes && !readOnly;+ const hasIndexes = indexes.length > 0;+ // An unreadable warehouse means we don't know the table is empty, so don't say so.+ const showEmptyState = !hasIndexes && warehouseError == null;const [editorState, setEditorState] = useState<{index?: TableIndexEntry;} | null>(null);@@ -96,6 +103,9 @@ function TransformIndexesContent({return (<>+ {warehouseError != null && (+ <UnreadableWarehouseAlert warehouseError={warehouseError} />+ )}<TitleSectionactions={<IndexPageActions@@ -106,9 +116,7 @@ function TransformIndexesContent({/>}>- {indexes.length === 0 ? (- <NoIndexes />- ) : (+ {hasIndexes && (<TransformIndexTableindexes={indexes}kindLabels={getKindLabels(transform.requestable_indexes)}@@ -117,6 +125,7 @@ function TransformIndexesContent({onDelete={deleteIndex}/>)}+ {showEmptyState && <NoIndexes />}</TitleSection>{editorState != null && (<IndexEditorModal@@ -130,6 +139,26 @@ function TransformIndexesContent({);}+function UnreadableWarehouseAlert({+ warehouseError,+}: {+ warehouseError: string;+}) {+ const applicationName = useSelector(getApplicationName);++ return (+ <Alert+ size="compact"+ color="warning"+ icon={<Icon name="warning" />}+ data-testid="warehouse-error-banner"+ >+ {c("{0} is the application name, {1} is the database's own error message")+ .t`${applicationName} couldn't read the indexes on this table, so statuses may be out of date. The database said: ${warehouseError}`}+ </Alert>+ );+}+function useDeleteIndex() {const [sendToast] = useToast();const [deleteTableIndex] = useDeleteTableIndexMutation();diff --git a/frontend/src/metabase/transforms/pages/TransformIndexesPage/TransformIndexesPage.unit.spec.tsx b/frontend/src/metabase/transforms/pages/TransformIndexesPage/TransformIndexesPage.unit.spec.tsxindex 9219f95c933..08a2f285ddc 100644--- a/frontend/src/metabase/transforms/pages/TransformIndexesPage/TransformIndexesPage.unit.spec.tsx+++ b/frontend/src/metabase/transforms/pages/TransformIndexesPage/TransformIndexesPage.unit.spec.tsx@@ -39,6 +39,7 @@ type SetupOpts = {indexes?: TableIndexEntry[];users?: UserListResult[];remoteSyncReadOnly?: boolean;+ warehouseError?: string | null;};function setup({@@ -46,6 +47,7 @@ function setup({indexes = [],users = [],remoteSyncReadOnly = false,+ warehouseError = null,}: SetupOpts = {}) {mockGetBoundingClientRect({ width: 1000, height: 600 });setupDatabasesEndpoints([@@ -58,7 +60,7 @@ function setup({setupUsersEndpoints(users);setupGetTransformEndpoint(transform);- setupListTableIndexesEndpoint(transform.id, indexes);+ setupListTableIndexesEndpoint(transform.id, indexes, warehouseError);const storeInitialState = createMockState({settings: mockSettings({@@ -191,6 +193,51 @@ describe("TransformIndexesPage", () => {).not.toBeInTheDocument();});+ it("warns with the warehouse's own error when the indexes could not be read", async () => {+ const warehouseError =+ "Code: 497. DB::Exception: Not enough privileges. (ACCESS_DENIED)";+ setup({+ indexes: [createMockTableIndexEntry({ name: "idx_orders_id" })],+ warehouseError,+ });+ await waitForLoaderToBeRemoved();++ expect(+ screen.getByText(+ `Metabase couldn't read the indexes on this table, so statuses may be out of date. The database said: ${warehouseError}`,+ ),+ ).toBeInTheDocument();+ expect(screen.getByText("idx_orders_id")).toBeInTheDocument();+ });++ it("does not claim the table has no indexes when the warehouse could not be read", async () => {+ setup({+ indexes: [],+ warehouseError:+ "Code: 497. DB::Exception: Not enough privileges. (ACCESS_DENIED)",+ });+ await waitForLoaderToBeRemoved();++ expect(screen.getByTestId("warehouse-error-banner")).toBeInTheDocument();+ expect(+ screen.queryByText(+ "Index the key columns of your transforms to make them faster and more efficient.",+ ),+ ).not.toBeInTheDocument();+ expect(+ screen.getByRole("button", { name: "Create index" }),+ ).toBeInTheDocument();+ });++ it("does not warn when the warehouse read succeeded", async () => {+ setup({ indexes: [createMockTableIndexEntry({ name: "idx_orders_id" })] });+ await waitForLoaderToBeRemoved();++ expect(+ screen.queryByTestId("warehouse-error-banner"),+ ).not.toBeInTheDocument();+ });+it("falls back to the index kind when the index has no name", async () => {setup({indexes: [diff --git a/frontend/test/__support__/server-mocks/index-manager.ts b/frontend/test/__support__/server-mocks/index-manager.tsindex 0536a31215a..3dc1b911107 100644--- a/frontend/test/__support__/server-mocks/index-manager.ts+++ b/frontend/test/__support__/server-mocks/index-manager.ts@@ -5,16 +5,23 @@ import type {TableIndexRequest,TransformId,} from "metabase-types/api";-import { createMockTableIndexRequest } from "metabase-types/api/mocks";+import {+ createMockListTableIndexesResponse,+ createMockTableIndexRequest,+} from "metabase-types/api/mocks";export function setupListTableIndexesEndpoint(transformId: TransformId,indexes: TableIndexEntry[] = [],+ warehouseError: string | null = null,) {fetchMock.get({url: `path:/api/index`,query: { "transform-id": transformId },- response: { data: indexes },+ response: createMockListTableIndexesResponse({+ data: indexes,+ warehouse_error: warehouseError,+ }),name: `listTableIndexes-${transformId}`,});}diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.cljindex cc38f22c987..72c29cc8966 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj@@ -293,6 +293,14 @@parts (filter identity (str/split (name s) #"\."))](str/join "." (map #(str "`" (escape-ident %) "`") parts))))+(defn- unquote-name+ "Undo [[quote-name]] for one identifier: strip the wrapping backticks and the backslash escapes inside them, so a+ name read back from DDL matches what the managed side stores. Bare names and expressions are left as they are."+ [^String s]+ (if (and (> (count s) 1) (str/starts-with? s "`") (str/ends-with? s "`"))+ (str/replace (subs s 1 (dec (count s))) #"\\(.)" "$1")+ s))+;;; +----------------------------------------------------------------------------------------------------------------+;;; | Indexes (Index Manager) |;;; +----------------------------------------------------------------------------------------------------------------+@@ -368,52 +376,87 @@`weird,name` becomes a bare element; a real expression like `lower(email)` stays one element."[expr](if-let [s (perf/not-empty expr)]- (perf/mapv #(driver.common/unquote-ident (str/trim %) \`)- (driver.common/split-top-level-commas (strip-wrapping-parens s)))+ (perf/mapv #(unquote-name (str/trim %))+ (driver.common/split-top-level-commas (strip-wrapping-parens s) \\))[]))-;; Named skip-indexes come from `system.data_skipping_indices`; the inline MergeTree sorting key-;; (`system.tables.sorting_key`) is emitted with `:name nil`. Blank `schema` falls back to `currentDatabase()`.+(def ^:private skip-index-line-regex+ "One data-skipping `INDEX` line of a `SHOW CREATE TABLE` statement. The name is back-quoted only when it has to be,+ with backslash escapes inside (see [[quote-name]]). Column definitions are always back-quoted, so a line that+ starts with the bare `INDEX` keyword is never one."+ #"^\s*INDEX\s+(`(?:[^`\\]|\\.)*`|\S+)\s+(.+?)\s+TYPE\s+(\w+(?:\([^)]*\))?)\s+GRANULARITY\s+(\d+),?\s*$")++(def ^:private order-by-line-regex+ "The MergeTree sorting key line. `PRIMARY KEY`, `PARTITION BY` and `TTL` print as lines of their own."+ #"^ORDER BY (.+)$")++(defn- line->definition+ "A statement line as a standalone clause: indentation and the list separator dropped."+ [line]+ (str/replace line #"^\s+|,?\s*$" ""))++(def ^:private index-defaults+ "The `::driver/table-index` fields a MergeTree index never varies: not unique, not primary, no covering columns, no+ partial predicate."+ {:is-unique false :is-primary false :is-valid true :include-columns [] :partial-predicate nil})++(defn- skip-index-line->index+ "The `::driver/table-index` for a data-skipping `INDEX` line, else nil."+ [line]+ (when-let [[_ index-name expr index-type] (re-matches skip-index-line-regex line)]+ (assoc index-defaults+ :name (unquote-name index-name)+ :kind :skip-index+ :access-method (first (str/split index-type #"\(" 2))+ :key-columns (expr->columns expr)+ :definition (line->definition line))))++(defn- order-by-line->index+ "The `::driver/table-index` for the `ORDER BY` line, else nil. `tuple()` is an unsorted table, so it yields nil too."+ [line]+ (when-let [[_ expr] (re-matches order-by-line-regex line)]+ (when (not= "tuple()" (str/trim expr))+ (assoc index-defaults+ :name nil+ :kind :order-by+ :access-method nil+ :key-columns (expr->columns expr)+ :definition (line->definition line)))))++(defn- create-table-statement->indexes+ "Parse a `SHOW CREATE TABLE` statement into `::driver/table-index` maps: the data-skipping indexes, then the sorting+ key."+ [statement]+ (let [lines (str/split-lines (or statement ""))]+ (into (vec (keep skip-index-line->index lines)) (keep order-by-line->index lines))))++(defn- table-missing-exception?+ "[[sql-jdbc/impl-table-known-to-not-exist?]] plus `CANNOT_GET_CREATE_TABLE_QUERY` (390), which only+ `SHOW CREATE TABLE` raises."+ [^SQLException e]+ (or (sql-jdbc/impl-table-known-to-not-exist? :clickhouse e)+ (str/starts-with? (or (ex-message e) "") "Code: 390.")))++(defmethod driver/humanize-index-error-message :clickhouse+ [_driver message]+ ;; Cut at the first `(ACCESS_DENIED)`-style code, which drops the `(version ...) (queryId=...)` tail.+ (or (second (re-find #"^(.*?\([A-Z][A-Z0-9_]+\))" message)) message))++;; One `SHOW CREATE TABLE` read, parsed by [[create-table-statement->indexes]]. It needs only `SHOW COLUMNS`, where+;; `system.data_skipping_indices` needs a grant Cloud Storage's shared ClickHouse doesn't give tenants.(defmethod driver/fetch-table-indexes :clickhouse[_driver database schema table](let [conn-spec (sql-jdbc.conn/db->pooled-connection-spec database)- db (perf/not-empty schema)- skip-idxs (->> (jdbc/query- conn-spec- [(str "SELECT name, type, type_full, expr, granularity "- "FROM system.data_skipping_indices "- "WHERE database = coalesce(?, currentDatabase()) AND table = ? "- "ORDER BY name")- db table])- (perf/mapv (fn [{:keys [name type type_full expr granularity]}]- {:name name- :kind :skip-index- :access-method type- :is-unique false- :is-primary false- :is-valid true- :key-columns (expr->columns expr)- :include-columns []- :partial-predicate nil- :definition (format "INDEX %s %s TYPE %s GRANULARITY %s"- name expr type_full granularity)})))- sorting (-> (jdbc/query- conn-spec- [(str "SELECT sorting_key FROM system.tables "- "WHERE database = coalesce(?, currentDatabase()) AND name = ?")- db table])- first :sorting_key)]- (cond-> skip-idxs- (perf/not-empty sorting) (conj {:name nil- :kind :order-by- :access-method nil- :is-unique false- :is-primary false- :is-valid true- :key-columns (expr->columns sorting)- :include-columns []- :partial-predicate nil- :definition (format "ORDER BY (%s)" sorting)}))))+ target (quote-name (if (seq schema) (keyword schema table) (keyword table)))]+ (try+ (-> (jdbc/query conn-spec [(str "SHOW CREATE TABLE " target)])+ first+ :statement+ create-table-statement->indexes)+ (catch SQLException e+ (if (table-missing-exception? e)+ []+ (throw e))))))(defn- create-table!-sql"Creates a ClickHouse table with the given name and column definitions. It assumes the engine is MergeTree,diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse/index_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse/index_test.cljindex 277a57d0afd..4dabd0d7911 100644--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse/index_test.clj+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse/index_test.clj@@ -6,7 +6,7 @@The rendering/capability checks are pure and need no connection. `order-by-inlined-live-test` and`skip-index-live-test` run the seams against a real ClickHouse and read the result back out of `system.tables`(the sorting key) and `system.data_skipping_indices` (the skip index), so the indexes are verified to actually take- effect on the physical table, not just to render."+ effect on the physical table, not just to render, and then again through `fetch-table-indexes`."(:require[clojure.java.jdbc :as jdbc][clojure.test :refer :all]@@ -122,13 +122,25 @@{:name name :type type :expr expr :granularity (long granularity)}))(def ^:private live-order-by-cases- "Each case drives one ORDER BY index through both live seams and asserts the sorting key the physical table reports."+ "Each case drives one ORDER BY index through both live seams and checks the sorting key from the catalog and through+ `fetch-table-indexes`. An unsorted table (`ORDER BY tuple()`) reads back as no index. `:definition` is the server's+ own formatting, which changes between versions, so it isn't pinned."[{:label "order-by index sets the MergeTree sorting key" :slug "idx":indexes [{:kind :order-by :columns [{:name "a"} {:name "b"}]}]- :expected "a, b"}+ :expected "a, b"+ :fetched [{:name nil+ :kind :order-by+ :access-method nil+ :is-unique false+ :is-primary false+ :is-valid true+ :key-columns ["a" "b"]+ :include-columns []+ :partial-predicate nil}]}{:label "no order-by index -> empty ORDER BY () (unsorted table)" :slug "noidx":indexes []- :expected ""}])+ :expected ""+ :fetched []}])(deftest ^:synchronized order-by-inlined-live-test(testing "an inlined ORDER BY actually sets the sorting key at both creation seams"@@ -136,7 +148,7 @@(let [details (mt/dbdef->connection-details :clickhouse :db {:database-name "default"})conn-spec (sql-jdbc.conn/connection-details->spec :clickhouse details)](mt/with-temp [:model/Database db {:engine :clickhouse, :details details}]- (doseq [{:keys [label slug indexes expected]} live-order-by-cases]+ (doseq [{:keys [label slug indexes expected fetched]} live-order-by-cases](testing label(let [ctas-table (str (gensym (str "mb_ob_ctas_" slug "_")))crt-table (str (gensym (str "mb_ob_crt_" slug "_")))@@ -150,7 +162,9 @@:query {:query "SELECT 1 AS a, 2 AS b"}:indexes indexes})](jdbc/execute! conn-spec (into [sql] params))- (is (= expected (sorting-key conn-spec "default" ctas-table))))+ (is (= expected (sorting-key conn-spec "default" ctas-table)))+ (testing "and fetch-table-indexes reports it, with the schema named"+ (is (=? fetched (driver/fetch-table-indexes :clickhouse db "default" ctas-table)))))(finally (drop! ctas-table))))(testing "CREATE TABLE seam (create-table!): create the table, read the sorting key back"(drop! crt-table)@@ -158,6 +172,8 @@(driver/create-table! :clickhouse (:id db) (keyword crt-table)order-by-columns {:indexes indexes})(is (= expected (sorting-key conn-spec "default" crt-table)))+ (testing "and fetch-table-indexes reports it, with a blank schema (the session database)"+ (is (=? fetched (driver/fetch-table-indexes :clickhouse db nil crt-table))))(finally (drop! crt-table))))))))))))(deftest ^:synchronized skip-index-live-test@@ -182,4 +198,67 @@(is (= {:type "minmax" :expr "(a)" :granularity 4}(-> (only-skip-index conn-spec "default" table)(select-keys [:type :expr :granularity])))))+ (testing "and fetch-table-indexes reports it (no sorting key on this table)"+ (is (=? [{:name "evt_minmax"+ :kind :skip-index+ :access-method "minmax"+ :is-unique false+ :is-primary false+ :is-valid true+ :key-columns ["a"]+ :include-columns []+ :partial-predicate nil}]+ (driver/fetch-table-indexes :clickhouse db "default" table))))(finally (drop!))))))))++(deftest ^:synchronized escaped-names-live-test+ (testing "a backtick in an index or column name survives the round trip through SHOW CREATE TABLE"+ (mt/test-driver :clickhouse+ (let [details (mt/dbdef->connection-details :clickhouse :db {:database-name "default"})+ conn-spec (sql-jdbc.conn/connection-details->spec :clickhouse details)+ table (str (gensym "mb_esc_"))+ column "col`tick"+ index {:name "ix`tick" :columns [{:name column}] :type :minmax :granularity 1}+ drop! (fn [] (jdbc/execute! conn-spec [(format "DROP TABLE IF EXISTS `%s`" table)]))]+ (mt/with-temp [:model/Database db {:engine :clickhouse, :details details}]+ (drop!)+ (try+ (driver/create-table! :clickhouse (:id db) (keyword table) [[column "Int64"]]+ {:indexes [{:kind :order-by :columns [{:name column}]}]})+ (driver/execute-raw-queries! :clickhouse conn-spec+ (driver/compile-create-index :clickhouse nil table index))+ (is (=? [{:name "ix`tick" :kind :skip-index :key-columns [column]}+ {:name nil :kind :order-by :key-columns [column]}]+ (driver/fetch-table-indexes :clickhouse db "default" table)))+ (finally (drop!))))))))++(deftest ^:synchronized restricted-user-live-test+ (testing "reading indexes needs only SELECT on the table; the system database stays off-limits"+ (mt/test-driver :clickhouse+ (let [admin-details (mt/dbdef->connection-details :clickhouse :db {:database-name "default"})+ admin-spec (sql-jdbc.conn/connection-details->spec :clickhouse admin-details)+ user "mb_index_reader"+ reader-details (assoc admin-details :user user :password "")+ reader-spec (sql-jdbc.conn/connection-details->spec :clickhouse reader-details)+ table (str (gensym "mb_reader_"))+ index {:name "evt_minmax" :columns [{:name "a"}] :type :minmax :granularity 4}+ exec! (fn [sql] (jdbc/execute! admin-spec [sql]))]+ (mt/with-temp [:model/Database db {:engine :clickhouse, :details reader-details}]+ (exec! (format "DROP TABLE IF EXISTS `%s`" table))+ (exec! (format "DROP USER IF EXISTS `%s`" user))+ (try+ (exec! (format "CREATE USER `%s` NOT IDENTIFIED" user))+ (exec! (format "GRANT SELECT ON default.* TO `%s`" user))+ (exec! (format "CREATE TABLE `%s` (`a` Int64, `b` Int64) ENGINE = MergeTree ORDER BY (`a`, `b`)" table))+ (driver/execute-raw-queries! :clickhouse admin-spec+ (driver/compile-create-index :clickhouse nil table index))+ (testing "the catalog table the old read used is denied to this user"+ (is (thrown-with-msg? Exception #"ACCESS_DENIED"+ (jdbc/query reader-spec ["SELECT count() FROM system.data_skipping_indices"]))))+ (testing "fetch-table-indexes still reads both index kinds"+ (is (=? [{:name "evt_minmax" :kind :skip-index :access-method "minmax" :key-columns ["a"]}+ {:name nil :kind :order-by :key-columns ["a" "b"]}]+ (driver/fetch-table-indexes :clickhouse db "default" table))))+ (finally+ (exec! (format "DROP TABLE IF EXISTS `%s`" table))+ (exec! (format "DROP USER IF EXISTS `%s`" user)))))))))diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.cljindex 0f2fabfc790..16b3a2c6c7a 100644--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj@@ -72,10 +72,211 @@;; a backtick-quoted name can hold a comma or paren; it must stay one column and come out bare"`weird,name`, b" ["weird,name" "b"]"`paren(col`, b" ["paren(col" "b"]- "`back``tick`" ["back`tick"] ; doubled backtick is an escaped backtick+ "`back\\`tick`" ["back`tick"] ; a backtick in a quoted name is backslash-escaped+ "`back\\\\slash`" ["back\\slash"]+ "`col\\`tick`, b" ["col`tick" "b"] ; the escaped backtick doesn't end the quoted span"" [] ; blank -> [], so :key-columns stays schema-validnil [])))+(defn- statement+ "Join `lines` into a `SHOW CREATE TABLE` statement, the way ClickHouse formats one."+ [& lines]+ (str/join "\n" lines))++(def ^:private every-skip-index-shape-statement+ "Skip-index line shapes: one column, several columns, an expression, a back-quoted name, a parameterised type, and+ an index literally named INDEX."+ (statement "CREATE TABLE tenant_a.t"+ "("+ " `id` Nullable(UInt64),"+ " `weird,name` Nullable(String),"+ " `email` Nullable(String),"+ " INDEX idx_id (id) TYPE minmax GRANULARITY 1,"+ " INDEX idx_multi (id, `weird,name`) TYPE bloom_filter GRANULARITY 2,"+ " INDEX idx_expr lower(email) TYPE bloom_filter GRANULARITY 1,"+ " INDEX `odd name` (`weird,name`) TYPE minmax GRANULARITY 1,"+ " INDEX idx_set (email) TYPE set(100) GRANULARITY 1,"+ " INDEX INDEX (`weird,name`, email) TYPE minmax GRANULARITY 1"+ ")"+ "ENGINE = MergeTree"+ "ORDER BY (id)"+ "SETTINGS allow_nullable_key = 1, index_granularity = 8192"))++(def ^:private keyword-named-columns-statement+ "Columns named INDEX and TYPE, and a PRIMARY KEY line that differs from the sorting key."+ (statement "CREATE TABLE tenant_a.`odd table`"+ "("+ " `INDEX` Int64,"+ " `TYPE` String,"+ " INDEX i1 (INDEX) TYPE minmax GRANULARITY 1"+ ")"+ "ENGINE = MergeTree"+ "PRIMARY KEY (INDEX)"+ "ORDER BY (INDEX, TYPE)"+ "SETTINGS index_granularity = 8192"))++(def ^:private unsorted-statement+ (statement "CREATE TABLE tenant_a.noidx"+ "("+ " `a` Int64"+ ")"+ "ENGINE = MergeTree"+ "ORDER BY tuple()"+ "SETTINGS index_granularity = 8192"))++(def ^:private shared-merge-tree-statement+ "Metabase Cloud Storage's engine, captured from a tenant database on stats.metabase.com."+ (statement "CREATE TABLE db_c6633c128ed24e74.test_index"+ "("+ " `session_id` String,"+ " `event_type` UInt8,"+ " `count` UInt64"+ ")"+ "ENGINE = SharedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')"+ "ORDER BY (event_type, session_id)"+ "SETTINGS allow_nullable_key = 1, index_granularity = 8192"))++(def ^:private pre-26-statement+ "23.3 and 25.2 print a single-column index expression and PRIMARY KEY bare; 26.6+ wraps them in parentheses.+ Captured from 25.2.2.39."+ (statement "CREATE TABLE default.mb_probe_kw"+ "("+ " `INDEX` Int64,"+ " `TYPE` String,"+ " `email` Nullable(String),"+ " INDEX i1 INDEX TYPE minmax GRANULARITY 1,"+ " INDEX i2 email TYPE bloom_filter GRANULARITY 3"+ ")"+ "ENGINE = MergeTree"+ "PRIMARY KEY INDEX"+ "ORDER BY (INDEX, TYPE)"+ "SETTINGS index_granularity = 8192"))++(def ^:private projection-statement+ "A projection carries its own, indented ORDER BY; the table's sorting key is one back-quoted column. Captured from+ 25.2.2.39 (26.8 prints the index expression as `(b)`)."+ (statement "CREATE TABLE default.mb_probe_shapes"+ "("+ " `a` Nullable(Int64),"+ " `weird name` Nullable(String),"+ " `b` String,"+ " INDEX ng b TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 1,"+ " PROJECTION p1"+ " ("+ " SELECT"+ " a,"+ " b"+ " ORDER BY b"+ " )"+ ")"+ "ENGINE = MergeTree"+ "ORDER BY `weird name`"+ "SETTINGS allow_nullable_key = 1, index_granularity = 8192"))++(def ^:private escaped-names-statement+ "Names holding a backtick or a backslash are back-quoted with backslash escapes. Captured from 26.8.5."+ (statement "CREATE TABLE default.mb_probe_esc"+ "("+ " `a` Int64,"+ " `col\\`tick` Int64,"+ " INDEX `ix\\`tick` (a) TYPE minmax GRANULARITY 1,"+ " INDEX `ix\\\\back` (`col\\`tick`) TYPE minmax GRANULARITY 1,"+ " INDEX `ix space` (a) TYPE minmax GRANULARITY 1"+ ")"+ "ENGINE = MergeTree"+ "ORDER BY `col\\`tick`"+ "SETTINGS index_granularity = 8192"))++(deftest ^:parallel create-table-statement->indexes-test+ (testing "every data-skipping INDEX line is parsed, in statement order (no live DB needed)"+ (let [idxs (#'clickhouse/create-table-statement->indexes every-skip-index-shape-statement)+ skip (filter #(= :skip-index (:kind %)) idxs)]+ (is (= ["idx_id" "idx_multi" "idx_expr" "odd name" "idx_set" "INDEX"]+ (map :name skip)))+ (is (= [["id"] ["id" "weird,name"] ["lower(email)"] ["weird,name"] ["email"] ["weird,name" "email"]]+ (map :key-columns skip)))+ (testing "the access method drops the type's arguments"+ (is (= ["minmax" "bloom_filter" "bloom_filter" "minmax" "set" "minmax"]+ (map :access-method skip))))+ (testing "the definition is the line itself, without indentation or the list separator"+ (is (= "INDEX idx_multi (id, `weird,name`) TYPE bloom_filter GRANULARITY 2"+ (:definition (second skip)))))+ (testing "the whole map matches the cross-driver shape"+ (is (= {:name "idx_id"+ :kind :skip-index+ :access-method "minmax"+ :is-unique false+ :is-primary false+ :is-valid true+ :key-columns ["id"]+ :include-columns []+ :partial-predicate nil+ :definition "INDEX idx_id (id) TYPE minmax GRANULARITY 1"}+ (first skip))))))+ (testing "the sorting key is emitted as one unnamed :order-by entry"+ (is (= [{:name nil+ :kind :order-by+ :access-method nil+ :is-unique false+ :is-primary false+ :is-valid true+ :key-columns ["id"]+ :include-columns []+ :partial-predicate nil+ :definition "ORDER BY (id)"}]+ (filter #(= :order-by (:kind %)) (#'clickhouse/create-table-statement->indexes+ every-skip-index-shape-statement)))))+ (testing "columns named INDEX/TYPE parse, and PRIMARY KEY is not mistaken for the sorting key"+ (is (=? [{:name "i1" :kind :skip-index :key-columns ["INDEX"]}+ {:name nil :kind :order-by :key-columns ["INDEX" "TYPE"]}]+ (#'clickhouse/create-table-statement->indexes keyword-named-columns-statement))))+ (testing "ORDER BY tuple() is an unsorted table: no entry, matching the empty catalog sorting key"+ (is (= [] (#'clickhouse/create-table-statement->indexes unsorted-statement)))))++(deftest ^:parallel create-table-statement->indexes-server-shapes-test+ (testing "a SharedMergeTree table with no skip index yields only its sorting key"+ (is (=? [{:name nil :kind :order-by :key-columns ["event_type" "session_id"]+ :definition "ORDER BY (event_type, session_id)"}]+ (#'clickhouse/create-table-statement->indexes shared-merge-tree-statement))))+ (testing "pre-26 servers print a single-column expression and PRIMARY KEY without parentheses"+ (is (=? [{:name "i1" :kind :skip-index :access-method "minmax" :key-columns ["INDEX"]+ :definition "INDEX i1 INDEX TYPE minmax GRANULARITY 1"}+ {:name "i2" :kind :skip-index :access-method "bloom_filter" :key-columns ["email"]}+ {:name nil :kind :order-by :key-columns ["INDEX" "TYPE"]}]+ (#'clickhouse/create-table-statement->indexes pre-26-statement))))+ (testing "a projection's indented ORDER BY is not the sorting key; a bare back-quoted single column is"+ (is (=? [{:name "ng" :kind :skip-index :access-method "ngrambf_v1" :key-columns ["b"]}+ {:name nil :kind :order-by :key-columns ["weird name"] :definition "ORDER BY `weird name`"}]+ (#'clickhouse/create-table-statement->indexes projection-statement))))+ (testing "back-quoted names come out with their backslash escapes undone, matching the managed side"+ (is (=? [{:name "ix`tick" :kind :skip-index :key-columns ["a"]}+ {:name "ix\\back" :kind :skip-index :key-columns ["col`tick"]}+ {:name "ix space" :kind :skip-index :key-columns ["a"]}+ {:name nil :kind :order-by :key-columns ["col`tick"]}]+ (#'clickhouse/create-table-statement->indexes escaped-names-statement)))))++(deftest ^:parallel table-missing-exception?-test+ (testing "the codes SHOW CREATE TABLE raises for a table or database that isn't there"+ (are [message] (#'clickhouse/table-missing-exception? (java.sql.SQLException. message))+ "Code: 60. DB::Exception: Table default.nope does not exist. (UNKNOWN_TABLE)"+ "Code: 81. DB::Exception: Database nope does not exist. (UNKNOWN_DATABASE)"+ "Code: 390. DB::Exception: Table `nope` doesn't exist. (CANNOT_GET_CREATE_TABLE_QUERY)"))+ (testing "a privilege error is not a missing table, so fetch-table-indexes rethrows it"+ (is (false? (#'clickhouse/table-missing-exception?+ (java.sql.SQLException. "Code: 497. DB::Exception: Not enough privileges. (ACCESS_DENIED)"))))))++(deftest ^:parallel humanize-index-error-message-test+ (testing "the error code ends the useful part of a message; the version/queryId tail is dropped"+ (is (= (str "Code: 497. DB::Exception: user_x: Not enough privileges. To execute this query, it's necessary to "+ "have the grant SELECT ON system.data_skipping_indices. (ACCESS_DENIED)")+ (driver/humanize-index-error-message+ :clickhouse+ (str "Code: 497. DB::Exception: user_x: Not enough privileges. To execute this query, it's necessary to "+ "have the grant SELECT ON system.data_skipping_indices. (ACCESS_DENIED) "+ "(version 26.6.1.2047 (official build)) (queryId= 71d1c3e4-0000-0000-0000-000000000000)")))))+ (testing "a message with no such code is kept as-is"+ (is (= "Connection refused" (driver/humanize-index-error-message :clickhouse "Connection refused")))))+(deftest ^:parallel inline-value-string-test(testing "inlined string literals escape the backslash before the quote";; ClickHouse treats `\` as an escape character inside a string literal, so doubling `'` alone (the defaultdiff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex 69625bb8353..c94841d9ed1 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -1841,6 +1841,17 @@(throw (ex-info (format "fetch-table-indexes is not implemented for driver %s" driver){:driver driver})))+(defmulti humanize-index-error-message+ "Trim `message`, from an exception an index operation raised ([[fetch-table-indexes]] or the DDL from+ [[compile-create-index]]), for display to the user. The default returns it unchanged."+ {:added "0.64.0", :arglists '([driver message])}+ dispatch-on-initialized-driver+ :hierarchy #'hierarchy)++(defmethod humanize-index-error-message :default+ [_driver message]+ message)+(defmulti drop-table!"Drop a table named `table-name`. If the table doesn't exist it will not be dropped. `table-name` may be qualifiedby schema e.g.diff --git a/src/metabase/driver/common.clj b/src/metabase/driver/common.cljindex 814791940ec..404fdd0b3a2 100644--- a/src/metabase/driver/common.clj+++ b/src/metabase/driver/common.clj@@ -180,19 +180,24 @@(defn split-top-level-commas"Split `s` on commas that aren't nested in parens or inside a `` ` ``/`'`/`\"` quote span, so neither a function key- like `toStartOfInterval(d, INTERVAL 1 DAY)` nor a quoted name like `\"weird,name\"` is torn at an inner comma."- [^String s]- (loop [i 0, depth 0, q nil, start 0, acc []]- (if (< i (.length s))- (let [c (nth s i)]- (cond- q (recur (inc i) depth (when-not (= c q) q) start acc)- (or (= c \`) (= c \') (= c \")) (recur (inc i) depth c start acc)- (= c \() (recur (inc i) (inc depth) q start acc)- (= c \)) (recur (inc i) (dec depth) q start acc)- (and (= c \,) (zero? depth)) (recur (inc i) depth q (inc i) (conj acc (subs s start i)))- :else (recur (inc i) depth q start acc)))- (conj acc (subs s start)))))+ like `toStartOfInterval(d, INTERVAL 1 DAY)` nor a quoted name like `\"weird,name\"` is torn at an inner comma.+ `escape-char` (ClickHouse's backslash) makes the character after it inside a quote span literal, so an escaped+ quote doesn't end the span."+ ([s]+ (split-top-level-commas s nil))+ ([^String s escape-char]+ (loop [i 0, depth 0, q nil, start 0, acc []]+ (if (< i (.length s))+ (let [c (nth s i)]+ (cond+ (and q (= c escape-char)) (recur (+ i 2) depth q start acc)+ q (recur (inc i) depth (when-not (= c q) q) start acc)+ (or (= c \`) (= c \') (= c \")) (recur (inc i) depth c start acc)+ (= c \() (recur (inc i) (inc depth) q start acc)+ (= c \)) (recur (inc i) (dec depth) q start acc)+ (and (= c \,) (zero? depth)) (recur (inc i) depth q (inc i) (conj acc (subs s start i)))+ :else (recur (inc i) depth q start acc)))+ (conj acc (subs s start))))))(defn unquote-ident"Strip a wrapping `quote-char` pair off a quoted identifier (doubled `quote-char` unescaped), so it matches the barediff --git a/src/metabase/indexes/reconcile.clj b/src/metabase/indexes/reconcile.cljindex ade4e07db06..50c26d97855 100644--- a/src/metabase/indexes/reconcile.clj+++ b/src/metabase/indexes/reconcile.clj@@ -6,7 +6,6 @@[metabase.driver :as driver][metabase.indexes.schema :as indexes.schema][metabase.util :as u]- [metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.warehouses.schema]))@@ -136,15 +135,21 @@(request-fields row)))](into (vec present) absent)))-(mu/defn fetch-warehouse-indexes :- [:maybe [:sequential :map]]- "Physical indexes on `table-name` (`schema`) in `database` via `driver/fetch-table-indexes`.- Returns `nil` if the driver can't introspect indexes or the warehouse is unreachable, so callers can distinguish- fetch failure from a successful empty index list."+(mu/defn fetch-warehouse-indexes :- [:sequential :map]+ "Physical indexes on `table-name` (`schema`) in `database` via `driver/fetch-table-indexes`. Throws when the+ warehouse can't be read, so an empty result always means no indexes."[database :- :metabase.warehouses.schema/databaseschema :- [:maybe :string]table-name :- :string]- (try- (vec (driver/fetch-table-indexes (:engine database) database schema table-name))- (catch Throwable t- (log/warnf "fetch-table-indexes failed for %s.%s: %s" schema table-name (ex-message t))- nil)))+ (vec (driver/fetch-table-indexes (:engine database) database schema table-name)))++(def ^:private max-error-message-length+ "Display cap for a driver's error message; the column itself is unbounded."+ 500)++(defn driver-error-message+ "The message of `t`, an exception `driver` raised, for display: trimmed by+ [[driver/humanize-index-error-message]], then length-capped."+ [driver ^Throwable t]+ (u/truncate (driver/humanize-index-error-message driver (or (ex-message t) (str t)))+ max-error-message-length))diff --git a/src/metabase/indexes_rest/api.clj b/src/metabase/indexes_rest/api.cljindex 248160bae88..0f5481976de 100644--- a/src/metabase/indexes_rest/api.clj+++ b/src/metabase/indexes_rest/api.clj@@ -13,6 +13,7 @@[metabase.indexes.schema :as schema][metabase.transforms-base.interface :as transforms-base.i][metabase.util.i18n :refer [tru]]+ [metabase.util.log :as log][metabase.util.malli.schema :as ms][toucan2.core :as t2]))@@ -62,19 +63,26 @@[{:keys [transform_id]}](api/write-check :model/Transform transform_id))-(api.macros/defendpoint :get "/" :- [:map [:data [:sequential Index]]]+(api.macros/defendpoint :get "/" :- [:map+ [:data [:sequential Index]]+ [:warehouse_error {:optional true} [:maybe :string]]]"A transform's indexes: those physically in the warehouse, merged with its managed requests. Each entry is flagged- `:metabase_managed`; managed ones also carry `:request` (status + definition)."+ `:metabase_managed`; managed ones also carry `:request` (status + definition). If the warehouse can't be read, the+ managed requests are still listed (as absent) and `:warehouse_error` carries the reason."[_route-params{:keys [transform-id]} :- [:map {:closed true} [:transform-id ms/PositiveInt]]](let [transform (api/read-check :model/Transform transform-id)- database-id (transforms-base.i/target-db-id transform)+ database (api/check-404 (indexes-rest.db/database (transforms-base.i/target-db-id transform))){:keys [schema] table-name :name} (:target transform)managed (table-index/select-for-transform transform-id)- warehouse (or (reconcile/fetch-warehouse-indexes (indexes-rest.db/database database-id)- schema table-name)- [])]- {:data (reconcile/merge-indexes managed warehouse)}))+ [warehouse warehouse-error]+ (try+ [(reconcile/fetch-warehouse-indexes database schema table-name)]+ (catch Exception e+ (log/warnf "fetch-table-indexes failed for %s.%s: %s" schema table-name (ex-message e))+ [[] (reconcile/driver-error-message (:engine database) e)]))]+ (cond-> {:data (reconcile/merge-indexes managed warehouse)}+ warehouse-error (assoc :warehouse_error warehouse-error))))(api.macros/defendpoint :get "/request/:id" :- RequestIndex"Fetch a single index request (e.g. to poll its status)."diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex e35ab324e33..709fa4089e6 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -675,6 +675,15 @@(= status :failed)(assoc :error_message "Index was not found on the target table after the transform ran."))))))+(defn- mark-indexes-unverifiable!+ "Fail the still-running `managed` requests with the driver's reason when the warehouse read that verification needs+ has failed. Only running rows change, so the run's generic `finally` backstop finds nothing left to relabel."+ [driver managed schema table-name ^Throwable t]+ (log/warnf "verify-managed-indexes!: could not read indexes for %s.%s: %s" schema table-name (ex-message t))+ (table-index/mark-unverified-running-indexes-failed!+ (into #{} (map :id) managed)+ (str "Couldn't read the table's indexes to verify this one: " (reconcile/driver-error-message driver t))))+(defn verify-managed-indexes!"Reconcile each index request against what's physically in the warehouse and set its `:status`.Only runs on full-create runs (same guard as [[apply-target-indexes!]])."@@ -687,11 +696,13 @@(when-let [managed (seq managed)](let [database (transforms-base.db/database (transforms-base.i/target-db-id transform)){:keys [schema] table-name :name} (:target transform)]- (if-some [warehouse-indexes (reconcile/fetch-warehouse-indexes database schema table-name)]+ (when-some [warehouse-indexes (try+ (reconcile/fetch-warehouse-indexes database schema table-name)+ (catch Exception e+ (mark-indexes-unverifiable! (:engine database) managed schema table-name e)+ nil))](apply-index-outcomes!- (reconcile/classify-index-outcomes managed (reconcile/warehouse-key-set warehouse-indexes)))- (log/warnf "verify-managed-indexes!: could not read indexes for %s.%s; leaving %d request(s) unchanged"- schema table-name (count managed))))))))+ (reconcile/classify-index-outcomes managed (reconcile/warehouse-key-set warehouse-indexes)))))))))(defn complete-execution!"Post-processing steps after a transform has been executed successfully.diff --git a/test/metabase/indexes/reconcile_test.clj b/test/metabase/indexes/reconcile_test.cljindex 222e183c146..e18215f49a7 100644--- a/test/metabase/indexes/reconcile_test.clj+++ b/test/metabase/indexes/reconcile_test.clj@@ -135,6 +135,15 @@(testing "preserves a successful empty warehouse response"(with-redefs [metabase.driver/fetch-table-indexes (fn [& _] [])](is (= [] (reconcile/fetch-warehouse-indexes (mt/db) "public" "t")))))- (testing "swallows driver/connection errors and returns nil"+ (testing "lets driver/connection errors through, so callers can report the reason"(with-redefs [metabase.driver/fetch-table-indexes (fn [& _] (throw (ex-info "boom" {})))]- (is (nil? (reconcile/fetch-warehouse-indexes (mt/db) "public" "t"))))))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (reconcile/fetch-warehouse-indexes (mt/db) "public" "t"))))))++(deftest ^:parallel driver-error-message-test+ (testing "a driver with no trimming of its own passes the message through"+ (is (= "Connection refused" (reconcile/driver-error-message :h2 (ex-info "Connection refused" {})))))+ (testing "a long message is capped for readability"+ (is (= 500 (count (reconcile/driver-error-message :h2 (ex-info (apply str (repeat 900 "x")) {}))))))+ (testing "an exception with no message still yields text"+ (is (seq (reconcile/driver-error-message :h2 (java.sql.SQLException.))))))diff --git a/test/metabase/indexes_rest/api_test.clj b/test/metabase/indexes_rest/api_test.cljindex d7e0c171606..832f09fc408 100644--- a/test/metabase/indexes_rest/api_test.clj+++ b/test/metabase/indexes_rest/api_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][metabase.driver]+ [metabase.indexes.reconcile :as reconcile][metabase.test :as mt][metabase.transforms-base.util :as transforms-base.u][metabase.transforms.query-test-util :as query-test-util]@@ -93,6 +94,27 @@(testing "nothing unmanaged was persisted"(is (= 1 (count (t2/select :model/TableIndex :transform_id transform-id))))))))))+(deftest warehouse-read-failure-is-reported-test+ (testing "a failed warehouse read still lists the managed requests, with the driver's reason"+ (mt/with-temp [:model/Transform {transform-id :id} (temp-transform-spec)]+ (let [created (mt/user-http-request :crowberto :post 200 "index/request"+ {:transform_id transform-id :structured btree})+ denied (str "Code: 497. DB::Exception: Not enough privileges. (ACCESS_DENIED) "+ "(version 26.6.1.2047 (official build))")]+ (with-redefs [metabase.driver/fetch-table-indexes (fn [& _] (throw (ex-info denied {})))]+ (let [{:keys [data warehouse_error]} (mt/user-http-request :crowberto :get 200+ (str "index?transform-id=" transform-id))]+ (testing "the response is still a 200 carrying the trimmed driver message"+ (is (= (reconcile/driver-error-message (:engine (mt/db)) (ex-info denied {})) warehouse_error)))+ (testing "the managed request is projected as absent rather than dropped"+ (is (= [(:id created)] (map #(get-in % [:request :id]) data)))+ (is (false? (:present_in_warehouse (first data))))))))+ (testing "a successful read carries no warehouse_error"+ (with-redefs [metabase.driver/fetch-table-indexes (fn [& _] [])]+ (is (not (contains? (mt/user-http-request :crowberto :get 200+ (str "index?transform-id=" transform-id))+ :warehouse_error))))))))+(deftest index-endpoints-require-transform-permission-test(testing "every index endpoint inherits the transform's permission: a user without access is blocked from all of them"(mt/with-temp [:model/Transform {transform-id :id} (temp-transform-spec)]diff --git a/test/metabase/transforms/index_test.clj b/test/metabase/transforms/index_test.cljindex b898fb3c9d4..54524e51cd7 100644--- a/test/metabase/transforms/index_test.clj+++ b/test/metabase/transforms/index_test.clj@@ -7,6 +7,7 @@[clojure.string :as str][clojure.test :refer :all][metabase.driver :as driver]+ [metabase.indexes.reconcile :as reconcile][metabase.test :as mt][metabase.test.data.sql :as sql.tx][metabase.transforms-base.util :as transforms-base.u]@@ -296,3 +297,33 @@{:engine :postgres :id (mt/id)} (assoc (:target transform) :transform-id tid))))(is (= :failed (t2/select-one-fn :status :model/TableIndex idx-id)))(is (re-find #"ddl boom" (t2/select-one-fn :error_message :model/TableIndex idx-id)))))))++(def ^:private access-denied-message+ "A ClickHouse privilege error, as Cloud Storage's shared ClickHouse raises it for a tenant reading `system`."+ "Code: 497. DB::Exception: Not enough privileges. (ACCESS_DENIED) (version 26.6.1.2047 (official build))")++(deftest ^:synchronized unreadable-warehouse-marks-running-indexes-failed-test+ (testing "a failed warehouse read fails the requests this run started, carrying the driver's own reason"+ (mt/with-temp [:model/Transform {tid :id} {:name (mt/random-name)+ :source {:type "query"}+ :source_database_id (mt/id)+ :target {:database (mt/id) :type "table" :schema "public" :name "t"}}+ :model/TableIndex {running-id :id} {:transform_id tid :index_name "running_idx"+ :status :running+ :structured {:kind :btree :name "running_idx"+ :columns [{:name "a"}]}}+ :model/TableIndex {pending-id :id} {:transform_id tid :index_name "pending_idx"+ :structured {:kind :btree :name "pending_idx"+ :columns [{:name "b"}]}}]+ (with-redefs [driver/fetch-table-indexes (fn [& _] (throw (ex-info access-denied-message {})))]+ (transforms-base.u/verify-managed-indexes!+ {:id tid+ :source {:type "query"}+ :target {:database (mt/id) :type "table" :schema "public" :name "t"}}))+ (testing "the running request is failed, with the driver message trimmed of its version tail"+ (is (= :failed (t2/select-one-fn :status :model/TableIndex running-id)))+ (is (= (str "Couldn't read the table's indexes to verify this one: "+ (reconcile/driver-error-message (:engine (mt/db)) (ex-info access-denied-message {})))+ (t2/select-one-fn :error_message :model/TableIndex running-id))))+ (testing "a request this run never started is left for the next one"+ (is (= :create-pending (t2/select-one-fn :status :model/TableIndex pending-id)))))))