The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GDGT-2519: Surface hidden tables in Schema Viewer ERD (#75031)
Stas Gavrylov586dfd96f8dc
diff --git a/enterprise/backend/src/metabase_enterprise/erd/impl.clj b/enterprise/backend/src/metabase_enterprise/erd/impl.cljindex dbde20d6c99..d355f475c23 100644--- a/enterprise/backend/src/metabase_enterprise/erd/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/erd/impl.clj@@ -48,6 +48,7 @@[:description {:optional true} [:maybe :string]][:owner {:optional true} ::erd-owner][:schema {:optional true} [:maybe :string]]+ [:visibility_type {:optional true} [:maybe :keyword]][:db_id :int][:fields [:sequential ::erd-field]]])@@ -77,7 +78,7 @@(`:db_id`, `:is_published`, `:collection_id`) plus the display columns theERD response surfaces (`:description`, owner fields hydrated below)."[:id :db_id :name :display_name :schema :is_published :collection_id- :description :owner_user_id :owner_email])+ :description :owner_user_id :owner_email :visibility_type])(defn- schema-clause"The rest of the database API treats a blank schema name as the union of nil@@ -93,7 +94,7 @@(into {} (map (juxt :id identity)) xs))(defn- readable-tables- "Fetch readable, active, visible tables in `database-id`.+ "Fetch readable, active tables in `database-id`.Optional `table-ids` restricts by IDs; optional `schema` restricts by schema.`schema=\"\"` matches both nil and empty-string schemas."[database-id & {:keys [table-ids schema] :as opts}]@@ -101,8 +102,7 @@[](let [where (cond-> [:and[:= :db_id database-id]- [:= :active true]- [:= :visibility_type nil]]+ [:= :active true]](contains? opts :table-ids) (conj [:in :id table-ids])(contains? opts :schema) (conj (schema-clause schema)))](->> (t2/select :model/Table@@ -233,6 +233,7 @@:description (:description table):owner (:owner table):schema (:schema table)+ :visibility_type (:visibility_type table):db_id (:db_id table):fields (mapv #(build-erd-field % target-field-id->table-id) fields)})diff --git a/enterprise/backend/test/metabase_enterprise/erd/erd_test.clj b/enterprise/backend/test/metabase_enterprise/erd/erd_test.cljindex 044f59b62ee..c0f9bc0f612 100644--- a/enterprise/backend/test/metabase_enterprise/erd/erd_test.clj+++ b/enterprise/backend/test/metabase_enterprise/erd/erd_test.clj@@ -274,9 +274,9 @@(is (empty? (:nodes response)))(is (empty? (:edges response))))))))-(deftest erd-endpoint-excludes-hidden-and-sensitive-metadata-test+(deftest erd-endpoint-includes-hidden-tables-and-all-field-visibility-test(mt/with-premium-features #{:schema-viewer}- (testing "hidden tables are omitted; all field visibility types are included"+ (testing "all tables (incl. hidden/technical/cruft) are surfaced; all field visibility types are included"(mt/with-temp [:model/Database {db-id :id} {}:model/Table {visible-table-id :id} {:db_id db-id :name "visible" :schema "PUBLIC"}:model/Field _ {:table_id visible-table-id :name "visible_id"@@ -291,18 +291,28 @@:database_type "INTEGER" :base_type :type/Integer:visibility_type "retired"}:model/Table _ {:db_id db-id :name "hidden_table" :schema "PUBLIC"- :visibility_type "hidden"}]- (let [response (mt/user-http-request :rasta :get 200 "ee/erd"- :database-id db-id- :schema "PUBLIC")- node-names (set (map :name (:nodes response)))- field-names (set (map :name (mapcat :fields (:nodes response))))]- (is (= #{"visible"} node-names))- (is (= #{"visible_id" "hidden_field" "secret" "retired_field"} field-names)))))))+ :visibility_type "hidden"}+ :model/Table _ {:db_id db-id :name "technical_table" :schema "PUBLIC"+ :visibility_type "technical"}+ :model/Table _ {:db_id db-id :name "cruft_table" :schema "PUBLIC"+ :visibility_type "cruft"}]+ (let [response (mt/user-http-request :rasta :get 200 "ee/erd"+ :database-id db-id+ :schema "PUBLIC")+ node-names (set (map :name (:nodes response)))+ field-names (set (map :name (mapcat :fields (:nodes response))))+ visibility-by-name (into {} (map (juxt :name :visibility_type)) (:nodes response))]+ (is (= #{"visible" "hidden_table" "technical_table" "cruft_table"} node-names))+ (is (= #{"visible_id" "hidden_field" "secret" "retired_field"} field-names))+ (is (= {"visible" nil+ "hidden_table" "hidden"+ "technical_table" "technical"+ "cruft_table" "cruft"}+ visibility-by-name)))))))-(deftest erd-endpoint-does-not-leak-hidden-fk-targets-test+(deftest erd-endpoint-resolves-hidden-fk-targets-test(mt/with-premium-features #{:schema-viewer}- (testing "FK targets in hidden tables are nulled and do not produce edges; FKs to sensitive fields resolve"+ (testing "FK targets in hidden tables resolve into nodes and edges; FKs to sensitive fields resolve too"(mt/with-temp [:model/Database {db-id :id} {}:model/Table {hidden-target-table-id :id} {:db_id db-id :name "hidden_target" :schema "PUBLIC":visibility_type "hidden"}@@ -331,12 +341,14 @@fields-by-id (into {} (map (juxt :id identity)) (:fields source-node))hidden-fk (get fields-by-id hidden-table-fk-id)sensitive-fk (get fields-by-id sensitive-field-fk-id)]- (is (not (contains? nodes-by-name "hidden_target")))+ (is (contains? nodes-by-name "hidden_target"))(is (contains? nodes-by-name "visible_target"))- (testing "FK to a field in a hidden table is nulled and produces no edge"- (is (nil? (:fk_target_table_id hidden-fk)))- (is (nil? (:fk_target_field_id hidden-fk)))- (is (not-any? #(= hidden-table-fk-id (:source_field_id %)) (:edges response))))+ (testing "FK to a field in a hidden table resolves and produces an edge"+ (is (= hidden-target-table-id (:fk_target_table_id hidden-fk)))+ (is (= hidden-target-field-id (:fk_target_field_id hidden-fk)))+ (is (some #(and (= hidden-table-fk-id (:source_field_id %))+ (= hidden-target-field-id (:target_field_id %)))+ (:edges response))))(testing "FK to a sensitive field in a visible table resolves normally"(is (= visible-target-table-id (:fk_target_table_id sensitive-fk)))(is (= sensitive-target-field-id (:fk_target_field_id sensitive-fk)))diff --git a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/SchemaPickerInput/SchemaPickerInput.tsx b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/SchemaPickerInput/SchemaPickerInput.tsxindex 58307084756..cae8ca9ed59 100644--- a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/SchemaPickerInput/SchemaPickerInput.tsx+++ b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/SchemaPickerInput/SchemaPickerInput.tsx@@ -52,6 +52,7 @@ export function SchemaPickerInput({opened={opened}onClose={() => setOpened(false)}models={["schema"]}+ includeHiddenSchemasonChange={handleChange}menuProps={{// This allows to toggle menu by clicking the trigger button.diff --git a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.tsx b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.tsxindex 1c7b786f51f..68e47ffbae0 100644--- a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.tsx+++ b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.tsx@@ -1,8 +1,9 @@import type { NodeProps } from "@xyflow/react";import cx from "classnames";import { memo, useCallback } from "react";+import { t } from "ttag";-import { Box, FixedSizeIcon, Group, Stack } from "metabase/ui";+import { Box, FixedSizeIcon, Group, Stack, Tooltip } from "metabase/ui";import { useSchemaViewerContext } from "../../SchemaViewerContext";import type { SchemaViewerFlowNode } from "../../types";@@ -22,6 +23,8 @@ export const SchemaViewerTableNode = memo(function SchemaViewerTableNode({const isConnectedToSelectedEdge = data.selectedFieldIds.size > 0;const isUserSelected = selectedNodeId === id;+ const hasHiddenVisibilityType = data.visibility_type !== null;+const handleDoubleClick = useCallback(() => {zoomToNode(id);}, [id, zoomToNode]);@@ -66,6 +69,11 @@ export const SchemaViewerTableNode = memo(function SchemaViewerTableNode({>{data.name}</Box>+ {hasHiddenVisibilityType && (+ <Tooltip label={t`This table is hidden`}>+ <FixedSizeIcon name="eye_crossed_out" c="text-secondary" />+ </Tooltip>+ )}</Group><Box className={S.fields}>{data.fields.map((field) => (diff --git a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.unit.spec.tsxnew file mode 100644index 00000000000..6fa07284d08--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.unit.spec.tsx@@ -0,0 +1,76 @@+import type { NodeProps } from "@xyflow/react";+import { ReactFlowProvider } from "@xyflow/react";++import { queryIcon, renderWithProviders, screen } from "__support__/ui";+import type { ConcreteTableId, TableVisibilityType } from "metabase-types/api";+import {+ createMockErdNode,+ createMockErdResponse,+} from "metabase-types/api/mocks";++import { SchemaViewerContext } from "../../SchemaViewerContext";+import type { SchemaViewerFlowNode } from "../../types";+import { toFlowGraph } from "../../utils/flow-graph";++import { SchemaViewerTableNode } from "./SchemaViewerTableNode";++function nodeProps(+ node: SchemaViewerFlowNode,+): NodeProps<SchemaViewerFlowNode> {+ return {+ id: node.id,+ data: node.data,+ type: "schemaViewerTable",+ dragging: false,+ zIndex: 2,+ selectable: true,+ deletable: false,+ selected: false,+ draggable: true,+ isConnectable: false,+ positionAbsoluteX: 0,+ positionAbsoluteY: 0,+ };+}++function setup({+ visibilityType = null,+}: { visibilityType?: TableVisibilityType } = {}) {+ const erdNode = createMockErdNode({+ table_id: 1,+ name: "Orders",+ visibility_type: visibilityType,+ });+ const [node] = toFlowGraph(createMockErdResponse({ nodes: [erdNode] })).nodes;++ const value = {+ visibleTableIds: new Set<ConcreteTableId>(),+ expandingTableIds: new Set<ConcreteTableId>(),+ expandToTable: jest.fn(),+ selectedNodeId: null,+ selectNode: jest.fn(),+ zoomToNode: jest.fn(),+ };++ renderWithProviders(+ <ReactFlowProvider>+ <SchemaViewerContext.Provider value={value}>+ <SchemaViewerTableNode {...nodeProps(node)} />+ </SchemaViewerContext.Provider>+ </ReactFlowProvider>,+ );+}++describe("SchemaViewerTableNode", () => {+ it("renders the hidden indicator for tables with a visibility_type", () => {+ setup({ visibilityType: "hidden" });+ expect(screen.getByText("Orders")).toBeInTheDocument();+ expect(queryIcon("eye_crossed_out")).toBeInTheDocument();+ });++ it("does not render the hidden indicator for normal tables", () => {+ setup({ visibilityType: null });+ expect(screen.getByText("Orders")).toBeInTheDocument();+ expect(queryIcon("eye_crossed_out")).not.toBeInTheDocument();+ });+});diff --git a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/flow-graph.ts b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/flow-graph.tsindex b4335b0860b..8c5f6db4709 100644--- a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/flow-graph.ts+++ b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/flow-graph.ts@@ -130,7 +130,7 @@ function getFlowGraphMemoKey(data: ErdResponse): string {`${f.id}:${f.name}:${f.display_name}:${f.database_type}:${f.base_type}:${f.effective_type ?? ""}:${f.semantic_type ?? ""}:${f.fk_target_field_id ?? ""}:${f.fk_target_table_id ?? ""}`,).join("|");- return `${node.table_id}:${node.name}:${node.display_name}:${node.description ?? ""}:${getOwnerKey(node.owner)}:${node.schema ?? ""}:${fieldKey}`;+ return `${node.table_id}:${node.name}:${node.display_name}:${node.description ?? ""}:${getOwnerKey(node.owner)}:${node.schema ?? ""}:${node.visibility_type ?? ""}:${fieldKey}`;}).sort().join(";");diff --git a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/test-utils.ts b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/test-utils.tsindex ff4c423d0a3..24fdfea5e15 100644--- a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/test-utils.ts+++ b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/test-utils.ts@@ -54,6 +54,7 @@ export function makeFlowNode({description: null,owner: null,schema: "public",+ visibility_type: null,db_id: 1,fields: resolvedFields.map((f, i) => ({id: i + 1,diff --git a/frontend/src/metabase-types/api/erd.ts b/frontend/src/metabase-types/api/erd.tsindex da95d950166..eaa2a61eda3 100644--- a/frontend/src/metabase-types/api/erd.ts+++ b/frontend/src/metabase-types/api/erd.ts@@ -5,6 +5,7 @@ import type {SchemaName,Table,TableId,+ TableVisibilityType,} from "./";export type ErdRelationship = "one-to-one" | "many-to-one";@@ -30,6 +31,7 @@ export type ErdNode = {description: string | null;owner: Table["owner"];schema: SchemaName | null;+ visibility_type: TableVisibilityType;db_id: DatabaseId;fields: ErdField[];};diff --git a/frontend/src/metabase-types/api/mocks/erd.ts b/frontend/src/metabase-types/api/mocks/erd.tsindex db50ffd13eb..dd298591a1d 100644--- a/frontend/src/metabase-types/api/mocks/erd.ts+++ b/frontend/src/metabase-types/api/mocks/erd.ts@@ -25,6 +25,7 @@ export const createMockErdNode = (opts?: Partial<ErdNode>): ErdNode => ({description: null,owner: null,schema: "public",+ visibility_type: null,db_id: 1,fields: [],...opts,diff --git a/frontend/src/metabase-types/api/table.ts b/frontend/src/metabase-types/api/table.tsindex 2ec22b9ea8b..b5b22b5f166 100644--- a/frontend/src/metabase-types/api/table.ts+++ b/frontend/src/metabase-types/api/table.ts@@ -21,9 +21,7 @@ export function isConcreteTableId(export type TableVisibilityType =| null- | "details-only"| "hidden"- | "normal"| "retired"| "sensitive"| "technical"diff --git a/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPicker/MiniPicker.tsx b/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPicker/MiniPicker.tsxindex 7e601c2ba0f..7fd6a600d60 100644--- a/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPicker/MiniPicker.tsx+++ b/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPicker/MiniPicker.tsx@@ -26,6 +26,7 @@ export type MiniPickerProps = {models: MiniPickerPickableItem["model"][];onBrowseAll?: () => void;shouldHide?: (item: MiniPickerItem | unknown) => boolean;+ includeHiddenSchemas?: boolean;shouldShowLibrary?: boolean;forceSearch?: boolean;showSearchInput?: boolean;@@ -53,6 +54,7 @@ export function MiniPicker({shouldShowLibrary = true,forceSearch = false,showSearchInput = false,+ includeHiddenSchemas = false,searchInputPlaceholder,searchParams,onSearchResults,@@ -111,6 +113,7 @@ export function MiniPicker({searchInputPlaceholder,searchParams,onSearchResults,+ includeHiddenSchemas,}}><Menudiff --git a/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPickerItemList.tsx b/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPickerItemList.tsxindex 0cfc11ec799..e72735f6b4d 100644--- a/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPickerItemList.tsx+++ b/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPickerItemList.tsx@@ -282,7 +282,8 @@ function DatabaseItemList({}: {parent: MiniPickerDatabaseItem | MiniPickerSchemaItem;}) {- const { setPath, onChange, isHidden, models } = useMiniPickerContext();+ const { setPath, onChange, isHidden, models, includeHiddenSchemas } =+ useMiniPickerContext();// Callers opt schemas in as a terminal pick by including "schema" in// `models`. Affects three things: schemas fire `onChange` instead of// `setPath`, single-schema DBs no longer auto-drill to tables, and the@@ -291,7 +292,11 @@ function DatabaseItemList({const { data: allSchemas, isLoading: isLoadingSchemas } =useListDatabaseSchemasQuery(parent.model === "database"- ? { id: parent.id, "can-query": true }+ ? {+ id: parent.id,+ "can-query": true,+ include_hidden: includeHiddenSchemas,+ }: skipToken,);diff --git a/frontend/src/metabase/common/components/Pickers/MiniPicker/context.ts b/frontend/src/metabase/common/components/Pickers/MiniPicker/context.tsindex feefa76257e..1b80350cd71 100644--- a/frontend/src/metabase/common/components/Pickers/MiniPicker/context.ts+++ b/frontend/src/metabase/common/components/Pickers/MiniPicker/context.ts@@ -33,6 +33,7 @@ export interface MiniPickerContextValue {shouldShowLibrary?: boolean;forceSearch?: boolean;showSearchInput?: boolean;+ includeHiddenSchemas?: boolean;searchInputPlaceholder?: string;searchParams?: MiniPickerSearchParams;onSearchResults?: (results: MiniPickerPickableItem[]) => void;diff --git a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/hooks.ts b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/hooks.tsindex 86a863c9319..4de0f84531a 100644--- a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/hooks.ts+++ b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/hooks.ts@@ -35,8 +35,6 @@ export function useHiddenSourceTables(return joinTablesInfo.filter((tableInfo) =>- !tableInfo.isSourceTable ||- (tableInfo.visibilityType !== null &&- tableInfo.visibilityType !== "normal"),+ !tableInfo.isSourceTable || tableInfo.visibilityType !== null,);}