Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GDGT-2519: Surface hidden tables in Schema Viewer ERD (#75031)

Stas Gavrylov586dfd96f8dc
diff --git a/enterprise/backend/src/metabase_enterprise/erd/impl.clj b/enterprise/backend/src/metabase_enterprise/erd/impl.clj
index dbde20d6c99..d355f475c23 100644
--- a/enterprise/backend/src/metabase_enterprise/erd/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/erd/impl.clj
@@ -48,6 +48,7 @@
[:description {:optional true} [:maybe :string]]
[:owner {:optional true} ::erd-owner]
[:schema {:optional true} [:maybe :string]]
+ [:visibility_type {:optional true} [:maybe :keyword]]
[:db_id :int]
[:fields [:sequential ::erd-field]]])
@@ -77,7 +78,7 @@
(`:db_id`, `:is_published`, `:collection_id`) plus the display columns the
ERD response surfaces (`:description`, owner fields hydrated below)."
[:id :db_id :name :display_name :schema :is_published :collection_id
- :description :owner_user_id :owner_email])
+ :description :owner_user_id :owner_email :visibility_type])
(defn- schema-clause
"The rest of the database API treats a blank schema name as the union of nil
@@ -93,7 +94,7 @@
(into {} (map (juxt :id identity)) xs))
(defn- readable-tables
- "Fetch readable, active, visible tables in `database-id`.
+ "Fetch readable, active tables in `database-id`.
Optional `table-ids` restricts by IDs; optional `schema` restricts by schema.
`schema=\"\"` matches both nil and empty-string schemas."
[database-id & {:keys [table-ids schema] :as opts}]
@@ -101,8 +102,7 @@
[]
(let [where (cond-> [:and
[:= :db_id database-id]
- [:= :active true]
- [:= :visibility_type nil]]
+ [:= :active true]]
(contains? opts :table-ids) (conj [:in :id table-ids])
(contains? opts :schema) (conj (schema-clause schema)))]
(->> (t2/select :model/Table
@@ -233,6 +233,7 @@
:description (:description table)
:owner (:owner table)
:schema (:schema table)
+ :visibility_type (:visibility_type table)
:db_id (:db_id table)
:fields (mapv #(build-erd-field % target-field-id->table-id) fields)})
diff --git a/enterprise/backend/test/metabase_enterprise/erd/erd_test.clj b/enterprise/backend/test/metabase_enterprise/erd/erd_test.clj
index 044f59b62ee..c0f9bc0f612 100644
--- a/enterprise/backend/test/metabase_enterprise/erd/erd_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/erd/erd_test.clj
@@ -274,9 +274,9 @@
(is (empty? (:nodes response)))
(is (empty? (:edges response))))))))
-(deftest erd-endpoint-excludes-hidden-and-sensitive-metadata-test
+(deftest erd-endpoint-includes-hidden-tables-and-all-field-visibility-test
(mt/with-premium-features #{:schema-viewer}
- (testing "hidden tables are omitted; all field visibility types are included"
+ (testing "all tables (incl. hidden/technical/cruft) are surfaced; all field visibility types are included"
(mt/with-temp [:model/Database {db-id :id} {}
:model/Table {visible-table-id :id} {:db_id db-id :name "visible" :schema "PUBLIC"}
:model/Field _ {:table_id visible-table-id :name "visible_id"
@@ -291,18 +291,28 @@
:database_type "INTEGER" :base_type :type/Integer
:visibility_type "retired"}
:model/Table _ {:db_id db-id :name "hidden_table" :schema "PUBLIC"
- :visibility_type "hidden"}]
- (let [response (mt/user-http-request :rasta :get 200 "ee/erd"
- :database-id db-id
- :schema "PUBLIC")
- node-names (set (map :name (:nodes response)))
- field-names (set (map :name (mapcat :fields (:nodes response))))]
- (is (= #{"visible"} node-names))
- (is (= #{"visible_id" "hidden_field" "secret" "retired_field"} field-names)))))))
+ :visibility_type "hidden"}
+ :model/Table _ {:db_id db-id :name "technical_table" :schema "PUBLIC"
+ :visibility_type "technical"}
+ :model/Table _ {:db_id db-id :name "cruft_table" :schema "PUBLIC"
+ :visibility_type "cruft"}]
+ (let [response (mt/user-http-request :rasta :get 200 "ee/erd"
+ :database-id db-id
+ :schema "PUBLIC")
+ node-names (set (map :name (:nodes response)))
+ field-names (set (map :name (mapcat :fields (:nodes response))))
+ visibility-by-name (into {} (map (juxt :name :visibility_type)) (:nodes response))]
+ (is (= #{"visible" "hidden_table" "technical_table" "cruft_table"} node-names))
+ (is (= #{"visible_id" "hidden_field" "secret" "retired_field"} field-names))
+ (is (= {"visible" nil
+ "hidden_table" "hidden"
+ "technical_table" "technical"
+ "cruft_table" "cruft"}
+ visibility-by-name)))))))
-(deftest erd-endpoint-does-not-leak-hidden-fk-targets-test
+(deftest erd-endpoint-resolves-hidden-fk-targets-test
(mt/with-premium-features #{:schema-viewer}
- (testing "FK targets in hidden tables are nulled and do not produce edges; FKs to sensitive fields resolve"
+ (testing "FK targets in hidden tables resolve into nodes and edges; FKs to sensitive fields resolve too"
(mt/with-temp [:model/Database {db-id :id} {}
:model/Table {hidden-target-table-id :id} {:db_id db-id :name "hidden_target" :schema "PUBLIC"
:visibility_type "hidden"}
@@ -331,12 +341,14 @@
fields-by-id (into {} (map (juxt :id identity)) (:fields source-node))
hidden-fk (get fields-by-id hidden-table-fk-id)
sensitive-fk (get fields-by-id sensitive-field-fk-id)]
- (is (not (contains? nodes-by-name "hidden_target")))
+ (is (contains? nodes-by-name "hidden_target"))
(is (contains? nodes-by-name "visible_target"))
- (testing "FK to a field in a hidden table is nulled and produces no edge"
- (is (nil? (:fk_target_table_id hidden-fk)))
- (is (nil? (:fk_target_field_id hidden-fk)))
- (is (not-any? #(= hidden-table-fk-id (:source_field_id %)) (:edges response))))
+ (testing "FK to a field in a hidden table resolves and produces an edge"
+ (is (= hidden-target-table-id (:fk_target_table_id hidden-fk)))
+ (is (= hidden-target-field-id (:fk_target_field_id hidden-fk)))
+ (is (some #(and (= hidden-table-fk-id (:source_field_id %))
+ (= hidden-target-field-id (:target_field_id %)))
+ (:edges response))))
(testing "FK to a sensitive field in a visible table resolves normally"
(is (= visible-target-table-id (:fk_target_table_id sensitive-fk)))
(is (= sensitive-target-field-id (:fk_target_field_id sensitive-fk)))
diff --git a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/SchemaPickerInput/SchemaPickerInput.tsx b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/SchemaPickerInput/SchemaPickerInput.tsx
index 58307084756..cae8ca9ed59 100644
--- a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/SchemaPickerInput/SchemaPickerInput.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/SchemaPickerInput/SchemaPickerInput.tsx
@@ -52,6 +52,7 @@ export function SchemaPickerInput({
opened={opened}
onClose={() => setOpened(false)}
models={["schema"]}
+ includeHiddenSchemas
onChange={handleChange}
menuProps={{
// This allows to toggle menu by clicking the trigger button.
diff --git a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.tsx b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.tsx
index 1c7b786f51f..68e47ffbae0 100644
--- a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.tsx
@@ -1,8 +1,9 @@
import type { NodeProps } from "@xyflow/react";
import cx from "classnames";
import { memo, useCallback } from "react";
+import { t } from "ttag";
-import { Box, FixedSizeIcon, Group, Stack } from "metabase/ui";
+import { Box, FixedSizeIcon, Group, Stack, Tooltip } from "metabase/ui";
import { useSchemaViewerContext } from "../../SchemaViewerContext";
import type { SchemaViewerFlowNode } from "../../types";
@@ -22,6 +23,8 @@ export const SchemaViewerTableNode = memo(function SchemaViewerTableNode({
const isConnectedToSelectedEdge = data.selectedFieldIds.size > 0;
const isUserSelected = selectedNodeId === id;
+ const hasHiddenVisibilityType = data.visibility_type !== null;
+
const handleDoubleClick = useCallback(() => {
zoomToNode(id);
}, [id, zoomToNode]);
@@ -66,6 +69,11 @@ export const SchemaViewerTableNode = memo(function SchemaViewerTableNode({
>
{data.name}
</Box>
+ {hasHiddenVisibilityType && (
+ <Tooltip label={t`This table is hidden`}>
+ <FixedSizeIcon name="eye_crossed_out" c="text-secondary" />
+ </Tooltip>
+ )}
</Group>
<Box className={S.fields}>
{data.fields.map((field) => (
diff --git a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.unit.spec.tsx
new file mode 100644
index 00000000000..6fa07284d08
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/components/TableNode/SchemaViewerTableNode.unit.spec.tsx
@@ -0,0 +1,76 @@
+import type { NodeProps } from "@xyflow/react";
+import { ReactFlowProvider } from "@xyflow/react";
+
+import { queryIcon, renderWithProviders, screen } from "__support__/ui";
+import type { ConcreteTableId, TableVisibilityType } from "metabase-types/api";
+import {
+ createMockErdNode,
+ createMockErdResponse,
+} from "metabase-types/api/mocks";
+
+import { SchemaViewerContext } from "../../SchemaViewerContext";
+import type { SchemaViewerFlowNode } from "../../types";
+import { toFlowGraph } from "../../utils/flow-graph";
+
+import { SchemaViewerTableNode } from "./SchemaViewerTableNode";
+
+function nodeProps(
+ node: SchemaViewerFlowNode,
+): NodeProps<SchemaViewerFlowNode> {
+ return {
+ id: node.id,
+ data: node.data,
+ type: "schemaViewerTable",
+ dragging: false,
+ zIndex: 2,
+ selectable: true,
+ deletable: false,
+ selected: false,
+ draggable: true,
+ isConnectable: false,
+ positionAbsoluteX: 0,
+ positionAbsoluteY: 0,
+ };
+}
+
+function setup({
+ visibilityType = null,
+}: { visibilityType?: TableVisibilityType } = {}) {
+ const erdNode = createMockErdNode({
+ table_id: 1,
+ name: "Orders",
+ visibility_type: visibilityType,
+ });
+ const [node] = toFlowGraph(createMockErdResponse({ nodes: [erdNode] })).nodes;
+
+ const value = {
+ visibleTableIds: new Set<ConcreteTableId>(),
+ expandingTableIds: new Set<ConcreteTableId>(),
+ expandToTable: jest.fn(),
+ selectedNodeId: null,
+ selectNode: jest.fn(),
+ zoomToNode: jest.fn(),
+ };
+
+ renderWithProviders(
+ <ReactFlowProvider>
+ <SchemaViewerContext.Provider value={value}>
+ <SchemaViewerTableNode {...nodeProps(node)} />
+ </SchemaViewerContext.Provider>
+ </ReactFlowProvider>,
+ );
+}
+
+describe("SchemaViewerTableNode", () => {
+ it("renders the hidden indicator for tables with a visibility_type", () => {
+ setup({ visibilityType: "hidden" });
+ expect(screen.getByText("Orders")).toBeInTheDocument();
+ expect(queryIcon("eye_crossed_out")).toBeInTheDocument();
+ });
+
+ it("does not render the hidden indicator for normal tables", () => {
+ setup({ visibilityType: null });
+ expect(screen.getByText("Orders")).toBeInTheDocument();
+ expect(queryIcon("eye_crossed_out")).not.toBeInTheDocument();
+ });
+});
diff --git a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/flow-graph.ts b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/flow-graph.ts
index b4335b0860b..8c5f6db4709 100644
--- a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/flow-graph.ts
+++ b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/flow-graph.ts
@@ -130,7 +130,7 @@ function getFlowGraphMemoKey(data: ErdResponse): string {
`${f.id}:${f.name}:${f.display_name}:${f.database_type}:${f.base_type}:${f.effective_type ?? ""}:${f.semantic_type ?? ""}:${f.fk_target_field_id ?? ""}:${f.fk_target_table_id ?? ""}`,
)
.join("|");
- return `${node.table_id}:${node.name}:${node.display_name}:${node.description ?? ""}:${getOwnerKey(node.owner)}:${node.schema ?? ""}:${fieldKey}`;
+ return `${node.table_id}:${node.name}:${node.display_name}:${node.description ?? ""}:${getOwnerKey(node.owner)}:${node.schema ?? ""}:${node.visibility_type ?? ""}:${fieldKey}`;
})
.sort()
.join(";");
diff --git a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/test-utils.ts b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/test-utils.ts
index ff4c423d0a3..24fdfea5e15 100644
--- a/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/test-utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/schema_viewer/components/SchemaViewer/utils/test-utils.ts
@@ -54,6 +54,7 @@ export function makeFlowNode({
description: null,
owner: null,
schema: "public",
+ visibility_type: null,
db_id: 1,
fields: resolvedFields.map((f, i) => ({
id: i + 1,
diff --git a/frontend/src/metabase-types/api/erd.ts b/frontend/src/metabase-types/api/erd.ts
index da95d950166..eaa2a61eda3 100644
--- a/frontend/src/metabase-types/api/erd.ts
+++ b/frontend/src/metabase-types/api/erd.ts
@@ -5,6 +5,7 @@ import type {
SchemaName,
Table,
TableId,
+ TableVisibilityType,
} from "./";
export type ErdRelationship = "one-to-one" | "many-to-one";
@@ -30,6 +31,7 @@ export type ErdNode = {
description: string | null;
owner: Table["owner"];
schema: SchemaName | null;
+ visibility_type: TableVisibilityType;
db_id: DatabaseId;
fields: ErdField[];
};
diff --git a/frontend/src/metabase-types/api/mocks/erd.ts b/frontend/src/metabase-types/api/mocks/erd.ts
index db50ffd13eb..dd298591a1d 100644
--- a/frontend/src/metabase-types/api/mocks/erd.ts
+++ b/frontend/src/metabase-types/api/mocks/erd.ts
@@ -25,6 +25,7 @@ export const createMockErdNode = (opts?: Partial<ErdNode>): ErdNode => ({
description: null,
owner: null,
schema: "public",
+ visibility_type: null,
db_id: 1,
fields: [],
...opts,
diff --git a/frontend/src/metabase-types/api/table.ts b/frontend/src/metabase-types/api/table.ts
index 2ec22b9ea8b..b5b22b5f166 100644
--- a/frontend/src/metabase-types/api/table.ts
+++ b/frontend/src/metabase-types/api/table.ts
@@ -21,9 +21,7 @@ export function isConcreteTableId(
export type TableVisibilityType =
| null
- | "details-only"
| "hidden"
- | "normal"
| "retired"
| "sensitive"
| "technical"
diff --git a/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPicker/MiniPicker.tsx b/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPicker/MiniPicker.tsx
index 7e601c2ba0f..7fd6a600d60 100644
--- a/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPicker/MiniPicker.tsx
+++ b/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPicker/MiniPicker.tsx
@@ -26,6 +26,7 @@ export type MiniPickerProps = {
models: MiniPickerPickableItem["model"][];
onBrowseAll?: () => void;
shouldHide?: (item: MiniPickerItem | unknown) => boolean;
+ includeHiddenSchemas?: boolean;
shouldShowLibrary?: boolean;
forceSearch?: boolean;
showSearchInput?: boolean;
@@ -53,6 +54,7 @@ export function MiniPicker({
shouldShowLibrary = true,
forceSearch = false,
showSearchInput = false,
+ includeHiddenSchemas = false,
searchInputPlaceholder,
searchParams,
onSearchResults,
@@ -111,6 +113,7 @@ export function MiniPicker({
searchInputPlaceholder,
searchParams,
onSearchResults,
+ includeHiddenSchemas,
}}
>
<Menu
diff --git a/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPickerItemList.tsx b/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPickerItemList.tsx
index 0cfc11ec799..e72735f6b4d 100644
--- a/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPickerItemList.tsx
+++ b/frontend/src/metabase/common/components/Pickers/MiniPicker/components/MiniPickerItemList.tsx
@@ -282,7 +282,8 @@ function DatabaseItemList({
}: {
parent: MiniPickerDatabaseItem | MiniPickerSchemaItem;
}) {
- const { setPath, onChange, isHidden, models } = useMiniPickerContext();
+ const { setPath, onChange, isHidden, models, includeHiddenSchemas } =
+ useMiniPickerContext();
// Callers opt schemas in as a terminal pick by including "schema" in
// `models`. Affects three things: schemas fire `onChange` instead of
// `setPath`, single-schema DBs no longer auto-drill to tables, and the
@@ -291,7 +292,11 @@ function DatabaseItemList({
const { data: allSchemas, isLoading: isLoadingSchemas } =
useListDatabaseSchemasQuery(
parent.model === "database"
- ? { id: parent.id, "can-query": true }
+ ? {
+ id: parent.id,
+ "can-query": true,
+ include_hidden: includeHiddenSchemas,
+ }
: skipToken,
);
diff --git a/frontend/src/metabase/common/components/Pickers/MiniPicker/context.ts b/frontend/src/metabase/common/components/Pickers/MiniPicker/context.ts
index feefa76257e..1b80350cd71 100644
--- a/frontend/src/metabase/common/components/Pickers/MiniPicker/context.ts
+++ b/frontend/src/metabase/common/components/Pickers/MiniPicker/context.ts
@@ -33,6 +33,7 @@ export interface MiniPickerContextValue {
shouldShowLibrary?: boolean;
forceSearch?: boolean;
showSearchInput?: boolean;
+ includeHiddenSchemas?: boolean;
searchInputPlaceholder?: string;
searchParams?: MiniPickerSearchParams;
onSearchResults?: (results: MiniPickerPickableItem[]) => void;
diff --git a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/hooks.ts b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/hooks.ts
index 86a863c9319..4de0f84531a 100644
--- a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/hooks.ts
+++ b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/hooks.ts
@@ -35,8 +35,6 @@ export function useHiddenSourceTables(
return joinTablesInfo.filter(
(tableInfo) =>
- !tableInfo.isSourceTable ||
- (tableInfo.visibilityType !== null &&
- tableInfo.visibilityType !== "normal"),
+ !tableInfo.isSourceTable || tableInfo.visibilityType !== null,
);
}