The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
feat: change data layer medallion inspired values to hidden, internal and published (#66330)
Jakub Chodorowiczf7801bbc47f7
diff --git a/e2e/support/helpers/e2e-datamodel-helpers.ts b/e2e/support/helpers/e2e-datamodel-helpers.tsindex 736004e3c01..2140bf8119d 100644--- a/e2e/support/helpers/e2e-datamodel-helpers.ts+++ b/e2e/support/helpers/e2e-datamodel-helpers.ts@@ -354,7 +354,7 @@ function getTableSectionSortableFields() {}function getTableSectionVisibilityTypeInput() {- return getTableSection().findByRole("textbox", { name: "Visibility type" });+ return getTableSection().findByRole("textbox", { name: "Visibility layer" });}function getTableSectionFieldNameInput(name: string) {diff --git a/e2e/test/scenarios/data-studio/data-model/data-studio-bulk-table.cy.spec.ts b/e2e/test/scenarios/data-studio/data-model/data-studio-bulk-table.cy.spec.tsindex 064b1d63102..0218036f3c4 100644--- a/e2e/test/scenarios/data-studio/data-model/data-studio-bulk-table.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/data-model/data-studio-bulk-table.cy.spec.ts@@ -180,8 +180,8 @@ describe("bulk table operations", () => {result: "success",});- H.selectHasValue("Visibility type", "").click();- H.selectDropdown().contains("Gold").click();+ H.selectHasValue("Visibility layer", "").click();+ H.selectDropdown().contains("Final").click();H.expectUnstructuredSnowplowEvent({event: "data_studio_bulk_attribute_updated",event_detail: "layer",@@ -304,8 +304,8 @@ describe("bulk table operations", () => {H.selectHasValue("Owner", "").click();H.selectDropdown().contains("Bobby Tables").click();- H.selectHasValue("Visibility type", "").click();- H.selectDropdown().contains("Gold").click();+ H.selectHasValue("Visibility layer", "").click();+ H.selectDropdown().contains("Final").click();H.selectHasValue("Entity type", "").click();H.selectDropdown().contains("Person").click();@@ -348,8 +348,8 @@ describe("bulk table operations", () => {H.selectHasValue("Owner", "").click();H.selectDropdown().contains("Bobby Tables").click();- H.selectHasValue("Visibility type", "").click();- H.selectDropdown().contains("Gold").click();+ H.selectHasValue("Visibility layer", "").click();+ H.selectDropdown().contains("Final").click();H.selectHasValue("Entity type", "").click();H.selectDropdown().contains("Person").click();diff --git a/e2e/test/scenarios/data-studio/data-model/data-studio-single-table.cy.spec.ts b/e2e/test/scenarios/data-studio/data-model/data-studio-single-table.cy.spec.tsindex 487f6c5be67..d8f42c82560 100644--- a/e2e/test/scenarios/data-studio/data-model/data-studio-single-table.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/data-model/data-studio-single-table.cy.spec.ts@@ -111,10 +111,10 @@ describe("Table editing", () => {.findByText("Table owner updated").should("be.visible");- H.selectHasValue("Visibility type", "Bronze").click();- H.selectDropdown().contains("Gold").click();+ H.selectHasValue("Visibility layer", "Internal").click();+ H.selectDropdown().contains("Final").click();H.undoToastListContainer()- .findByText("Table visibility type updated")+ .findByText("Table visibility layer updated").should("be.visible");H.selectHasValue("Entity type", "Transaction").click();@@ -134,7 +134,7 @@ describe("Table editing", () => {TablePicker.getTable("Orders").click();H.selectHasValue("Owner", "Bobby Tables");- H.selectHasValue("Visibility type", "Gold");+ H.selectHasValue("Visibility layer", "Final");H.selectHasValue("Entity type", "Person");H.selectHasValue("Source", "Ingested");});diff --git a/e2e/test/scenarios/data-studio/data-model/datamodel-data-studio-search.cy.spec.ts b/e2e/test/scenarios/data-studio/data-model/datamodel-data-studio-search.cy.spec.tsindex 0ec5e5f8d76..f6face95c66 100644--- a/e2e/test/scenarios/data-studio/data-model/datamodel-data-studio-search.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/data-model/datamodel-data-studio-search.cy.spec.ts@@ -188,7 +188,7 @@ describe("Search", () => {TablePicker.openFilterPopover();cy.findByTestId("table-picker-filter").within(() => {- TablePicker.selectFilterOption("Visibility type", "Bronze");+ TablePicker.selectFilterOption("Visibility layer", "Internal");TablePicker.applyFilters();});@@ -202,7 +202,7 @@ describe("Search", () => {"be.visible",);- TablePicker.selectFilterOption("Visibility type", "Gold");+ TablePicker.selectFilterOption("Visibility layer", "Final");TablePicker.get().findByText("No tables found").should("be.visible");cy.findByRole("heading", { name: /2 tables selected/i }).should("not.exist",diff --git a/e2e/test/scenarios/data-studio/data-model/datamodel-data-studio.cy.spec.ts b/e2e/test/scenarios/data-studio/data-model/datamodel-data-studio.cy.spec.tsindex 27e2b78bc23..d38bd4c4cae 100644--- a/e2e/test/scenarios/data-studio/data-model/datamodel-data-studio.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/data-model/datamodel-data-studio.cy.spec.ts@@ -547,18 +547,18 @@ describe("scenarios > data studio > datamodel", () => {});describe("Filtering", () => {- it("should filter tables by visibility type", () => {+ it("should filter tables by visibility layer", () => {updateTableAttributes({databaseId: SAMPLE_DB_ID,displayName: "Orders",- attributes: { data_layer: "gold" },- }).as("goldTableId");+ attributes: { data_layer: "final" },+ }).as("finalTableId");updateTableAttributes({databaseId: SAMPLE_DB_ID,displayName: "Products",- attributes: { data_layer: "silver" },- }).as("silverTableId");+ attributes: { data_layer: "hidden" },+ }).as("hiddenTableId");H.DataModel.visitDataStudio();@@ -569,7 +569,7 @@ describe("scenarios > data studio > datamodel", () => {H.DataModel.TablePicker.getFilterForm().should("not.exist");TablePicker.openFilterPopover();- TablePicker.selectFilterOption("Visibility type", "Gold");+ TablePicker.selectFilterOption("Visibility layer", "Final");TablePicker.applyFilters();H.expectUnstructuredSnowplowEvent({event: "data_studio_table_picker_filters_applied",@@ -578,8 +578,8 @@ describe("scenarios > data studio > datamodel", () => {event: "data_studio_table_picker_search_performed",});- cy.get<TableId>("@goldTableId").then(expectTableVisible);- cy.get<TableId>("@silverTableId").then(expectTableNotVisible);+ cy.get<TableId>("@finalTableId").then(expectTableVisible);+ cy.get<TableId>("@hiddenTableId").then(expectTableNotVisible);});it("should filter tables owned by unspecified", () => {@@ -3524,7 +3524,7 @@ describe("scenarios > data studio > datamodel", () => {});H.DataModel.TableSection.getVisibilityTypeInput().click();- H.popover().findByText("Copper").click();+ H.popover().findByText("Hidden").click();cy.wait("@updateTable");FieldSection.getPreviewButton().click();diff --git a/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj b/enterprise/backend/src/metabase_enterprise/data_studio/api/table.cljindex e7eb1ca3a5a..46ec1f61285 100644--- a/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj+++ b/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj@@ -170,9 +170,9 @@(defn- maybe-sync-unhidden-tables![existing-tables {:keys [data_layer] :as body}]- ;; sync any tables that are changed from copper to something else- (sync-unhidden-tables (when (and (contains? body :data_layer) (not= :copper data_layer))- (filter #(= :copper (:data_layer %)) existing-tables))))+ ;; sync any tables that are changed from hidden to something else+ (sync-unhidden-tables (when (and (contains? body :data_layer) (not= :hidden data_layer))+ (filter #(= :hidden (:data_layer %)) existing-tables))))(api.macros/defendpoint :post "/edit" :- [:map {:closed true}]"Bulk updating tables."diff --git a/enterprise/backend/test/metabase_enterprise/data_studio/api/table_test.clj b/enterprise/backend/test/metabase_enterprise/data_studio/api/table_test.cljindex 5a37ea4c51b..ce7356b7c65 100644--- a/enterprise/backend/test/metabase_enterprise/data_studio/api/table_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_studio/api/table_test.clj@@ -78,35 +78,35 @@:model/Table {table-2-id :id} {:db_id db-id}](testing "updating data_layer syncs to visibility_type for all tables"- ;; Update two tables to gold, which should sync to nil visibility_type+ ;; Update two tables to internal, which should sync to nil visibility_type(mt/user-http-request :crowberto :post 200 "ee/data-studio/table/edit"{:table_ids [table-1-id table-2-id]- :data_layer "gold"})- (is (= :gold (t2/select-one-fn :data_layer :model/Table :id table-1-id)))+ :data_layer "internal"})+ (is (= :internal (t2/select-one-fn :data_layer :model/Table :id table-1-id)))(is (= nil (t2/select-one-fn :visibility_type :model/Table :id table-1-id)))- (is (= :gold (t2/select-one-fn :data_layer :model/Table :id table-2-id)))+ (is (= :internal (t2/select-one-fn :data_layer :model/Table :id table-2-id)))(is (= nil (t2/select-one-fn :visibility_type :model/Table :id table-2-id))))- (testing "updating data_layer to copper syncs to hidden visibility_type"- ;; Update one table back to copper, which should sync to :hidden+ (testing "updating data_layer to hidden syncs to hidden visibility_type"+ ;; Update one table back to hidden, which should sync to :hidden(mt/user-http-request :crowberto :post 200 "ee/data-studio/table/edit"{:table_ids [table-1-id]- :data_layer "copper"})- (is (= :copper (t2/select-one-fn :data_layer :model/Table :id table-1-id)))+ :data_layer "hidden"})+ (is (= :hidden (t2/select-one-fn :data_layer :model/Table :id table-1-id)))(is (= :hidden (t2/select-one-fn :visibility_type :model/Table :id table-1-id))))(testing "cannot update both visibility_type and data_layer at once"(mt/user-http-request :crowberto :post 400 "ee/data-studio/table/edit"{:table_ids [table-1-id]:visibility_type "hidden"- :data_layer "copper"}))))))+ :data_layer "hidden"}))))))(deftest requests-data-studio-feature-flag-test(mt/with-premium-features #{}(is (= "Data Studio is a paid feature not currently available to your instance. Please upgrade to use it. Learn more at metabase.com/upgrade/"(:message (mt/user-http-request :crowberto :post 402 "ee/data-studio/table/edit"{:table_ids [(mt/id :users)]- :data_layer "gold"}))))))+ :data_layer "final"}))))))(deftest data-analyst-can-access-endpoints-test(mt/with-premium-features #{:data-studio}@@ -123,7 +123,7 @@(testing "data analyst can edit tables"(is (= {} (mt/user-http-request analyst-id :post 200 "ee/data-studio/table/edit"{:table_ids [table-id]- :data_layer "gold"}))))+ :data_layer "final"}))))(testing "data analyst can get selection info"(is (map? (mt/user-http-request analyst-id :post 200 "ee/data-studio/table/selection"{:table_ids [table-id]}))))@@ -147,7 +147,7 @@(is (= "You don't have permissions to do that."(mt/user-http-request user-id :post 403 "ee/data-studio/table/edit"{:table_ids [table-id]- :data_layer "gold"}))))+ :data_layer "final"}))))(testing "regular user cannot get selection info"(is (= "You don't have permissions to do that."(mt/user-http-request user-id :post 403 "ee/data-studio/table/selection"@@ -315,51 +315,43 @@(testing "empty selectors returns no tables"(is (= nil (selectors->table-ids {}))))))))-(deftest trigger-sync-on-data-layer-change-from-copper-test+(deftest trigger-sync-on-data-layer-change-from-hidden-test(mt/with-premium-features #{:data-studio}- (testing "Changing data_layer from copper to another value triggers sync"+ (testing "Changing data_layer from hidden to another value triggers sync"(mt/with-temp [:model/Database {db-id :id} {}- :model/Table {copper-1 :id} {:db_id db-id, :data_layer :copper}- :model/Table {copper-2 :id} {:db_id db-id, :data_layer :copper}- :model/Table {copper-3 :id} {:db_id db-id, :data_layer :copper}- :model/Table {copper-4 :id} {:db_id db-id, :data_layer :copper}- :model/Table {gold-1 :id} {:db_id db-id, :data_layer :gold}- :model/Table {gold-2 :id} {:db_id db-id, :data_layer :gold}]+ :model/Table {hidden-1 :id} {:db_id db-id, :data_layer :hidden}+ :model/Table {hidden-2 :id} {:db_id db-id, :data_layer :hidden}+ :model/Table {hidden-3 :id} {:db_id db-id, :data_layer :hidden}+ :model/Table {internal-1 :id} {:db_id db-id, :data_layer :internal}+ :model/Table {final-1 :id} {:db_id db-id, :data_layer :final}](let [synced-ids (atom #{})](mt/with-dynamic-fn-redefs [api.table/sync-unhidden-tables (fn [tables] (reset! synced-ids (set (map :id tables))))]- (testing "Changing from copper to gold triggers sync"+ (testing "Changing from hidden to final triggers sync"(reset! synced-ids #{})(mt/user-http-request :crowberto :post 200 "ee/data-studio/table/edit"- {:table_ids [copper-1 copper-2]- :data_layer "gold"})- (is (= #{copper-1 copper-2} @synced-ids)))+ {:table_ids [hidden-1 hidden-2]+ :data_layer "final"})+ (is (= #{hidden-1 hidden-2} @synced-ids)))- (testing "Changing from copper to silver triggers sync"+ (testing "Changing from hidden to internal triggers sync"(reset! synced-ids #{})(mt/user-http-request :crowberto :post 200 "ee/data-studio/table/edit"- {:table_ids [copper-3]- :data_layer "silver"})- (is (= #{copper-3} @synced-ids)))+ {:table_ids [hidden-3]+ :data_layer "internal"})+ (is (= #{hidden-3} @synced-ids)))- (testing "Changing from copper to bronze triggers sync"+ (testing "Not changing from hidden does not trigger sync"(reset! synced-ids #{})(mt/user-http-request :crowberto :post 200 "ee/data-studio/table/edit"- {:table_ids [copper-4]- :data_layer "bronze"})- (is (= #{copper-4} @synced-ids)))-- (testing "Not changing from copper does not trigger sync"- (reset! synced-ids #{})- (mt/user-http-request :crowberto :post 200 "ee/data-studio/table/edit"- {:table_ids [gold-1]- :data_layer "silver"})+ {:table_ids [internal-1]+ :data_layer "final"})(is (= #{} @synced-ids)))- (testing "Changing to copper does not trigger sync"+ (testing "Changing to hidden does not trigger sync"(reset! synced-ids #{})(mt/user-http-request :crowberto :post 200 "ee/data-studio/table/edit"- {:table_ids [gold-2]- :data_layer "copper"})+ {:table_ids [final-1]+ :data_layer "hidden"})(is (= #{} @synced-ids)))))))))(deftest bulk-edit-test@@ -378,15 +370,15 @@(is (= "You don't have permissions to do that."(mt/user-http-request :rasta :post 403 "ee/data-studio/table/edit"{:database_ids [clojure jvm]- :data_layer "copper"}))))+ :data_layer "hidden"}))))(testing "simple happy path updating with db ids"(mt/user-http-request :crowberto :post 200 "ee/data-studio/table/edit"{:database_ids [clojure jvm]- :data_layer "copper"+ :data_layer "hidden":data_authority "authoritative":data_source "ingested"})- (is (= #{:copper} (t2/select-fn-set :data_layer :model/Table :db_id [:in [clojure jvm]])))+ (is (= #{:hidden} (t2/select-fn-set :data_layer :model/Table :db_id [:in [clojure jvm]])))(is (= #{:authoritative} (t2/select-fn-set :data_authority :model/Table :db_id [:in [clojure jvm]])))(is (= #{:ingested} (t2/select-fn-set :data_source :model/Table :db_id [:in [clojure jvm]]))))@@ -395,13 +387,13 @@{:database_ids [clojure]:table_ids [classes]:schema_ids [(format "%d:jre" jvm)]- :data_layer "silver"})- (is (= {vars :silver- namespaces :silver- beans :copper- classes :silver- gc :silver- jit :silver}+ :data_layer "internal"})+ (is (= {vars :internal+ namespaces :internal+ beans :hidden+ classes :internal+ gc :internal+ jit :internal}(t2/select-pk->fn :data_layer :model/Table :db_id [:in [clojure jvm]]))))(testing "can update owner_email"diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/components/TableSection/TableAttributesEditSingle.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/components/TableSection/TableAttributesEditSingle.tsxindex 3d1ecfc52b3..2ebdd8d7d69 100644--- a/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/components/TableSection/TableAttributesEditSingle.tsx+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/data-model/components/TableSection/TableAttributesEditSingle.tsx@@ -87,9 +87,9 @@ export function TableAttributesEditSingle({ table }: Props) {});if (error) {- sendErrorToast(t`Failed to update table visibility type`);+ sendErrorToast(t`Failed to update table visibility layer`);} else {- sendSuccessToast(t`Table visibility type updated`, async () => {+ sendSuccessToast(t`Table visibility layer updated`, async () => {const { error } = await updateTable({id: table.id,data_layer: table.data_layer,@@ -163,7 +163,7 @@ export function TableAttributesEditSingle({ table }: Props) {/><LayerInput- value={table.data_layer ?? "copper"}+ value={table.data_layer ?? "hidden"}onChange={handleLayerChange}classNames={{ input: S.input, label: S.label }}className={S.gridLabelInput}diff --git a/frontend/src/metabase-types/api/table.ts b/frontend/src/metabase-types/api/table.tsindex 06ee85f7660..1724bd43fd9 100644--- a/frontend/src/metabase-types/api/table.ts+++ b/frontend/src/metabase-types/api/table.ts@@ -29,7 +29,7 @@ export type TableVisibilityType =| "technical"| "cruft";-export type TableDataLayer = "gold" | "silver" | "bronze" | "copper";+export type TableDataLayer = "hidden" | "internal" | "final";export type TableDataSource =| "ingested"diff --git a/frontend/src/metabase/metadata/components/LayerInput.tsx b/frontend/src/metabase/metadata/components/LayerInput.tsxindex 01807cac494..21c6ac9a4eb 100644--- a/frontend/src/metabase/metadata/components/LayerInput.tsx+++ b/frontend/src/metabase/metadata/components/LayerInput.tsx@@ -1,7 +1,6 @@import type { FocusEvent } from "react";import { t } from "ttag";-import type { ColorName } from "metabase/lib/colors/types";import { Group, Icon, Select, SelectItem, type SelectProps } from "metabase/ui";import type { TableDataLayer } from "metabase-types/api";@@ -10,7 +9,7 @@ interface Props extends Omit<SelectProps, "data" | "value" | "onChange"> {onChange: (value: TableDataLayer | null) => void;}-const dataLayers = ["copper", "bronze", "silver", "gold"] as const;+const dataLayers = ["hidden", "internal", "final"] as const;export const LayerInput = ({comboboxProps,@@ -37,28 +36,25 @@ export const LayerInput = ({...comboboxProps,}}data={[- { value: "copper" as const, label: t`Copper` },- { value: "bronze" as const, label: t`Bronze` },- { value: "silver" as const, label: t`Silver` },- { value: "gold" as const, label: t`Gold` },+ { value: "hidden" as const, label: t`Hidden` },+ { value: "internal" as const, label: t`Internal` },+ { value: "final" as const, label: t`Final` },]}- label={t`Visibility type`}+ label={t`Visibility layer`}renderOption={(item) => {const selected = item.option.value === value;return (<SelectItem selected={selected}><Group align="center" gap="sm" justify="center">- <Icon c={getColor(item.option.value)} name="medallion" />+ <VisibilityIcon value={item.option.value} /><span>{item.option.label}</span></Group></SelectItem>);}}- leftSection={- value ? <Icon c={getColor(value)} name="medallion" /> : undefined- }- placeholder={t`Select a visibility type`}+ leftSection={value ? <VisibilityIcon value={value} /> : undefined}+ placeholder={t`Select visibility layer`}value={value}onChange={(value) => onChange(value)}onFocus={handleFocus}@@ -71,9 +67,20 @@ function isDataLayer(value: string): value is TableDataLayer {return dataLayers.some((layer) => layer === value);}-function getColor(value: TableDataLayer | string): ColorName {+function VisibilityIcon({ value }: { value: string | null }): React.ReactNode {+ if (value == null) {+ return null;+ }+if (isDataLayer(value)) {- return value;+ return <Icon name={VISIBILITY_ICONS[value]} />;}- return "copper";++ return null;}++const VISIBILITY_ICONS = {+ hidden: "eye_filled",+ internal: "database",+ final: "published",+} as const;diff --git a/frontend/src/metabase/ui/components/icons/Icon/icons/index.ts b/frontend/src/metabase/ui/components/icons/Icon/icons/index.tsindex 8668b6cc390..34d073af9d9 100644--- a/frontend/src/metabase/ui/components/icons/Icon/icons/index.ts+++ b/frontend/src/metabase/ui/components/icons/Icon/icons/index.ts@@ -370,6 +370,8 @@ import progress_component from "./progress.svg?component";import progress_source from "./progress.svg?source";import publish_component from "./publish.svg?component";import publish_source from "./publish.svg?source";+import published_component from "./published.svg?component";+import published_source from "./published.svg?source";import pulse_component from "./pulse.svg?component";import pulse_source from "./pulse.svg?source";import question_component from "./question.svg?component";@@ -1267,6 +1269,10 @@ export const Icons: Record<IconName, { component: React.VFC; source: string }> =component: publish_component,source: publish_source,},+ published: {+ component: published_component,+ source: published_source,+ },pulse: {component: pulse_component,source: pulse_source,@@ -1846,6 +1852,7 @@ export type IconName =| "popular"| "progress"| "publish"+ | "published"| "pulse"| "receipt"| "recents"diff --git a/frontend/src/metabase/ui/components/icons/Icon/icons/published.svg b/frontend/src/metabase/ui/components/icons/Icon/icons/published.svgnew file mode 100644index 00000000000..a9ec8ee85d4--- /dev/null+++ b/frontend/src/metabase/ui/components/icons/Icon/icons/published.svg@@ -0,0 +1,3 @@+<svg viewBox="0 0 16 16" fill="currentcolor" xmlns="http://www.w3.org/2000/svg">+ <path d="M12.293 1.25a1.229 1.229 0 0 1 1.227 1.229v11.044a1.227 1.227 0 0 1-1.227 1.227h-7.67a2.149 2.149 0 0 1-2.137-1.935l-.01-.213V3.398A2.148 2.148 0 0 1 4.622 1.25h7.67zm-7.67 10.433a.92.92 0 0 0-.92.92l.004.09a.923.923 0 0 0 .916.83h7.67v-1.84h-7.67zm0-9.204a.92.92 0 0 0-.92.92v7.265c.285-.135.598-.209.92-.209h7.67V2.48h-7.67zm4.78 2.326a.615.615 0 0 1 .869.868L7.818 8.127a.614.614 0 0 1-.868 0L5.722 6.9a.614.614 0 0 1 .868-.867l.793.792 2.02-2.02z" />+</svg>diff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_alerts/v_alerts.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_alerts/v_alerts.yamlindex 7b8264662af..9acba7eef16 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_alerts/v_alerts.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_alerts/v_alerts.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_audit_log/v_audit_log.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_audit_log/v_audit_log.yamlindex 4d7c223ebae..ed80f46f232 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_audit_log/v_audit_log.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_audit_log/v_audit_log.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_content/v_content.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_content/v_content.yamlindex d802222f7ab..805e6f83c68 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_content/v_content.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_content/v_content.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_dashboardcard/v_dashboardcard.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_dashboardcard/v_dashboardcard.yamlindex 9fb3a408cc8..dfc469e74f0 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_dashboardcard/v_dashboardcard.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_dashboardcard/v_dashboardcard.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_databases/v_databases.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_databases/v_databases.yamlindex dcb9f59e59a..5d9eab3c239 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_databases/v_databases.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_databases/v_databases.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_fields/v_fields.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_fields/v_fields.yamlindex deaf947c468..0a8a278eaed 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_fields/v_fields.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_fields/v_fields.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_group_members/v_group_members.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_group_members/v_group_members.yamlindex 9774f96e91e..d6ea2e0a4bf 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_group_members/v_group_members.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_group_members/v_group_members.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/v_query_log.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/v_query_log.yamlindex 989ff81cc60..a80faee5d74 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/v_query_log.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_query_log/v_query_log.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_subscriptions/v_subscriptions.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_subscriptions/v_subscriptions.yamlindex aa8bec70a52..a29ff1a95ea 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_subscriptions/v_subscriptions.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_subscriptions/v_subscriptions.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_tables/v_tables.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_tables/v_tables.yamlindex 7910e07df46..bd3ed2f41a1 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_tables/v_tables.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_tables/v_tables.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_task_runs/v_task_runs.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_task_runs/v_task_runs.yamlindex d34843be032..b604444f258 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_task_runs/v_task_runs.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_task_runs/v_task_runs.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_tasks/v_tasks.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_tasks/v_tasks.yamlindex 59b41093334..2d49a6e4910 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_tasks/v_tasks.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_tasks/v_tasks.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_users/v_users.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_users/v_users.yamlindex 174da81dd29..e01df3f4475 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_users/v_users.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_users/v_users.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_view_log/v_view_log.yaml b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_view_log/v_view_log.yamlindex 8ba78d54c45..b7a6d1c9268 100644--- a/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_view_log/v_view_log.yaml+++ b/resources/instance_analytics/databases/Internal Metabase Database/schemas/public/tables/v_view_log/v_view_log.yaml@@ -20,7 +20,7 @@ serdes/meta:archived_at: nullcollection_id: nulldata_authority: unconfigured-data_layer: bronze+data_layer: internaldata_source: nulldatabase_require_filter: nulldeactivated_at: nulldiff --git a/resources/migrations/058_update_migrations.yaml b/resources/migrations/058_update_migrations.yamlindex be77e83f820..d56008593ec 100644--- a/resources/migrations/058_update_migrations.yaml+++ b/resources/migrations/058_update_migrations.yaml@@ -1384,6 +1384,27 @@ databaseChangeLog:AND c.type IN ('library', 'library-data', 'library-metrics'))+ - changeSet:+ id: v58.2026-01-27T13:00:00+ author: wotbrew+ comment: Drop medallion names for data_layer enum use (hidden, internal instead)+ changes:+ - sql:+ sql: >-+ UPDATE metabase_table+ SET data_layer = CASE data_layer+ WHEN 'copper' THEN 'hidden'+ ELSE 'final'+ END+ rollback:+ - sql:+ sql: >-+ UPDATE metabase_table+ SET data_layer = CASE data_layer+ WHEN 'hidden' THEN 'copper'+ ELSE 'bronze'+ END+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.cljindex 645d6ea8809..f8f3169716a 100644--- a/src/metabase/sync/util.clj+++ b/src/metabase/sync/util.clj@@ -346,7 +346,7 @@(def ^:private sync-tables-kv-args{:active true- ;; TODO (Ngoc 2025-11-13) replace this with `metabase_table.data_layer = copper` see the docstring of+ ;; TODO (Ngoc 2025-11-13) replace this with `metabase_table.data_layer = hidden` see the docstring of;; [[metabase.warehouse-schema.models.table/data-layer-types]]:visibility_type nil})diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex e9ae6a59101..4edfec00404 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -33,27 +33,26 @@(def data-layers"Valid values for `Table.data_layer`.- :gold - highest quality, fully visible, synced- :silver - high quality, visible, synced- :bronze - acceptable quality, visible, synced- :copper - low quality, hidden, not synced"- #{:gold :silver :bronze :copper})+ :final - tables published for downstream consumption+ :internal - acceptable quality, visible, synced+ :hidden - low quality, hidden, not synced"+ #{:final :internal :hidden})(defn- visibility-type->data-layer"Convert legacy visibility_type to data_layer.Used when updating via the legacy field."[visibility-type](if (contains? #{:hidden :retired :sensitive :technical :cruft} visibility-type)- :copper- :gold))+ :hidden+ :internal))(defn- data-layer->visibility-type"Convert data_layer back to legacy visibility_type.Used for rollback compatibility to v56."[data-layer](case data-layer- :copper :hidden- ;; gold, silver, bronze all map to visible (nil)+ :hidden :hidden+ ;; internal,final all map to visible (nil)nil))(def field-orderings@@ -161,7 +160,7 @@[table](let [defaults {:display_name (humanization/name->human-readable-name (:name table)):field_order (driver/default-field-order (t2/select-one-fn :engine :model/Database :id (:db_id table)))- :data_layer :bronze}]+ :data_layer :internal}](merge defaults table)))(t2/define-before-delete :model/Tablediff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.cljindex eb841f1d43f..2801e7606dd 100644--- a/test/metabase/warehouse_schema_rest/api/table_test.clj+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj@@ -419,7 +419,7 @@{:display_name "Userz":description "What a nice table!":data_source "transform"- :data_layer "copper"+ :data_layer "hidden":owner_email "[email protected]":owner_user_id (mt/user->id :crowberto)})(is (= (merge@@ -435,7 +435,7 @@:display_name "Userz":is_writable nil:data_source "transform"- :data_layer "copper"+ :data_layer "hidden";; exclusive later (not now):owner_email "[email protected]":owner_user_id (mt/user->id :crowberto)})@@ -1286,14 +1286,14 @@{:display_name "Products2"}](list-tables :term "P")))- (mt/user-http-request :crowberto :put 200 (format "table/%d" products2-id) {:data_layer "gold"})+ (mt/user-http-request :crowberto :put 200 (format "table/%d" products2-id) {:data_layer "final"})(is (=? [{:display_name "Products2"}]- (list-tables :term "P" :data-layer "gold")))+ (list-tables :term "P" :data-layer "final")))(is (=? [{:display_name "People"}{:display_name "Products"}]- (list-tables :term "P" :data-layer "bronze")))))))+ (list-tables :term "P" :data-layer "internal")))))))(deftest ^:parallel update-table-visibility-sync-test(testing "PUT /api/table/:id visibility field synchronization"@@ -1301,20 +1301,20 @@(testing "updating visibility_type syncs to data_layer"(mt/user-http-request :crowberto :put 200 (format "table/%d" (u/the-id table)){:visibility_type "hidden"})- (is (= :copper (t2/select-one-fn :data_layer :model/Table :id (u/the-id table))))+ (is (= :hidden (t2/select-one-fn :data_layer :model/Table :id (u/the-id table))))(is (= :hidden (t2/select-one-fn :visibility_type :model/Table :id (u/the-id table)))))(testing "updating data_layer syncs to visibility_type"(mt/user-http-request :crowberto :put 200 (format "table/%d" (u/the-id table))- {:data_layer "gold"})- (is (= :gold (t2/select-one-fn :data_layer :model/Table :id (u/the-id table))))+ {:data_layer "internal"})+ (is (= :internal (t2/select-one-fn :data_layer :model/Table :id (u/the-id table))))(is (= nil (t2/select-one-fn :visibility_type :model/Table :id (u/the-id table)))))(testing "cannot update both visibility_type and data_layer at once"(is (= "Cannot update both visibility_type and data_layer"(mt/user-http-request :crowberto :put 400 (format "table/%d" (u/the-id table)){:visibility_type "hidden"- :data_layer "copper"})))))))+ :data_layer "hidden"})))))));; NOTE: unused-only-filter-test moved to enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj;; because it depends on EE event handlers to populate the dependency tablediff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex 95ebb283de6..e26cd426f55 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -724,7 +724,7 @@:display_name "Categories":entity_type "entity/GenericTable":initial_sync_status "complete"- :data_layer "copper"+ :data_layer "hidden":fields [(merge(field-details (t2/select-one :model/Field :id (mt/id :categories :id))){:table_id (mt/id :categories)