The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Hide the collections to sync on new git sync setups behind remote-sync-collections-enabled (#84802)
Alexander Polyankind5084b45dcb2
diff --git a/e2e/test/scenarios/admin/remote-sync-setup.cy.spec.ts b/e2e/test/scenarios/admin/remote-sync-setup.cy.spec.tsnew file mode 100644index 00000000000..ad5479d44c4--- /dev/null+++ b/e2e/test/scenarios/admin/remote-sync-setup.cy.spec.ts@@ -0,0 +1,104 @@+const { H } = cy;++const LOCAL_GIT_URL = "file://" + H.LOCAL_GIT_PATH + "/.git";++const FIRST_SETUP_TRANSFORM_NAME = "First Setup Transform";++describe("Remote Sync first-time setup", () => {+ beforeEach(() => {+ H.restore("postgres-writable");+ H.resetSnowplow();+ cy.signInAsAdmin();+ H.activateToken("pro-self-hosted");+ H.setupGitSync();+ H.interceptTask();+ H.updateSetting("transforms-enabled", true);+ cy.intercept("PUT", "/api/ee/remote-sync/settings").as("saveSettings");+ cy.intercept("POST", "/api/ee/data-studio/table/publish-tables").as(+ "publishTables",+ );+ cy.intercept("POST", "/api/transform").as("createTransform");+ });++ afterEach(() => {+ H.expectNoBadSnowplowEvents();+ });++ it("sets up git sync from scratch, creating Library and syncing it with transforms", () => {+ H.updateSetting("remote-sync-collections-enabled", false);++ cy.visit("/admin/settings/remote-sync");+ cy.findByLabelText(/repository url/i)+ .should("be.visible")+ .clear()+ .type(LOCAL_GIT_URL);+ cy.findByTestId("admin-layout-content").findByText("Read-write").click();+ cy.findByTestId("admin-layout-content")+ .findByText("Collections to sync")+ .should("not.exist");+ cy.button("Set up remote sync").click();+ cy.wait("@saveSettings").its("response.statusCode").should("eq", 200);++ cy.log("Publish a table to the Library");+ H.DataModel.visitDataStudio();+ H.DataModel.TablePicker.getDatabase("Sample Database").click();+ H.DataModel.TablePicker.getTable("Products").click();+ cy.findByRole("button", { name: /Publish/ }).click();+ H.modal().findByText("Publish this table").click();+ cy.wait("@publishTables");++ cy.log("Create a transform");+ cy.visit("/data-studio/transforms");+ cy.button("Create a transform").click();+ H.popover().findByText("SQL query").click();+ H.popover().findByText("Writable Postgres12").click();+ H.NativeEditor.type("SELECT 1", { allowFastSet: true }).blur();+ H.DataStudio.Transforms.saveChangesButton().click();+ H.modal().within(() => {+ cy.findByLabelText("Name").clear().type(FIRST_SETUP_TRANSFORM_NAME);+ cy.button("Save").click();+ });+ cy.wait("@createTransform").its("response.statusCode").should("eq", 200);++ cy.log("The push lists both the table and the transform");+ H.clickPushOption();+ H.modal().within(() => {+ cy.findAllByText("PRODUCTS").should("have.length.at.least", 1);+ cy.findByText(FIRST_SETUP_TRANSFORM_NAME)+ .scrollIntoView()+ .should("be.visible");+ cy.button(/Push changes/).click();+ });+ H.waitForTask({ taskName: "export" });++ cy.log("The repository holds the table and the transform");+ H.wrapSyncedCollectionFiles();+ cy.get<string[]>("@syncedCollectionFiles").then((files) => {+ expect(files).to.include.members([+ "databases/sample_database/schemas/public/tables/products/products___tableusersettings.yaml",+ "collections/transforms/first_setup_transform.yaml",+ ]);+ });+ });++ it("opens branch picker after push while collections to sync are hidden", () => {+ H.updateSetting("remote-sync-collections-enabled", false);+ H.configureGitWithNewSyncedCollection("read-write");+ H.createBranchViaSettings("first-branch");++ cy.visit("/collection/root");+ H.moveCollectionItemToSyncedCollection(+ "Orders, Count",+ "Test Synced Collection",+ );+ H.clickPushOption();+ H.modal()+ .button(/Push changes/)+ .click();+ H.waitForTask({ taskName: "export" });++ cy.log("Branch picker opens with both branches listed");+ H.createBranchViaSettings("second-branch");+ H.getSettingsBranchSwitcher().should("contain.text", "second-branch");+ });+});diff --git a/e2e/test/scenarios/admin/remote-sync.cy.spec.ts b/e2e/test/scenarios/admin/remote-sync.cy.spec.tsindex e6056c40d82..1093010a27e 100644--- a/e2e/test/scenarios/admin/remote-sync.cy.spec.ts+++ b/e2e/test/scenarios/admin/remote-sync.cy.spec.ts@@ -40,6 +40,7 @@ const setup = (snapshot = "default") => {H.resetSnowplow();cy.signInAsAdmin();H.activateToken("pro-self-hosted");+ H.updateSetting("remote-sync-collections-enabled", true);H.setupGitSync();H.interceptTask();};diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/init.cljindex 572d468596e..235a91b4651 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj@@ -42,6 +42,7 @@(defn- remote-sync-init [];; Only stale rows: a fresh open row may belong to a worker on another node sharing this app DB.(remote-sync.task/supersede-stale-tasks!)+ (settings/pin-remote-sync-collections-enabled!)(events/convert-legacy-transforms-root!)(if (settings/remote-sync-enabled)(dodiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.cljindex f5835b5a8d3..58e6bf62dea 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj@@ -104,6 +104,20 @@:encryption :no:default false)+(defsetting remote-sync-collections-enabled+ (deferred-tru "Whether git sync settings let admins choose collections to sync, besides the Library and transforms.")+ :type :boolean+ :visibility :admin+ :export? false+ :encryption :no+ :default false)++(defn pin-remote-sync-collections-enabled!+ "Stores [[remote-sync-collections-enabled]] when nothing has set it yet, enabled only where remote sync is already configured."+ []+ (when (= :default (setting/get-raw-value-source :remote-sync-collections-enabled))+ (remote-sync-collections-enabled! (remote-sync-enabled))))+(defsetting remote-sync-check-changes-cache-ttl-seconds(deferred-tru "Time-to-live in seconds for the remote changes check cache. Default is 60 seconds."):type :integerdiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/init_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/init_test.cljindex 3c5c7397076..eb23365b334 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/init_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/init_test.clj@@ -6,9 +6,11 @@[metabase-enterprise.remote-sync.impl :as impl][metabase-enterprise.remote-sync.init :as init][metabase-enterprise.remote-sync.models.remote-sync-object :as remote-sync.object]+ [metabase-enterprise.remote-sync.settings :as settings][metabase-enterprise.remote-sync.test-helpers :as th][metabase.collections.models.collection :as collection][metabase.collections.test-utils :as collections.tu]+ [metabase.settings.core :as setting][metabase.startup.core :as startup][metabase.test :as mt][metabase.test.fixtures :as fixtures]@@ -41,6 +43,21 @@(#'init/remote-sync-init)(is (empty? @calls)))))))+(deftest pin-remote-sync-collections-enabled-test+ (testing "First startup stores whether remote sync was already configured, so existing setups keep choosing collections"+ (doseq [[url expected] [["https://github.com/foo/bar.git" true] [nil false]]]+ (testing (pr-str url)+ (mt/with-temporary-setting-values [:remote-sync-url url+ :remote-sync-collections-enabled nil]+ (settings/pin-remote-sync-collections-enabled!)+ (is (= :database (setting/get-raw-value-source :remote-sync-collections-enabled)))+ (is (= expected (settings/remote-sync-collections-enabled)))))))+ (testing "Later startups keep stored value"+ (mt/with-temporary-setting-values [:remote-sync-url nil+ :remote-sync-collections-enabled true]+ (settings/pin-remote-sync-collections-enabled!)+ (is (true? (settings/remote-sync-collections-enabled))))))+(deftest remote-sync-init-read-only-without-branch-throws-test(testing "Read-only with enabled sync but no branch throws"(mt/with-temporary-setting-values [:remote-sync-url "file://my/repo.git"diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/BranchSwitcher.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/BranchSwitcher.tsxindex 6d0ff0673c5..0e90408ace1 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/BranchSwitcher.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/BranchSwitcher.tsx@@ -137,6 +137,7 @@ export const BranchSwitcher = ({position="bottom-start"width={320}withinPortal+ middlewares={{ flip: false, shift: true }}><Combobox.Target><Buttondiff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.setup.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.setup.spec.tsxindex 79adc288e6e..8009e59c9c4 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.setup.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.setup.spec.tsx@@ -48,6 +48,7 @@ const setupEndpoints = ({remoteSyncBranch = "main",remoteSyncAutoImport = false,remoteSyncTransforms = false,+ remoteSyncCollectionsEnabled,// Unjustified type cast. FIXMElibraryCollection = null as Collection | null,// Unjustified type cast. FIXME@@ -67,6 +68,7 @@ const setupEndpoints = ({remoteSyncBranch?: string;remoteSyncAutoImport?: boolean;remoteSyncTransforms?: boolean;+ remoteSyncCollectionsEnabled?: boolean;libraryCollection?: Collection | null;dirty?: any[];rootCollectionItems?: CollectionItem[];@@ -84,6 +86,7 @@ const setupEndpoints = ({"remote-sync-branch": remoteSyncBranch,"remote-sync-auto-import": remoteSyncAutoImport,"remote-sync-transforms": remoteSyncTransforms,+ "remote-sync-collections-enabled": remoteSyncCollectionsEnabled,"development-mode?": isDevInstance,});@@ -122,11 +125,13 @@ const createStoreState = ({isDevInstance = false,remoteSyncEnabled = false,remoteSyncType = "read-only" as const,+ remoteSyncCollectionsEnabled = true,}: {isAdmin?: boolean;isDevInstance?: boolean;remoteSyncEnabled?: boolean;remoteSyncType?: "read-only" | "read-write";+ remoteSyncCollectionsEnabled?: boolean;} = {}) => {return createMockState({currentUser: createMockUser({ is_superuser: isAdmin }),@@ -134,6 +139,7 @@ const createStoreState = ({"development-mode?": isDevInstance,"remote-sync-enabled": remoteSyncEnabled,"remote-sync-type": remoteSyncType,+ "remote-sync-collections-enabled": remoteSyncCollectionsEnabled,}),});};@@ -146,6 +152,7 @@ interface SetupOpts {remoteSyncType?: "read-only" | "read-write";remoteSyncBranch?: string;remoteSyncTransforms?: boolean;+ remoteSyncCollectionsEnabled?: boolean;libraryCollection?: Collection | null;dirty?: any[];rootCollectionItems?: CollectionItem[];@@ -167,6 +174,7 @@ export const setup = ({remoteSyncType = "read-only",remoteSyncBranch = "main",remoteSyncTransforms = false,+ remoteSyncCollectionsEnabled = true,libraryCollection = null,dirty = [],rootCollectionItems = [],@@ -188,6 +196,7 @@ export const setup = ({remoteSyncType,remoteSyncBranch,remoteSyncTransforms,+ remoteSyncCollectionsEnabled,libraryCollection,dirty,rootCollectionItems,@@ -205,6 +214,7 @@ export const setup = ({isDevInstance,remoteSyncEnabled,remoteSyncType,+ remoteSyncCollectionsEnabled,}),},);diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.tsxindex bd8313d35e2..57524222998 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.tsx@@ -55,6 +55,9 @@ export const RemoteSyncSettingsForm = ({const { data: settingValues } = useGetSettingsQuery();const { data: settingDetails } = useGetAdminSettingsDetailsQuery();const isRemoteSyncEnabled = !!useSetting(REMOTE_SYNC_KEY);+ const isCollectionsSyncEnabled = !!useSetting(+ "remote-sync-collections-enabled",+ );const { data: dirtyData } = useGetRemoteSyncChangesQuery(undefined, {refetchOnMountOrArgChange: true,});@@ -106,9 +109,9 @@ export const RemoteSyncSettingsForm = ({<BranchSwitcherSection dirty={dirtyEntities} />)}{isReadOnly && <ReadOnlyBranchSection />}- {(isRemoteSyncEnabled || isReadWrite) && !isModalVariant && (- <CollectionsToSyncSection />- )}+ {isCollectionsSyncEnabled &&+ (isRemoteSyncEnabled || isReadWrite) &&+ !isModalVariant && <CollectionsToSyncSection />}{isModalVariant && isReadWrite && (<RemoteSyncSettingsSection title={t`Content to sync`}><TopLevelCollectionsList skipCollections />diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.unit.spec.tsxindex 7dfc3c0ecde..d0aa6a91b85 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.unit.spec.tsx@@ -994,6 +994,54 @@ describe("RemoteSyncSettingsForm", () => {expect(screen.getByText("Collections to sync")).toBeInTheDocument();});+ it("should not display collections section in admin variant when remote-sync-collections-enabled is off", async () => {+ setup({+ remoteSyncEnabled: false,+ remoteSyncType: "read-write",+ remoteSyncCollectionsEnabled: false,+ });++ await waitFor(() => {+ expect(screen.getByLabelText("Read-write")).toBeChecked();+ });++ expect(screen.queryByText("Collections to sync")).not.toBeInTheDocument();+ });++ it("should create and sync library and sync transforms by default when collections section is hidden", async () => {+ PLUGIN_TRANSFORMS.isEnabled = true;+ setup({+ remoteSyncEnabled: false,+ remoteSyncType: "read-write",+ remoteSyncCollectionsEnabled: false,+ libraryCollection: null,+ hasLibraryFeature: true,+ createdLibraryCollection: createMockLibraryCollection({ id: 555 }),+ });++ await userEvent.type(+ screen.getByLabelText(/Repository URL/i),+ "https://github.com/foo/bar.git",+ );+ await userEvent.click(+ screen.getByRole("button", { name: /Set up remote sync/i }),+ );++ const posts = await findRequests("POST");+ expect(+ posts.filter((request) => request.url.includes("/api/ee/library")),+ ).toHaveLength(1);+ const puts = await findRequests("PUT");+ const settingsRequest = puts.find((request) =>+ request.url.includes("/api/ee/remote-sync/settings"),+ );+ expect(settingsRequest?.body).toMatchObject({+ "remote-sync-transforms": true,+ collections: { 555: true },+ });+ PLUGIN_TRANSFORMS.isEnabled = false;+ });+it("should not display collections section in modal variant when read-write mode is selected", async () => {setup({remoteSyncEnabled: false,diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.tsindex 9e2c5420544..c8076901d0f 100644--- a/frontend/src/metabase-types/api/settings.ts+++ b/frontend/src/metabase-types/api/settings.ts@@ -805,6 +805,7 @@ export interface EnterpriseSettings extends Settings {"remote-sync-type"?: RemoteSyncType | null;"remote-sync-auto-import"?: boolean | null;"remote-sync-transforms"?: boolean | null;+ "remote-sync-collections-enabled"?: boolean | null;"login-page-illustration"?: IllustrationSettingValue;"login-page-illustration-custom"?: string;"landing-page-illustration"?: IllustrationSettingValue;