The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Remote-Sync: Support other git providers (#69675)
Nathan Voxland140d1adcb76f
diff --git a/e2e/support/helpers/e2e-remote-sync-helpers.ts b/e2e/support/helpers/e2e-remote-sync-helpers.tsindex 8413eb3156a..fb0f37404bb 100644--- a/e2e/support/helpers/e2e-remote-sync-helpers.ts+++ b/e2e/support/helpers/e2e-remote-sync-helpers.ts@@ -54,7 +54,7 @@ export const commitToRepo = (// Setup remote sync via the APIexport function configureGit(syncType: "read-write" | "read-only",- syncUrl = LOCAL_GIT_PATH + "/.git",+ syncUrl = "file://" + LOCAL_GIT_PATH + "/.git",collections?: Record<number, boolean>,) {cy.request("PUT", "/api/ee/remote-sync/settings", {@@ -69,7 +69,7 @@ export function configureGit(// Setup remote sync via the API and wait for/trigger the initial importexport function configureGitAndPullChanges(syncType: "read-write" | "read-only",- syncUrl = LOCAL_GIT_PATH + "/.git",+ syncUrl = "file://" + LOCAL_GIT_PATH + "/.git",) {configureGit(syncType, syncUrl);@@ -87,7 +87,7 @@ export function configureGitAndPullChanges(export function configureGitWithNewSyncedCollection(syncType: "read-write" | "read-only",collectionName = "Test Synced Collection",- syncUrl = LOCAL_GIT_PATH + "/.git",+ syncUrl = "file://" + LOCAL_GIT_PATH + "/.git",) {return cy.request("POST", "/api/collection", { name: collectionName })diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.cljindex 326b76ae742..e5fe3e1dc14 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj@@ -40,7 +40,7 @@:can-read-from-env? false)(defsetting remote-sync-url- (deferred-tru "The location of your git repository, e.g. https://github.com/acme-inco/metabase.git")+ (deferred-tru "The location of your git repository, e.g. https://git-host.example.com/yourcompany/repo.git"):type :string:visibility :admin:encryption :no@@ -176,10 +176,9 @@([{:keys [remote-sync-url remote-sync-token remote-sync-branch remote-sync-type]}](when-not (or (not (str/index-of remote-sync-url ":"))(str/starts-with? remote-sync-url "file://")- (and (or (str/starts-with? remote-sync-url "http://")- (str/starts-with? remote-sync-url "https://"))- (str/index-of remote-sync-url "github.com")))- (throw (ex-info "Invalid Repository URL format"+ (str/starts-with? remote-sync-url "http://")+ (str/starts-with? remote-sync-url "https://"))+ (throw (ex-info "Invalid repository URL: only HTTPS URLs are supported (e.g., https://git-host.example.com/yourcompany/repo.git)"{:url remote-sync-url})))(let [source (git/git-source remote-sync-url "HEAD" remote-sync-token)]diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.cljindex e3e3352671e..842622c3b20 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj@@ -10,6 +10,7 @@[metabase.util.log :as log])(:import(java.io File)+ (java.net URI)(org.apache.commons.io FileUtils)(org.eclipse.jgit.api Git GitCommand TransportCommand)(org.eclipse.jgit.dircache DirCache DirCacheEntry)@@ -43,11 +44,30 @@(analytics/inc! :metabase-remote-sync/git-operations-failed analytics-labels)(throw (clean-git-exception e command false))))))-(defn- call-remote-command [^TransportCommand command {:keys [^String token]}]+(defmulti credentials-provider+ "Creates a JGit CredentialsProvider based on the authentication method.++ Dispatches on auth-method keyword. The credentials argument is method-specific+ and can be any data structure appropriate for that authentication method.++ Returns a CredentialsProvider instance or nil if no authentication is needed."+ {:arglists '([remote-url credentials])}+ (fn [remote-url _credentials] (keyword (u/lower-case-en (.getHost (URI. remote-url))))))++(defmethod credentials-provider :default+ [_remote-url ^String token]+ (UsernamePasswordCredentialsProvider. "x-access-token" token))++(defmethod credentials-provider :bitbucket.org+ [_auth-method ^String token]+ (when token+ (UsernamePasswordCredentialsProvider. "x-token-auth" token)))++(defn- call-remote-command [^TransportCommand command {:keys [^String token ^String remote-url]}](let [analytics-labels {:operation (-> command .getClass .getSimpleName) :remote true};; GitHub convention: use "x-access-token" as username when authenticating with a personal access token;; For Gitlab any values can be used as the user name so x-access-token works just as well- credentials-provider (when token (UsernamePasswordCredentialsProvider. "x-access-token" token))]+ credentials-provider (when token (credentials-provider remote-url token))](analytics/inc! :metabase-remote-sync/git-operations analytics-labels)(try@@ -94,7 +114,7 @@(u/prog1 (call-remote-command (-> (Git/cloneRepository)(.setDirectory repo-path)(.setURI remote-url)- (.setBare true)) {:token token})+ (.setBare true)) {:token token :remote-url remote-url})(log/info "Successfully cloned repository" {:repo-path repo-path}))(catch Exception e(throw (ex-info (format "Failed to clone git repository: %s" (ex-message e))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.cljindex 41a208ae69b..f3ba6977df0 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj@@ -538,7 +538,7 @@(testing "PUT /api/ee/remote-sync/settings handles invalid settings"(let [response (mt/user-http-request :crowberto :put 400 "ee/remote-sync/settings"{:remote-sync-url "asdf://invalid-url"})]- (is (= "Invalid Repository URL format" (:error response))))))+ (is (= "Invalid repository URL: only HTTPS URLs are supported (e.g., https://git-host.example.com/yourcompany/repo.git)" (:error response))))))(deftest settings-cannot-change-with-dirty-data(testing "PUT /api/ee/remote-sync/settings doesn't allow losing dirty data"diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/settings_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/settings_test.cljindex 12b7235b0de..fc0af503da6 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/settings_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/settings_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][metabase-enterprise.remote-sync.settings :as settings]+ [metabase-enterprise.remote-sync.source.git :as git][metabase.collections.models.collection.root :as collection.root][metabase.settings.core :as setting][metabase.test :as mt]))@@ -43,6 +44,40 @@(settings/check-and-update-remote-settings! (assoc default-settings :remote-sync-token nil))(is (= nil (settings/remote-sync-token))))))))+(deftest check-git-settings-rejects-non-https-urls+ (testing "git:// URLs are rejected with a helpful error message"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Invalid repository URL: only HTTPS URLs are supported"+ (settings/check-git-settings! {:remote-sync-url "git://github.com/foo/bar.git"+ :remote-sync-token nil+ :remote-sync-branch "main"+ :remote-sync-type :read-only}))))+ (testing "ssh:// URLs are rejected with a helpful error message"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Invalid repository URL: only HTTPS URLs are supported"+ (settings/check-git-settings! {:remote-sync-url "ssh://[email protected]/foo/bar.git"+ :remote-sync-token nil+ :remote-sync-branch "main"+ :remote-sync-type :read-only}))))+ (testing "Non-GitHub HTTPS URLs are accepted"+ (with-redefs [git/git-source (fn [& _] nil)+ git/branches (fn [_] ["main"])]+ (is (nil? (settings/check-git-settings! {:remote-sync-url "https://gitlab.com/foo/bar.git"+ :remote-sync-token nil+ :remote-sync-branch "main"+ :remote-sync-type :read-only}))+ "GitLab HTTPS URLs should be accepted")+ (is (nil? (settings/check-git-settings! {:remote-sync-url "https://bitbucket.org/foo/bar.git"+ :remote-sync-token nil+ :remote-sync-branch "main"+ :remote-sync-type :read-only}))+ "Bitbucket HTTPS URLs should be accepted")+ (is (nil? (settings/check-git-settings! {:remote-sync-url "https://dev.azure.com/org/project/_git/repo"+ :remote-sync-token nil+ :remote-sync-branch "main"+ :remote-sync-type :read-only}))+ "Azure DevOps HTTPS URLs should be accepted"))))+(deftest cannot-set-remote-sync-type-to-invalid-value(is (thrown-with-msg? clojure.lang.ExceptionInfo#"Remote-sync-type set to an unsupported value"diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.cljindex 91752cf428f..aafe703e784 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj@@ -10,7 +10,8 @@(:import (java.io File)(org.apache.commons.io FileUtils)(org.eclipse.jgit.api Git)- (org.eclipse.jgit.lib PersonIdent)))+ (org.eclipse.jgit.lib PersonIdent)+ (org.eclipse.jgit.transport UsernamePasswordCredentialsProvider)))(set! *warn-on-reflection* true)@@ -482,3 +483,11 @@(is (= ["Add new file" "Initial commit"](map :message (git/log repaired-source)))"Should be able to fetch after origin repair")))))++(deftest ^:parallel credentials-provider-test+ (testing "GitHub URL uses x-access-token"+ (let [provider (git/credentials-provider "https://github.com/org/repo.git" "my-token")]+ (is (instance? UsernamePasswordCredentialsProvider provider))))+ (testing "Bitbucket URL uses x-token-auth"+ (let [provider (#'git/credentials-provider "https://bitbucket.org/org/repo" "my-token")]+ (is (instance? UsernamePasswordCredentialsProvider provider)))))diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.tsxindex 52d2cceb7cb..5c3f5d8cd43 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.tsx@@ -7,6 +7,7 @@ import {useGetSettingsQuery,useListCollectionItemsQuery,} from "metabase/api";+import { getErrorMessage } from "metabase/api/utils";import { ExternalLink } from "metabase/common/components/ExternalLink";import { useDocsUrl, useSetting, useToast } from "metabase/common/hooks";import { useConfirmation } from "metabase/common/hooks/use-confirmation";@@ -212,7 +213,7 @@ export const RemoteSyncSettingsForm = (props: RemoteSyncSettingsFormProps) => {onSaveSuccess?.();} catch (error) {sendToast({- message: t`Settings could not be saved`,+ message: getErrorMessage(error, t`Settings could not be saved`),icon: "warning",});throw error;@@ -374,7 +375,7 @@ export const RemoteSyncSettingsForm = (props: RemoteSyncSettingsFormProps) => {<FormTextInputname={URL_KEY}label={t`Repository URL`}- placeholder="https://github.com/yourcompany/metabase-library.git"+ placeholder="https://git-host.example.com/yourcompany/repo.git"labelProps={{ mb: "0.75rem" }}{...getEnvSettingProps(settingDetails?.[URL_KEY])}/>diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.unit.spec.tsxindex 1a58159e376..849ea0e3cd9 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.unit.spec.tsx@@ -1,4 +1,5 @@import userEvent from "@testing-library/user-event";+import fetchMock from "fetch-mock";import { screen, waitFor } from "__support__/ui";import { PLUGIN_TRANSFORMS } from "metabase/plugins";@@ -482,6 +483,84 @@ describe("RemoteSyncSettingsForm", () => {});});+ describe("URL validation", () => {+ it("should show a validation error for non-HTTPS URLs", async () => {+ setup({+ remoteSyncType: "read-only",+ remoteSyncUrl: "",+ remoteSyncEnabled: false,+ });++ const urlInput = screen.getByLabelText(/Repository URL/i);+ await userEvent.type(urlInput, "git://github.com/foo/bar.git");++ // Tab away to trigger validation+ await userEvent.tab();++ await waitFor(() => {+ expect(+ screen.getByText(/Only HTTPS URLs are supported/),+ ).toBeInTheDocument();+ });+ });++ it("should not show a validation error for HTTPS URLs", async () => {+ setup({+ remoteSyncType: "read-only",+ remoteSyncUrl: "",+ remoteSyncEnabled: false,+ });++ const urlInput = screen.getByLabelText(/Repository URL/i);+ await userEvent.type(urlInput, "https://github.com/foo/bar.git");++ // Tab away to trigger validation+ await userEvent.tab();++ // Give validation time to run, then verify no error+ await waitFor(() => {+ expect(+ screen.queryByText(/Only HTTPS URLs are supported/),+ ).not.toBeInTheDocument();+ });+ });+ });++ describe("save error handling", () => {+ it("should show backend error message in toast when save fails", async () => {+ setup({+ remoteSyncType: "read-only",+ remoteSyncUrl: "",+ remoteSyncEnabled: false,+ });++ // Override the settings endpoint to return an error+ fetchMock.removeRoute("remote-sync-settings");+ fetchMock.put(+ "path:/api/ee/remote-sync/settings",+ {+ status: 400,+ body: {+ message: "Invalid branch name",+ },+ },+ { name: "remote-sync-settings" },+ );++ const urlInput = screen.getByLabelText(/Repository URL/i);+ await userEvent.type(urlInput, "https://github.com/foo/bar.git");++ const submitButton = screen.getByRole("button", {+ name: /Set up Remote Sync/i,+ });+ await userEvent.click(submitButton);++ await waitFor(() => {+ expect(screen.getByText(/Invalid branch name/)).toBeInTheDocument();+ });+ });+ });+describe("collections to sync section", () => {it("should display collections section in admin variant when read-write mode is selected during initial setup", async () => {setup({diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.tsindex a78a8bf5214..ae72d53cb60 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts@@ -1,4 +1,5 @@import type { TagDescription } from "@reduxjs/toolkit/query";+import { t } from "ttag";import * as Yup from "yup";import {@@ -21,7 +22,19 @@ export const SYNC_LIBRARY_PENDING_KEY = "sync-library-pending";export const REMOTE_SYNC_SCHEMA = Yup.object({[REMOTE_SYNC_KEY]: Yup.boolean().nullable().default(true),- [URL_KEY]: Yup.string().nullable().default(null),+ [URL_KEY]: Yup.string()+ .nullable()+ .default(null)+ .test(+ "https-url",+ () =>+ t`Only HTTPS URLs are supported (e.g., https://git-host.example.com/yourcompany/repo.git)`,+ (value) =>+ !value ||+ value.startsWith("https://") ||+ value.startsWith("http://") ||+ value.startsWith("file://"),+ ),[TOKEN_KEY]: Yup.string().nullable().default(null),[AUTO_IMPORT_KEY]: Yup.boolean().nullable().default(false),[TRANSFORMS_KEY]: Yup.boolean().nullable().default(false),