Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Remote-Sync: Support other git providers (#69675)

Nathan Voxland140d1adcb76f
diff --git a/e2e/support/helpers/e2e-remote-sync-helpers.ts b/e2e/support/helpers/e2e-remote-sync-helpers.ts
index 8413eb3156a..fb0f37404bb 100644
--- a/e2e/support/helpers/e2e-remote-sync-helpers.ts
+++ b/e2e/support/helpers/e2e-remote-sync-helpers.ts
@@ -54,7 +54,7 @@ export const commitToRepo = (
// Setup remote sync via the API
export function configureGit(
syncType: "read-write" | "read-only",
- syncUrl = LOCAL_GIT_PATH + "/.git",
+ syncUrl = "file://" + LOCAL_GIT_PATH + "/.git",
collections?: Record<number, boolean>,
) {
cy.request("PUT", "/api/ee/remote-sync/settings", {
@@ -69,7 +69,7 @@ export function configureGit(
// Setup remote sync via the API and wait for/trigger the initial import
export function configureGitAndPullChanges(
syncType: "read-write" | "read-only",
- syncUrl = LOCAL_GIT_PATH + "/.git",
+ syncUrl = "file://" + LOCAL_GIT_PATH + "/.git",
) {
configureGit(syncType, syncUrl);
@@ -87,7 +87,7 @@ export function configureGitAndPullChanges(
export function configureGitWithNewSyncedCollection(
syncType: "read-write" | "read-only",
collectionName = "Test Synced Collection",
- syncUrl = LOCAL_GIT_PATH + "/.git",
+ syncUrl = "file://" + LOCAL_GIT_PATH + "/.git",
) {
return cy
.request("POST", "/api/collection", { name: collectionName })
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
index 326b76ae742..e5fe3e1dc14 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
@@ -40,7 +40,7 @@
:can-read-from-env? false)
(defsetting remote-sync-url
- (deferred-tru "The location of your git repository, e.g. https://github.com/acme-inco/metabase.git")
+ (deferred-tru "The location of your git repository, e.g. https://git-host.example.com/yourcompany/repo.git")
:type :string
:visibility :admin
:encryption :no
@@ -176,10 +176,9 @@
([{:keys [remote-sync-url remote-sync-token remote-sync-branch remote-sync-type]}]
(when-not (or (not (str/index-of remote-sync-url ":"))
(str/starts-with? remote-sync-url "file://")
- (and (or (str/starts-with? remote-sync-url "http://")
- (str/starts-with? remote-sync-url "https://"))
- (str/index-of remote-sync-url "github.com")))
- (throw (ex-info "Invalid Repository URL format"
+ (str/starts-with? remote-sync-url "http://")
+ (str/starts-with? remote-sync-url "https://"))
+ (throw (ex-info "Invalid repository URL: only HTTPS URLs are supported (e.g., https://git-host.example.com/yourcompany/repo.git)"
{:url remote-sync-url})))
(let [source (git/git-source remote-sync-url "HEAD" remote-sync-token)]
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
index e3e3352671e..842622c3b20 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
@@ -10,6 +10,7 @@
[metabase.util.log :as log])
(:import
(java.io File)
+ (java.net URI)
(org.apache.commons.io FileUtils)
(org.eclipse.jgit.api Git GitCommand TransportCommand)
(org.eclipse.jgit.dircache DirCache DirCacheEntry)
@@ -43,11 +44,30 @@
(analytics/inc! :metabase-remote-sync/git-operations-failed analytics-labels)
(throw (clean-git-exception e command false))))))
-(defn- call-remote-command [^TransportCommand command {:keys [^String token]}]
+(defmulti credentials-provider
+ "Creates a JGit CredentialsProvider based on the authentication method.
+
+ Dispatches on auth-method keyword. The credentials argument is method-specific
+ and can be any data structure appropriate for that authentication method.
+
+ Returns a CredentialsProvider instance or nil if no authentication is needed."
+ {:arglists '([remote-url credentials])}
+ (fn [remote-url _credentials] (keyword (u/lower-case-en (.getHost (URI. remote-url))))))
+
+(defmethod credentials-provider :default
+ [_remote-url ^String token]
+ (UsernamePasswordCredentialsProvider. "x-access-token" token))
+
+(defmethod credentials-provider :bitbucket.org
+ [_auth-method ^String token]
+ (when token
+ (UsernamePasswordCredentialsProvider. "x-token-auth" token)))
+
+(defn- call-remote-command [^TransportCommand command {:keys [^String token ^String remote-url]}]
(let [analytics-labels {:operation (-> command .getClass .getSimpleName) :remote true}
;; GitHub convention: use "x-access-token" as username when authenticating with a personal access token
;; For Gitlab any values can be used as the user name so x-access-token works just as well
- credentials-provider (when token (UsernamePasswordCredentialsProvider. "x-access-token" token))]
+ credentials-provider (when token (credentials-provider remote-url token))]
(analytics/inc! :metabase-remote-sync/git-operations analytics-labels)
(try
@@ -94,7 +114,7 @@
(u/prog1 (call-remote-command (-> (Git/cloneRepository)
(.setDirectory repo-path)
(.setURI remote-url)
- (.setBare true)) {:token token})
+ (.setBare true)) {:token token :remote-url remote-url})
(log/info "Successfully cloned repository" {:repo-path repo-path}))
(catch Exception e
(throw (ex-info (format "Failed to clone git repository: %s" (ex-message e))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
index 41a208ae69b..f3ba6977df0 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
@@ -538,7 +538,7 @@
(testing "PUT /api/ee/remote-sync/settings handles invalid settings"
(let [response (mt/user-http-request :crowberto :put 400 "ee/remote-sync/settings"
{:remote-sync-url "asdf://invalid-url"})]
- (is (= "Invalid Repository URL format" (:error response))))))
+ (is (= "Invalid repository URL: only HTTPS URLs are supported (e.g., https://git-host.example.com/yourcompany/repo.git)" (:error response))))))
(deftest settings-cannot-change-with-dirty-data
(testing "PUT /api/ee/remote-sync/settings doesn't allow losing dirty data"
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/settings_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/settings_test.clj
index 12b7235b0de..fc0af503da6 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/settings_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/settings_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer :all]
[metabase-enterprise.remote-sync.settings :as settings]
+ [metabase-enterprise.remote-sync.source.git :as git]
[metabase.collections.models.collection.root :as collection.root]
[metabase.settings.core :as setting]
[metabase.test :as mt]))
@@ -43,6 +44,40 @@
(settings/check-and-update-remote-settings! (assoc default-settings :remote-sync-token nil))
(is (= nil (settings/remote-sync-token))))))))
+(deftest check-git-settings-rejects-non-https-urls
+ (testing "git:// URLs are rejected with a helpful error message"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Invalid repository URL: only HTTPS URLs are supported"
+ (settings/check-git-settings! {:remote-sync-url "git://github.com/foo/bar.git"
+ :remote-sync-token nil
+ :remote-sync-branch "main"
+ :remote-sync-type :read-only}))))
+ (testing "ssh:// URLs are rejected with a helpful error message"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Invalid repository URL: only HTTPS URLs are supported"
+ (settings/check-git-settings! {:remote-sync-url "ssh://[email protected]/foo/bar.git"
+ :remote-sync-token nil
+ :remote-sync-branch "main"
+ :remote-sync-type :read-only}))))
+ (testing "Non-GitHub HTTPS URLs are accepted"
+ (with-redefs [git/git-source (fn [& _] nil)
+ git/branches (fn [_] ["main"])]
+ (is (nil? (settings/check-git-settings! {:remote-sync-url "https://gitlab.com/foo/bar.git"
+ :remote-sync-token nil
+ :remote-sync-branch "main"
+ :remote-sync-type :read-only}))
+ "GitLab HTTPS URLs should be accepted")
+ (is (nil? (settings/check-git-settings! {:remote-sync-url "https://bitbucket.org/foo/bar.git"
+ :remote-sync-token nil
+ :remote-sync-branch "main"
+ :remote-sync-type :read-only}))
+ "Bitbucket HTTPS URLs should be accepted")
+ (is (nil? (settings/check-git-settings! {:remote-sync-url "https://dev.azure.com/org/project/_git/repo"
+ :remote-sync-token nil
+ :remote-sync-branch "main"
+ :remote-sync-type :read-only}))
+ "Azure DevOps HTTPS URLs should be accepted"))))
+
(deftest cannot-set-remote-sync-type-to-invalid-value
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"Remote-sync-type set to an unsupported value"
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
index 91752cf428f..aafe703e784 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
@@ -10,7 +10,8 @@
(:import (java.io File)
(org.apache.commons.io FileUtils)
(org.eclipse.jgit.api Git)
- (org.eclipse.jgit.lib PersonIdent)))
+ (org.eclipse.jgit.lib PersonIdent)
+ (org.eclipse.jgit.transport UsernamePasswordCredentialsProvider)))
(set! *warn-on-reflection* true)
@@ -482,3 +483,11 @@
(is (= ["Add new file" "Initial commit"]
(map :message (git/log repaired-source)))
"Should be able to fetch after origin repair")))))
+
+(deftest ^:parallel credentials-provider-test
+ (testing "GitHub URL uses x-access-token"
+ (let [provider (git/credentials-provider "https://github.com/org/repo.git" "my-token")]
+ (is (instance? UsernamePasswordCredentialsProvider provider))))
+ (testing "Bitbucket URL uses x-token-auth"
+ (let [provider (#'git/credentials-provider "https://bitbucket.org/org/repo" "my-token")]
+ (is (instance? UsernamePasswordCredentialsProvider provider)))))
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.tsx
index 52d2cceb7cb..5c3f5d8cd43 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.tsx
@@ -7,6 +7,7 @@ import {
useGetSettingsQuery,
useListCollectionItemsQuery,
} from "metabase/api";
+import { getErrorMessage } from "metabase/api/utils";
import { ExternalLink } from "metabase/common/components/ExternalLink";
import { useDocsUrl, useSetting, useToast } from "metabase/common/hooks";
import { useConfirmation } from "metabase/common/hooks/use-confirmation";
@@ -212,7 +213,7 @@ export const RemoteSyncSettingsForm = (props: RemoteSyncSettingsFormProps) => {
onSaveSuccess?.();
} catch (error) {
sendToast({
- message: t`Settings could not be saved`,
+ message: getErrorMessage(error, t`Settings could not be saved`),
icon: "warning",
});
throw error;
@@ -374,7 +375,7 @@ export const RemoteSyncSettingsForm = (props: RemoteSyncSettingsFormProps) => {
<FormTextInput
name={URL_KEY}
label={t`Repository URL`}
- placeholder="https://github.com/yourcompany/metabase-library.git"
+ placeholder="https://git-host.example.com/yourcompany/repo.git"
labelProps={{ mb: "0.75rem" }}
{...getEnvSettingProps(settingDetails?.[URL_KEY])}
/>
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.unit.spec.tsx
index 1a58159e376..849ea0e3cd9 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/RemoteSyncAdminSettings/RemoteSyncSettingsForm.unit.spec.tsx
@@ -1,4 +1,5 @@
import userEvent from "@testing-library/user-event";
+import fetchMock from "fetch-mock";
import { screen, waitFor } from "__support__/ui";
import { PLUGIN_TRANSFORMS } from "metabase/plugins";
@@ -482,6 +483,84 @@ describe("RemoteSyncSettingsForm", () => {
});
});
+ describe("URL validation", () => {
+ it("should show a validation error for non-HTTPS URLs", async () => {
+ setup({
+ remoteSyncType: "read-only",
+ remoteSyncUrl: "",
+ remoteSyncEnabled: false,
+ });
+
+ const urlInput = screen.getByLabelText(/Repository URL/i);
+ await userEvent.type(urlInput, "git://github.com/foo/bar.git");
+
+ // Tab away to trigger validation
+ await userEvent.tab();
+
+ await waitFor(() => {
+ expect(
+ screen.getByText(/Only HTTPS URLs are supported/),
+ ).toBeInTheDocument();
+ });
+ });
+
+ it("should not show a validation error for HTTPS URLs", async () => {
+ setup({
+ remoteSyncType: "read-only",
+ remoteSyncUrl: "",
+ remoteSyncEnabled: false,
+ });
+
+ const urlInput = screen.getByLabelText(/Repository URL/i);
+ await userEvent.type(urlInput, "https://github.com/foo/bar.git");
+
+ // Tab away to trigger validation
+ await userEvent.tab();
+
+ // Give validation time to run, then verify no error
+ await waitFor(() => {
+ expect(
+ screen.queryByText(/Only HTTPS URLs are supported/),
+ ).not.toBeInTheDocument();
+ });
+ });
+ });
+
+ describe("save error handling", () => {
+ it("should show backend error message in toast when save fails", async () => {
+ setup({
+ remoteSyncType: "read-only",
+ remoteSyncUrl: "",
+ remoteSyncEnabled: false,
+ });
+
+ // Override the settings endpoint to return an error
+ fetchMock.removeRoute("remote-sync-settings");
+ fetchMock.put(
+ "path:/api/ee/remote-sync/settings",
+ {
+ status: 400,
+ body: {
+ message: "Invalid branch name",
+ },
+ },
+ { name: "remote-sync-settings" },
+ );
+
+ const urlInput = screen.getByLabelText(/Repository URL/i);
+ await userEvent.type(urlInput, "https://github.com/foo/bar.git");
+
+ const submitButton = screen.getByRole("button", {
+ name: /Set up Remote Sync/i,
+ });
+ await userEvent.click(submitButton);
+
+ await waitFor(() => {
+ expect(screen.getByText(/Invalid branch name/)).toBeInTheDocument();
+ });
+ });
+ });
+
describe("collections to sync section", () => {
it("should display collections section in admin variant when read-write mode is selected during initial setup", async () => {
setup({
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts
index a78a8bf5214..ae72d53cb60 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/constants.ts
@@ -1,4 +1,5 @@
import type { TagDescription } from "@reduxjs/toolkit/query";
+import { t } from "ttag";
import * as Yup from "yup";
import {
@@ -21,7 +22,19 @@ export const SYNC_LIBRARY_PENDING_KEY = "sync-library-pending";
export const REMOTE_SYNC_SCHEMA = Yup.object({
[REMOTE_SYNC_KEY]: Yup.boolean().nullable().default(true),
- [URL_KEY]: Yup.string().nullable().default(null),
+ [URL_KEY]: Yup.string()
+ .nullable()
+ .default(null)
+ .test(
+ "https-url",
+ () =>
+ t`Only HTTPS URLs are supported (e.g., https://git-host.example.com/yourcompany/repo.git)`,
+ (value) =>
+ !value ||
+ value.startsWith("https://") ||
+ value.startsWith("http://") ||
+ value.startsWith("file://"),
+ ),
[TOKEN_KEY]: Yup.string().nullable().default(null),
[AUTO_IMPORT_KEY]: Yup.boolean().nullable().default(false),
[TRANSFORMS_KEY]: Yup.boolean().nullable().default(false),