Skip to content

The Unified and Autonomous Development Platform

Fix issue #370 - Import code and issues from Bitbucket Cloud

Robin Shen9f5140229c61
diff --git a/server-plugin/pom.xml b/server-plugin/pom.xml
index 80c4adaba7..caa9889168 100644
--- a/server-plugin/pom.xml
+++ b/server-plugin/pom.xml
@@ -32,5 +32,6 @@
<module>server-plugin-import-youtrack</module>
<module>server-plugin-import-gitlab</module>
<module>server-plugin-import-gitea</module>
+ <module>server-plugin-import-bitbucketcloud</module>
</modules>
</project>
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/pom.xml b/server-plugin/server-plugin-import-bitbucketcloud/pom.xml
new file mode 100644
index 0000000000..0b2597958f
--- /dev/null
+++ b/server-plugin/server-plugin-import-bitbucketcloud/pom.xml
@@ -0,0 +1,13 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>server-plugin-import-bitbucketcloud</artifactId>
+ <parent>
+ <groupId>io.onedev</groupId>
+ <artifactId>server-plugin</artifactId>
+ <version>4.8.1</version>
+ </parent>
+ <properties>
+ <moduleClass>io.onedev.server.plugin.imports.bitbucketcloud.BitbucketPluginModule</moduleClass>
+ </properties>
+</project>
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.java
new file mode 100644
index 0000000000..cc46d19406
--- /dev/null
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.java
@@ -0,0 +1,39 @@
+package io.onedev.server.plugin.imports.bitbucketcloud;
+
+import java.io.Serializable;
+import java.util.Collection;
+
+import com.google.common.collect.Lists;
+
+import io.onedev.commons.launcher.loader.AbstractPluginModule;
+import io.onedev.server.imports.ProjectImporter;
+import io.onedev.server.imports.ProjectImporterContribution;
+
+/**
+ * NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.
+ *
+ */
+
+public class BitbucketPluginModule extends AbstractPluginModule {
+ @Override
+ protected void configure() {
+ super.configure();
+
+ // put your guice bindings here
+ contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {
+
+ @Override
+ public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {
+ return Lists.newArrayList(new BitbucketProjectImporter());
+ }
+
+ @Override
+ public int getOrder() {
+ return 250;
+ }
+
+ });
+
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java
new file mode 100644
index 0000000000..b55c8f01f8
--- /dev/null
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java
@@ -0,0 +1,102 @@
+package io.onedev.server.plugin.imports.bitbucketcloud;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import javax.ws.rs.client.Client;
+
+import org.apache.http.client.utils.URIBuilder;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import io.onedev.commons.utils.ExplicitException;
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.imports.ProjectImporter;
+import io.onedev.server.model.Project;
+import io.onedev.server.storage.StorageManager;
+import io.onedev.server.util.SimpleLogger;
+
+public class BitbucketProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ProjectImportOption> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getName() {
+ return ImportUtils.NAME;
+ }
+
+ @Override
+ public String doImport(ImportServer where, ProjectImportSource what, ProjectImportOption how,
+ boolean dryRun, SimpleLogger logger) {
+ Collection<Long> projectIds = new ArrayList<>();
+ Client client = where.newClient();
+ try {
+ for (ProjectMapping projectMapping: what.getProjectMappings()) {
+ logger.log("Cloning code from repository " + projectMapping.getBitbucketRepo() + "...");
+
+ String apiEndpoint = where.getApiEndpoint("/repositories/" + projectMapping.getBitbucketRepo());
+ JsonNode repoNode = ImportUtils.get(client, apiEndpoint, logger);
+ Project project = new Project();
+ project.setName(projectMapping.getOneDevProject());
+ project.setDescription(repoNode.get("description").asText(null));
+
+ boolean isPrivate = repoNode.get("is_private").asBoolean();
+ if (!isPrivate && how.getPublicRole() != null)
+ project.setDefaultRole(how.getPublicRole());
+
+ String cloneUrl = null;
+ for (JsonNode cloneNode: repoNode.get("links").get("clone")) {
+ if (cloneNode.get("name").asText().equals("https")) {
+ cloneUrl = cloneNode.get("href").asText();
+ break;
+ }
+ }
+ if (cloneUrl == null)
+ throw new ExplicitException("Https clone url not found");
+
+ URIBuilder builder = new URIBuilder(cloneUrl);
+ if (isPrivate)
+ builder.setUserInfo(where.getUserName(), where.getAppPassword());
+
+ if (!dryRun) {
+ OneDev.getInstance(ProjectManager.class).clone(project, builder.build().toString());
+ projectIds.add(project.getId());
+ }
+ }
+
+ return "Repositories imported successfully";
+ } catch (Exception e) {
+ for (Long projectId: projectIds)
+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ throw new RuntimeException(e);
+ } finally {
+ client.close();
+ }
+ }
+
+ @Override
+ public ProjectImportSource getWhat(ImportServer where, SimpleLogger logger) {
+ ProjectImportSource importSource = new ProjectImportSource();
+ Client client = where.newClient();
+ try {
+ String apiEndpoint = where.getApiEndpoint("/repositories?role=member");
+ for (JsonNode repoNode: ImportUtils.list(client, apiEndpoint, logger)) {
+ String fullName = repoNode.get("full_name").asText();
+ ProjectMapping projectMapping = new ProjectMapping();
+ projectMapping.setBitbucketRepo(fullName);
+ projectMapping.setOneDevProject(fullName.replace("/", "-"));
+ importSource.getProjectMappings().add(projectMapping);
+ }
+ } finally {
+ client.close();
+ }
+ return importSource;
+ }
+
+ @Override
+ public ProjectImportOption getHow(ImportServer where, ProjectImportSource what, SimpleLogger logger) {
+ return new ProjectImportOption();
+ }
+
+}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java
new file mode 100644
index 0000000000..53f35453b7
--- /dev/null
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java
@@ -0,0 +1,112 @@
+package io.onedev.server.plugin.imports.bitbucketcloud;
+
+import java.io.Serializable;
+
+import javax.validation.ConstraintValidatorContext;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Invocation;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Response;
+
+import org.apache.wicket.MetaDataKey;
+import org.glassfish.jersey.client.ClientProperties;
+import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
+import org.hibernate.validator.constraints.NotEmpty;
+
+import io.onedev.commons.utils.StringUtils;
+import io.onedev.server.util.JerseyUtils;
+import io.onedev.server.util.validation.Validatable;
+import io.onedev.server.util.validation.annotation.ClassValidating;
+import io.onedev.server.web.editable.annotation.Editable;
+import io.onedev.server.web.editable.annotation.Password;
+
+@Editable
+@ClassValidating
+public class ImportServer implements Serializable, Validatable {
+
+ private static final long serialVersionUID = 1L;
+
+ static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ };
+
+ static final String PROP_APP_PASSWORD = "appPassword";
+
+ private String userName;
+
+ private String appPassword;
+
+ @Editable(order=10, name="Bitbucket Login Name")
+ @NotEmpty
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ @Editable(order=100, name="Bitbucket App Password", description="Bitbucket app password should be generated with "
+ + "permission <b>account/read</b>, <b>repositories/read</b> and <b>issues:read</b>")
+ @Password
+ @NotEmpty
+ public String getAppPassword() {
+ return appPassword;
+ }
+
+ public void setAppPassword(String appPassword) {
+ this.appPassword = appPassword;
+ }
+
+ public String getApiEndpoint(String apiPath) {
+ return "https://api.bitbucket.org/2.0/" + StringUtils.stripStart(apiPath, "/");
+ }
+
+ @Override
+ public boolean isValid(ConstraintValidatorContext context) {
+ Client client = ClientBuilder.newClient();
+ client.register(HttpAuthenticationFeature.basic(getUserName(), getAppPassword()));
+ try {
+ String apiEndpoint = getApiEndpoint("/user");
+ WebTarget target = client.target(apiEndpoint);
+ Invocation.Builder builder = target.request();
+ try (Response response = builder.get()) {
+ if (response.getStatus() == 401) {
+ context.disableDefaultConstraintViolation();
+ String errorMessage = "Authentication failed";
+ context.buildConstraintViolationWithTemplate(errorMessage)
+ .addPropertyNode(PROP_APP_PASSWORD).addConstraintViolation();
+ return false;
+ } else {
+ String errorMessage = JerseyUtils.checkStatus(apiEndpoint, response);
+ if (errorMessage != null) {
+ context.disableDefaultConstraintViolation();
+ context.buildConstraintViolationWithTemplate(errorMessage).addConstraintViolation();
+ return false;
+ }
+ }
+ }
+ } catch (Exception e) {
+ context.disableDefaultConstraintViolation();
+ String errorMessage = "Error connecting api service";
+ if (e.getMessage() != null)
+ errorMessage += ": " + e.getMessage();
+ context.buildConstraintViolationWithTemplate(errorMessage).addConstraintViolation();
+ return false;
+ } finally {
+ client.close();
+ }
+ return true;
+ }
+
+ Client newClient() {
+ Client client = ClientBuilder.newClient();
+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);
+ client.register(HttpAuthenticationFeature.basic(getUserName(), getAppPassword()));
+ return client;
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportUtils.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportUtils.java
new file mode 100644
index 0000000000..5b9cb58bea
--- /dev/null
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportUtils.java
@@ -0,0 +1,95 @@
+package io.onedev.server.plugin.imports.bitbucketcloud;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.Invocation;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Response;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.client.utils.URIBuilder;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import io.onedev.commons.utils.ExplicitException;
+import io.onedev.server.util.SimpleLogger;
+
+public class ImportUtils {
+
+ static final String NAME = "Bitbucket Cloud";
+
+ static final int PER_PAGE = 50;
+
+ static List<JsonNode> list(Client client, String apiEndpoint, SimpleLogger logger) {
+ List<JsonNode> result = new ArrayList<>();
+ list(client, apiEndpoint, new PageDataConsumer() {
+
+ @Override
+ public void consume(List<JsonNode> pageData) {
+ result.addAll(pageData);
+ }
+
+ }, logger);
+ return result;
+ }
+
+ static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,
+ SimpleLogger logger) {
+ URI uri;
+ try {
+ uri = new URIBuilder(apiEndpoint)
+ .addParameter("pagelen", String.valueOf(PER_PAGE)).build();
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+
+ while (true) {
+ try {
+ List<JsonNode> pageData = new ArrayList<>();
+ JsonNode resultNode = get(client, uri.toString(), logger);
+ if (resultNode.hasNonNull("values")) {
+ for (JsonNode each: resultNode.get("values"))
+ pageData.add(each);
+ pageDataConsumer.consume(pageData);
+ }
+ if (resultNode.hasNonNull("next"))
+ uri = new URIBuilder(resultNode.get("next").asText()).build();
+ else
+ break;
+ } catch (URISyntaxException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static JsonNode get(Client client, String apiEndpoint, SimpleLogger logger) {
+ WebTarget target = client.target(apiEndpoint);
+ Invocation.Builder builder = target.request();
+ while (true) {
+ try (Response response = builder.get()) {
+ int status = response.getStatus();
+ if (status != 200) {
+ String errorMessage = response.readEntity(String.class);
+ if (StringUtils.isNotBlank(errorMessage)) {
+ throw new ExplicitException(String.format("Http request failed (url: %s, status code: %d, error message: %s)",
+ apiEndpoint, status, errorMessage));
+ } else {
+ throw new ExplicitException(String.format("Http request failed (status: %s)", status));
+ }
+ }
+ return response.readEntity(JsonNode.class);
+ }
+ }
+ }
+
+ static interface PageDataConsumer {
+
+ void consume(List<JsonNode> pageData) throws InterruptedException;
+
+ }
+
+}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportOption.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportOption.java
new file mode 100644
index 0000000000..44a12ce103
--- /dev/null
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportOption.java
@@ -0,0 +1,40 @@
+package io.onedev.server.plugin.imports.bitbucketcloud;
+
+import java.io.Serializable;
+
+import javax.annotation.Nullable;
+
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.RoleManager;
+import io.onedev.server.model.Role;
+import io.onedev.server.web.editable.annotation.Editable;
+import io.onedev.server.web.editable.annotation.RoleChoice;
+
+@Editable
+public class ProjectImportOption implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private String publicRoleName;
+
+ @Editable(order=100, name="Public Role", description="If specified, all public repositories imported from GitHub "
+ + "will use this as default role. Private repositories are not affected")
+ @RoleChoice
+ @Nullable
+ public String getPublicRoleName() {
+ return publicRoleName;
+ }
+
+ public void setPublicRoleName(String publicRoleName) {
+ this.publicRoleName = publicRoleName;
+ }
+
+ @Nullable
+ public Role getPublicRole() {
+ if (publicRoleName != null)
+ return OneDev.getInstance(RoleManager.class).find(publicRoleName);
+ else
+ return null;
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportSource.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportSource.java
new file mode 100644
index 0000000000..318382b776
--- /dev/null
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportSource.java
@@ -0,0 +1,64 @@
+package io.onedev.server.plugin.imports.bitbucketcloud;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.validation.ConstraintValidatorContext;
+import javax.validation.constraints.Size;
+
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.util.validation.Validatable;
+import io.onedev.server.util.validation.annotation.ClassValidating;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+@ClassValidating
+public class ProjectImportSource implements Serializable, Validatable {
+
+ private static final long serialVersionUID = 1L;
+
+ private List<ProjectMapping> projectMappings = new ArrayList<>();
+
+ @Editable(order=100, name="Repositories to Import")
+ @Size(min=1, max=10000, message="No repositories to import")
+ public List<ProjectMapping> getProjectMappings() {
+ return projectMappings;
+ }
+
+ public void setProjectMappings(List<ProjectMapping> projectMappings) {
+ this.projectMappings = projectMappings;
+ }
+
+ @Override
+ public boolean isValid(ConstraintValidatorContext context) {
+ boolean isValid = true;
+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
+ for (int i=0; i<projectMappings.size(); i++) {
+ if (projectManager.find(projectMappings.get(i).getOneDevProject()) != null) {
+ context.buildConstraintViolationWithTemplate("Project name already used")
+ .addPropertyNode("projectMappings")
+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
+ .inIterable().atIndex(i).addConstraintViolation();
+ isValid = false;
+ } else {
+ for (int j=0; j<projectMappings.size(); j++) {
+ if (j != i && projectMappings.get(j).getOneDevProject().equals(projectMappings.get(i).getOneDevProject())) {
+ context.buildConstraintViolationWithTemplate("Duplicate project name")
+ .addPropertyNode("projectMappings")
+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
+ .inIterable().atIndex(i).addConstraintViolation();
+ isValid = false;
+ break;
+ }
+ }
+ }
+ }
+
+ if (!isValid)
+ context.disableDefaultConstraintViolation();
+ return isValid;
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectMapping.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectMapping.java
new file mode 100644
index 0000000000..185a7d51e3
--- /dev/null
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectMapping.java
@@ -0,0 +1,44 @@
+package io.onedev.server.plugin.imports.bitbucketcloud;
+
+import java.io.Serializable;
+
+import org.hibernate.validator.constraints.NotEmpty;
+
+import io.onedev.server.util.validation.annotation.ProjectName;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+public class ProjectMapping implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String PROP_ONEDEV_PROJECT = "oneDevProject";
+
+ private String bitbucketRepo;
+
+ private String oneDevProject;
+
+ @Editable(order=100, name="Bitbucket Repository", description="Specify Bitbucket repository in form of "
+ + "<tt>workspace/repository</tt>")
+ @NotEmpty
+ public String getBitbucketRepo() {
+ return bitbucketRepo;
+ }
+
+ public void setBitbucketRepo(String bitbucketRepo) {
+ this.bitbucketRepo = bitbucketRepo;
+ }
+
+ @Editable(order=200, name="OneDev Project", description="Specify OneDev project to be created as "
+ + "result of importing")
+ @ProjectName
+ @NotEmpty
+ public String getOneDevProject() {
+ return oneDevProject;
+ }
+
+ public void setOneDevProject(String oneDevProject) {
+ this.oneDevProject = oneDevProject;
+ }
+
+}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java
index df2ccfb803..2b34c53c40 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java
@@ -65,7 +65,7 @@ public class ImportServer implements Serializable, Validatable {
this.apiUrl = apiUrl;
}
- @Editable(order=200, name="YouTrack User Name", description="Specify YouTrack user name. This account should have permission to:"
+ @Editable(order=200, name="YouTrack Login Name", description="Specify YouTrack login name. This account should have permission to:"
+ "<ul>"
+ "<li>Read full information and issues of the projects you want to import"
+ "<li>Read issue tags"
diff --git a/server-product/pom.xml b/server-product/pom.xml
index ddcb480de2..680529e52d 100644
--- a/server-product/pom.xml
+++ b/server-product/pom.xml
@@ -90,6 +90,11 @@
<artifactId>server-plugin-import-youtrack</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>io.onedev</groupId>
+ <artifactId>server-plugin-import-bitbucketcloud</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
<properties>
<executables>bin/*.sh, boot/wrapper-*</executables>