The Unified and Autonomous Development Platform
Fix issue #370 - Import code and issues from Bitbucket Cloud
Robin Shen9f5140229c61
diff --git a/server-plugin/pom.xml b/server-plugin/pom.xmlindex 80c4adaba7..caa9889168 100644--- a/server-plugin/pom.xml+++ b/server-plugin/pom.xml@@ -32,5 +32,6 @@<module>server-plugin-import-youtrack</module><module>server-plugin-import-gitlab</module><module>server-plugin-import-gitea</module>+ <module>server-plugin-import-bitbucketcloud</module></modules></project>\ No newline at end of filediff --git a/server-plugin/server-plugin-import-bitbucketcloud/pom.xml b/server-plugin/server-plugin-import-bitbucketcloud/pom.xmlnew file mode 100644index 0000000000..0b2597958f--- /dev/null+++ b/server-plugin/server-plugin-import-bitbucketcloud/pom.xml@@ -0,0 +1,13 @@+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">+ <modelVersion>4.0.0</modelVersion>+ <artifactId>server-plugin-import-bitbucketcloud</artifactId>+ <parent>+ <groupId>io.onedev</groupId>+ <artifactId>server-plugin</artifactId>+ <version>4.8.1</version>+ </parent>+ <properties>+ <moduleClass>io.onedev.server.plugin.imports.bitbucketcloud.BitbucketPluginModule</moduleClass>+ </properties>+</project>diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.javanew file mode 100644index 0000000000..cc46d19406--- /dev/null+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.java@@ -0,0 +1,39 @@+package io.onedev.server.plugin.imports.bitbucketcloud;++import java.io.Serializable;+import java.util.Collection;++import com.google.common.collect.Lists;++import io.onedev.commons.launcher.loader.AbstractPluginModule;+import io.onedev.server.imports.ProjectImporter;+import io.onedev.server.imports.ProjectImporterContribution;++/**+ * NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.+ *+ */++public class BitbucketPluginModule extends AbstractPluginModule {+ @Override+ protected void configure() {+ super.configure();++ // put your guice bindings here+ contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {++ @Override+ public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {+ return Lists.newArrayList(new BitbucketProjectImporter());+ }++ @Override+ public int getOrder() {+ return 250;+ }++ });++ }++}diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.javanew file mode 100644index 0000000000..b55c8f01f8--- /dev/null+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java@@ -0,0 +1,102 @@+package io.onedev.server.plugin.imports.bitbucketcloud;++import java.util.ArrayList;+import java.util.Collection;++import javax.ws.rs.client.Client;++import org.apache.http.client.utils.URIBuilder;++import com.fasterxml.jackson.databind.JsonNode;++import io.onedev.commons.utils.ExplicitException;+import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.imports.ProjectImporter;+import io.onedev.server.model.Project;+import io.onedev.server.storage.StorageManager;+import io.onedev.server.util.SimpleLogger;++public class BitbucketProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ProjectImportOption> {++ private static final long serialVersionUID = 1L;++ @Override+ public String getName() {+ return ImportUtils.NAME;+ }++ @Override+ public String doImport(ImportServer where, ProjectImportSource what, ProjectImportOption how,+ boolean dryRun, SimpleLogger logger) {+ Collection<Long> projectIds = new ArrayList<>();+ Client client = where.newClient();+ try {+ for (ProjectMapping projectMapping: what.getProjectMappings()) {+ logger.log("Cloning code from repository " + projectMapping.getBitbucketRepo() + "...");++ String apiEndpoint = where.getApiEndpoint("/repositories/" + projectMapping.getBitbucketRepo());+ JsonNode repoNode = ImportUtils.get(client, apiEndpoint, logger);+ Project project = new Project();+ project.setName(projectMapping.getOneDevProject());+ project.setDescription(repoNode.get("description").asText(null));++ boolean isPrivate = repoNode.get("is_private").asBoolean();+ if (!isPrivate && how.getPublicRole() != null)+ project.setDefaultRole(how.getPublicRole());++ String cloneUrl = null;+ for (JsonNode cloneNode: repoNode.get("links").get("clone")) {+ if (cloneNode.get("name").asText().equals("https")) {+ cloneUrl = cloneNode.get("href").asText();+ break;+ }+ }+ if (cloneUrl == null)+ throw new ExplicitException("Https clone url not found");++ URIBuilder builder = new URIBuilder(cloneUrl);+ if (isPrivate)+ builder.setUserInfo(where.getUserName(), where.getAppPassword());++ if (!dryRun) {+ OneDev.getInstance(ProjectManager.class).clone(project, builder.build().toString());+ projectIds.add(project.getId());+ }+ }++ return "Repositories imported successfully";+ } catch (Exception e) {+ for (Long projectId: projectIds)+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);+ throw new RuntimeException(e);+ } finally {+ client.close();+ }+ }++ @Override+ public ProjectImportSource getWhat(ImportServer where, SimpleLogger logger) {+ ProjectImportSource importSource = new ProjectImportSource();+ Client client = where.newClient();+ try {+ String apiEndpoint = where.getApiEndpoint("/repositories?role=member");+ for (JsonNode repoNode: ImportUtils.list(client, apiEndpoint, logger)) {+ String fullName = repoNode.get("full_name").asText();+ ProjectMapping projectMapping = new ProjectMapping();+ projectMapping.setBitbucketRepo(fullName);+ projectMapping.setOneDevProject(fullName.replace("/", "-"));+ importSource.getProjectMappings().add(projectMapping);+ }+ } finally {+ client.close();+ }+ return importSource;+ }++ @Override+ public ProjectImportOption getHow(ImportServer where, ProjectImportSource what, SimpleLogger logger) {+ return new ProjectImportOption();+ }++}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.javanew file mode 100644index 0000000000..53f35453b7--- /dev/null+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java@@ -0,0 +1,112 @@+package io.onedev.server.plugin.imports.bitbucketcloud;++import java.io.Serializable;++import javax.validation.ConstraintValidatorContext;+import javax.ws.rs.client.Client;+import javax.ws.rs.client.ClientBuilder;+import javax.ws.rs.client.Invocation;+import javax.ws.rs.client.WebTarget;+import javax.ws.rs.core.Response;++import org.apache.wicket.MetaDataKey;+import org.glassfish.jersey.client.ClientProperties;+import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;+import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.commons.utils.StringUtils;+import io.onedev.server.util.JerseyUtils;+import io.onedev.server.util.validation.Validatable;+import io.onedev.server.util.validation.annotation.ClassValidating;+import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.Password;++@Editable+@ClassValidating+public class ImportServer implements Serializable, Validatable {++ private static final long serialVersionUID = 1L;++ static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {++ private static final long serialVersionUID = 1L;++ };++ static final String PROP_APP_PASSWORD = "appPassword";++ private String userName;++ private String appPassword;++ @Editable(order=10, name="Bitbucket Login Name")+ @NotEmpty+ public String getUserName() {+ return userName;+ }++ public void setUserName(String userName) {+ this.userName = userName;+ }++ @Editable(order=100, name="Bitbucket App Password", description="Bitbucket app password should be generated with "+ + "permission <b>account/read</b>, <b>repositories/read</b> and <b>issues:read</b>")+ @Password+ @NotEmpty+ public String getAppPassword() {+ return appPassword;+ }++ public void setAppPassword(String appPassword) {+ this.appPassword = appPassword;+ }++ public String getApiEndpoint(String apiPath) {+ return "https://api.bitbucket.org/2.0/" + StringUtils.stripStart(apiPath, "/");+ }++ @Override+ public boolean isValid(ConstraintValidatorContext context) {+ Client client = ClientBuilder.newClient();+ client.register(HttpAuthenticationFeature.basic(getUserName(), getAppPassword()));+ try {+ String apiEndpoint = getApiEndpoint("/user");+ WebTarget target = client.target(apiEndpoint);+ Invocation.Builder builder = target.request();+ try (Response response = builder.get()) {+ if (response.getStatus() == 401) {+ context.disableDefaultConstraintViolation();+ String errorMessage = "Authentication failed";+ context.buildConstraintViolationWithTemplate(errorMessage)+ .addPropertyNode(PROP_APP_PASSWORD).addConstraintViolation();+ return false;+ } else {+ String errorMessage = JerseyUtils.checkStatus(apiEndpoint, response);+ if (errorMessage != null) {+ context.disableDefaultConstraintViolation();+ context.buildConstraintViolationWithTemplate(errorMessage).addConstraintViolation();+ return false;+ }+ }+ }+ } catch (Exception e) {+ context.disableDefaultConstraintViolation();+ String errorMessage = "Error connecting api service";+ if (e.getMessage() != null)+ errorMessage += ": " + e.getMessage();+ context.buildConstraintViolationWithTemplate(errorMessage).addConstraintViolation();+ return false;+ } finally {+ client.close();+ }+ return true;+ }++ Client newClient() {+ Client client = ClientBuilder.newClient();+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);+ client.register(HttpAuthenticationFeature.basic(getUserName(), getAppPassword()));+ return client;+ }++}diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportUtils.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportUtils.javanew file mode 100644index 0000000000..5b9cb58bea--- /dev/null+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportUtils.java@@ -0,0 +1,95 @@+package io.onedev.server.plugin.imports.bitbucketcloud;++import java.net.URI;+import java.net.URISyntaxException;+import java.util.ArrayList;+import java.util.List;++import javax.ws.rs.client.Client;+import javax.ws.rs.client.Invocation;+import javax.ws.rs.client.WebTarget;+import javax.ws.rs.core.Response;++import org.apache.commons.lang3.StringUtils;+import org.apache.http.client.utils.URIBuilder;++import com.fasterxml.jackson.databind.JsonNode;++import io.onedev.commons.utils.ExplicitException;+import io.onedev.server.util.SimpleLogger;++public class ImportUtils {++ static final String NAME = "Bitbucket Cloud";++ static final int PER_PAGE = 50;++ static List<JsonNode> list(Client client, String apiEndpoint, SimpleLogger logger) {+ List<JsonNode> result = new ArrayList<>();+ list(client, apiEndpoint, new PageDataConsumer() {++ @Override+ public void consume(List<JsonNode> pageData) {+ result.addAll(pageData);+ }++ }, logger);+ return result;+ }++ static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,+ SimpleLogger logger) {+ URI uri;+ try {+ uri = new URIBuilder(apiEndpoint)+ .addParameter("pagelen", String.valueOf(PER_PAGE)).build();+ } catch (URISyntaxException e) {+ throw new RuntimeException(e);+ }++ while (true) {+ try {+ List<JsonNode> pageData = new ArrayList<>();+ JsonNode resultNode = get(client, uri.toString(), logger);+ if (resultNode.hasNonNull("values")) {+ for (JsonNode each: resultNode.get("values"))+ pageData.add(each);+ pageDataConsumer.consume(pageData);+ }+ if (resultNode.hasNonNull("next"))+ uri = new URIBuilder(resultNode.get("next").asText()).build();+ else+ break;+ } catch (URISyntaxException | InterruptedException e) {+ throw new RuntimeException(e);+ }+ }+ }++ static JsonNode get(Client client, String apiEndpoint, SimpleLogger logger) {+ WebTarget target = client.target(apiEndpoint);+ Invocation.Builder builder = target.request();+ while (true) {+ try (Response response = builder.get()) {+ int status = response.getStatus();+ if (status != 200) {+ String errorMessage = response.readEntity(String.class);+ if (StringUtils.isNotBlank(errorMessage)) {+ throw new ExplicitException(String.format("Http request failed (url: %s, status code: %d, error message: %s)",+ apiEndpoint, status, errorMessage));+ } else {+ throw new ExplicitException(String.format("Http request failed (status: %s)", status));+ }+ }+ return response.readEntity(JsonNode.class);+ }+ }+ }++ static interface PageDataConsumer {++ void consume(List<JsonNode> pageData) throws InterruptedException;++ }++}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportOption.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportOption.javanew file mode 100644index 0000000000..44a12ce103--- /dev/null+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportOption.java@@ -0,0 +1,40 @@+package io.onedev.server.plugin.imports.bitbucketcloud;++import java.io.Serializable;++import javax.annotation.Nullable;++import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.RoleManager;+import io.onedev.server.model.Role;+import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.RoleChoice;++@Editable+public class ProjectImportOption implements Serializable {++ private static final long serialVersionUID = 1L;++ private String publicRoleName;++ @Editable(order=100, name="Public Role", description="If specified, all public repositories imported from GitHub "+ + "will use this as default role. Private repositories are not affected")+ @RoleChoice+ @Nullable+ public String getPublicRoleName() {+ return publicRoleName;+ }++ public void setPublicRoleName(String publicRoleName) {+ this.publicRoleName = publicRoleName;+ }++ @Nullable+ public Role getPublicRole() {+ if (publicRoleName != null)+ return OneDev.getInstance(RoleManager.class).find(publicRoleName);+ else+ return null;+ }++}diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportSource.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportSource.javanew file mode 100644index 0000000000..318382b776--- /dev/null+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportSource.java@@ -0,0 +1,64 @@+package io.onedev.server.plugin.imports.bitbucketcloud;++import java.io.Serializable;+import java.util.ArrayList;+import java.util.List;++import javax.validation.ConstraintValidatorContext;+import javax.validation.constraints.Size;++import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.util.validation.Validatable;+import io.onedev.server.util.validation.annotation.ClassValidating;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+@ClassValidating+public class ProjectImportSource implements Serializable, Validatable {++ private static final long serialVersionUID = 1L;++ private List<ProjectMapping> projectMappings = new ArrayList<>();++ @Editable(order=100, name="Repositories to Import")+ @Size(min=1, max=10000, message="No repositories to import")+ public List<ProjectMapping> getProjectMappings() {+ return projectMappings;+ }++ public void setProjectMappings(List<ProjectMapping> projectMappings) {+ this.projectMappings = projectMappings;+ }++ @Override+ public boolean isValid(ConstraintValidatorContext context) {+ boolean isValid = true;+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);+ for (int i=0; i<projectMappings.size(); i++) {+ if (projectManager.find(projectMappings.get(i).getOneDevProject()) != null) {+ context.buildConstraintViolationWithTemplate("Project name already used")+ .addPropertyNode("projectMappings")+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)+ .inIterable().atIndex(i).addConstraintViolation();+ isValid = false;+ } else {+ for (int j=0; j<projectMappings.size(); j++) {+ if (j != i && projectMappings.get(j).getOneDevProject().equals(projectMappings.get(i).getOneDevProject())) {+ context.buildConstraintViolationWithTemplate("Duplicate project name")+ .addPropertyNode("projectMappings")+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)+ .inIterable().atIndex(i).addConstraintViolation();+ isValid = false;+ break;+ }+ }+ }+ }++ if (!isValid)+ context.disableDefaultConstraintViolation();+ return isValid;+ }++}diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectMapping.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectMapping.javanew file mode 100644index 0000000000..185a7d51e3--- /dev/null+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectMapping.java@@ -0,0 +1,44 @@+package io.onedev.server.plugin.imports.bitbucketcloud;++import java.io.Serializable;++import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.server.util.validation.annotation.ProjectName;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public class ProjectMapping implements Serializable {++ private static final long serialVersionUID = 1L;++ public static final String PROP_ONEDEV_PROJECT = "oneDevProject";++ private String bitbucketRepo;++ private String oneDevProject;++ @Editable(order=100, name="Bitbucket Repository", description="Specify Bitbucket repository in form of "+ + "<tt>workspace/repository</tt>")+ @NotEmpty+ public String getBitbucketRepo() {+ return bitbucketRepo;+ }++ public void setBitbucketRepo(String bitbucketRepo) {+ this.bitbucketRepo = bitbucketRepo;+ }++ @Editable(order=200, name="OneDev Project", description="Specify OneDev project to be created as "+ + "result of importing")+ @ProjectName+ @NotEmpty+ public String getOneDevProject() {+ return oneDevProject;+ }++ public void setOneDevProject(String oneDevProject) {+ this.oneDevProject = oneDevProject;+ }++}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.javaindex df2ccfb803..2b34c53c40 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java@@ -65,7 +65,7 @@ public class ImportServer implements Serializable, Validatable {this.apiUrl = apiUrl;}- @Editable(order=200, name="YouTrack User Name", description="Specify YouTrack user name. This account should have permission to:"+ @Editable(order=200, name="YouTrack Login Name", description="Specify YouTrack login name. This account should have permission to:"+ "<ul>"+ "<li>Read full information and issues of the projects you want to import"+ "<li>Read issue tags"diff --git a/server-product/pom.xml b/server-product/pom.xmlindex ddcb480de2..680529e52d 100644--- a/server-product/pom.xml+++ b/server-product/pom.xml@@ -90,6 +90,11 @@<artifactId>server-plugin-import-youtrack</artifactId><version>${project.version}</version></dependency>+ <dependency>+ <groupId>io.onedev</groupId>+ <artifactId>server-plugin-import-bitbucketcloud</artifactId>+ <version>${project.version}</version>+ </dependency></dependencies><properties><executables>bin/*.sh, boot/wrapper-*</executables>