Skip to content

The Unified and Autonomous Development Platform

Fix issue #934 - Improve importing from multiple sources

Robin Shence7ea6daadee
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java
index 82a118adaa..95f302f31b 100644
--- a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java
+++ b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java
@@ -311,7 +311,7 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
updateManager.save(update);
project.setUpdate(update);
dao.persist(project);
- FileUtils.createDir(storageManager.getProjectDir(project.getId()));
+ FileUtils.cleanDir(storageManager.getProjectDir(project.getId()));
checkGitDir(project.getId());
checkGitHooksAndConfig(project.getId());
UserAuthorization authorization = new UserAuthorization();
diff --git a/server-core/src/main/java/io/onedev/server/imports/IssueImporter.java b/server-core/src/main/java/io/onedev/server/imports/IssueImporter.java
index 4b51085646..02acb0dd9a 100644
--- a/server-core/src/main/java/io/onedev/server/imports/IssueImporter.java
+++ b/server-core/src/main/java/io/onedev/server/imports/IssueImporter.java
@@ -16,6 +16,6 @@ public interface IssueImporter extends Serializable {
List<ImportStep<? extends Serializable>> getSteps();
@Nullable
- public abstract String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger);
+ public abstract String doImport(Project project, boolean dryRun, TaskLogger logger);
}
diff --git a/server-core/src/main/java/io/onedev/server/storage/DefaultStorageManager.java b/server-core/src/main/java/io/onedev/server/storage/DefaultStorageManager.java
index f7b6dbc532..19c9f19d1b 100644
--- a/server-core/src/main/java/io/onedev/server/storage/DefaultStorageManager.java
+++ b/server-core/src/main/java/io/onedev/server/storage/DefaultStorageManager.java
@@ -216,13 +216,4 @@ public class DefaultStorageManager implements StorageManager, Serializable {
}
}
- @Override
- public void deleteProjectDir(Long projectId) {
- try {
- new File(getProjectDir(projectId), DELETE_MARK).createNewFile();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
}
diff --git a/server-core/src/main/java/io/onedev/server/storage/StorageManager.java b/server-core/src/main/java/io/onedev/server/storage/StorageManager.java
index 5e4438a6ed..2e5098b94e 100644
--- a/server-core/src/main/java/io/onedev/server/storage/StorageManager.java
+++ b/server-core/src/main/java/io/onedev/server/storage/StorageManager.java
@@ -62,6 +62,5 @@ public interface StorageManager {
* will be exist after calling this method
*/
File getBuildDir(Long projectId, Long buildNumber);
-
- void deleteProjectDir(Long projectId);
+
}
diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java
index 8521644efb..38090bc05b 100644
--- a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java
+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java
@@ -88,7 +88,7 @@ public class IssueImportPage<Where extends Serializable, What extends Serializab
@Override
public String call() throws Exception {
- return importer.doImport(getProject(), true, false, logger);
+ return importer.doImport(getProject(), false, logger);
}
});
@@ -115,7 +115,7 @@ public class IssueImportPage<Where extends Serializable, What extends Serializab
@Override
public String call() throws Exception {
- return importer.doImport(getProject(), true, true, logger);
+ return importer.doImport(getProject(), true, logger);
}
});
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java
index 7f1936903f..f7558e042d 100644
--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java
@@ -60,7 +60,8 @@ public class BitbucketProjectImporter implements ProjectImporter {
protected ImportRepositories newSetting() {
ImportRepositories repositories = new ImportRepositories();
String workspace = workspaceStep.getSetting().getWorkspace();
- for (String repository: serverStep.getSetting().listRepositories(workspace)) {
+ for (String repository: serverStep.getSetting().listRepositories(
+ workspace, workspaceStep.getSetting().isIncludeForks())) {
ProjectMapping projectMapping = new ProjectMapping();
projectMapping.setBitbucketRepo(repository);
projectMapping.setOneDevProject(repository);
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportRepositories.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportRepositories.java
index efb67f89e0..469e800e9b 100644
--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportRepositories.java
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportRepositories.java
@@ -12,6 +12,7 @@ import org.apache.shiro.authz.UnauthorizedException;
import io.onedev.server.OneDev;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.model.Project;
+import io.onedev.server.security.SecurityUtils;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@@ -41,21 +42,10 @@ public class ImportRepositories implements Serializable, Validatable {
boolean isValid = true;
ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
for (int i=0; i<projectMappings.size(); i++) {
- String errorMessage = null;
try {
Project project = projectManager.setup(projectMappings.get(i).getOneDevProject());
- if (!project.isNew())
- errorMessage = "Project already exists";
- } catch (UnauthorizedException e) {
- errorMessage = e.getMessage();
- }
- if (errorMessage != null) {
- context.buildConstraintViolationWithTemplate(errorMessage)
- .addPropertyNode("projectMappings")
- .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
- .inIterable().atIndex(i).addConstraintViolation();
- isValid = false;
- } else {
+ if (!project.isNew() && !SecurityUtils.canManage(project))
+ throw new UnauthorizedException("Project management permission is required");
for (int j=0; j<projectMappings.size(); j++) {
if (j != i && projectMappings.get(j).getOneDevProject().equals(projectMappings.get(i).getOneDevProject())) {
context.buildConstraintViolationWithTemplate("Duplicate project")
@@ -66,6 +56,12 @@ public class ImportRepositories implements Serializable, Validatable {
break;
}
}
+ } catch (UnauthorizedException e) {
+ context.buildConstraintViolationWithTemplate(e.getMessage())
+ .addPropertyNode("projectMappings")
+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
+ .inIterable().atIndex(i).addConstraintViolation();
+ isValid = false;
}
}
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java
index 94c590e2fe..93a824d5fb 100644
--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java
@@ -4,12 +4,12 @@ import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.validation.ConstraintValidatorContext;
+import javax.validation.constraints.NotEmpty;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Invocation;
@@ -19,12 +19,10 @@ import javax.ws.rs.core.Response;
import org.apache.http.client.utils.URIBuilder;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
-import javax.validation.constraints.NotEmpty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Preconditions;
import io.onedev.commons.bootstrap.SensitiveMasker;
import io.onedev.commons.utils.ExplicitException;
@@ -34,7 +32,6 @@ import io.onedev.server.OneDev;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.git.command.LsRemoteCommand;
import io.onedev.server.model.Project;
-import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.CollectionUtils;
import io.onedev.server.util.JerseyUtils;
import io.onedev.server.util.JerseyUtils.PageDataConsumer;
@@ -110,7 +107,7 @@ public class ImportServer implements Serializable, Validatable {
return workspaces;
}
- List<String> listRepositories(String workspaceId) {
+ List<String> listRepositories(String workspaceId, boolean includeForks) {
Client client = newClient();
try {
List<String> repositories = new ArrayList<>();
@@ -124,7 +121,8 @@ public class ImportServer implements Serializable, Validatable {
}
})) {
- repositories.add(repoNode.get("full_name").asText());
+ if (includeForks || repoNode.get("parent") == null)
+ repositories.add(repoNode.get("full_name").asText());
}
return repositories;
@@ -176,18 +174,13 @@ public class ImportServer implements Serializable, Validatable {
}
String importProjects(ImportRepositories repositories, ImportOption option, boolean dryRun, TaskLogger logger) {
- Collection<Long> projectIds = new ArrayList<>();
-
Client client = newClient();
try {
for (ProjectMapping projectMapping: repositories.getProjectMappings()) {
- logger.log("Cloning code from repository " + projectMapping.getBitbucketRepo() + "...");
-
String apiEndpoint = getApiEndpoint("/repositories/" + projectMapping.getBitbucketRepo());
JsonNode repoNode = JerseyUtils.get(client, apiEndpoint, logger);
ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
Project project = projectManager.setup(projectMapping.getOneDevProject());
- Preconditions.checkState(project.isNew());
project.setDescription(repoNode.get("description").asText(null));
@@ -195,43 +188,48 @@ public class ImportServer implements Serializable, Validatable {
if (!isPrivate && option.getPublicRole() != null)
project.setDefaultRole(option.getPublicRole());
- String cloneUrl = null;
- for (JsonNode cloneNode: repoNode.get("links").get("clone")) {
- if (cloneNode.get("name").asText().equals("https")) {
- cloneUrl = cloneNode.get("href").asText();
- break;
- }
- }
- if (cloneUrl == null)
- throw new ExplicitException("Https clone url not found");
-
- URIBuilder builder = new URIBuilder(cloneUrl);
- builder.setUserInfo(getUserName(), getAppPassword());
-
- SensitiveMasker.push(new SensitiveMasker() {
-
- @Override
- public String mask(String text) {
- return StringUtils.replace(text, getAppPassword(), "******");
+ if (project.isNew() || project.getDefaultBranch() == null) {
+ logger.log("Cloning code from repository " + projectMapping.getBitbucketRepo() + "...");
+
+ String cloneUrl = null;
+ for (JsonNode cloneNode: repoNode.get("links").get("clone")) {
+ if (cloneNode.get("name").asText().equals("https")) {
+ cloneUrl = cloneNode.get("href").asText();
+ break;
+ }
}
+ if (cloneUrl == null)
+ throw new ExplicitException("Https clone url not found");
- });
- try {
- if (dryRun) {
- new LsRemoteCommand(builder.build().toString()).refs("HEAD").quiet(true).run();
- } else {
- projectManager.clone(project, builder.build().toString());
- projectIds.add(project.getId());
+ URIBuilder builder = new URIBuilder(cloneUrl);
+ builder.setUserInfo(getUserName(), getAppPassword());
+
+ SensitiveMasker.push(new SensitiveMasker() {
+
+ @Override
+ public String mask(String text) {
+ return StringUtils.replace(text, getAppPassword(), "******");
+ }
+
+ });
+ try {
+ if (dryRun) {
+ new LsRemoteCommand(builder.build().toString()).refs("HEAD").quiet(true).run();
+ } else {
+ if (project.isNew())
+ projectManager.create(project);
+ projectManager.clone(project, builder.build().toString());
+ }
+ } finally {
+ SensitiveMasker.pop();
}
- } finally {
- SensitiveMasker.pop();
+ } else {
+ logger.warning("Skipping code clone as the project already has code");
}
}
return "Repositories imported successfully";
- } catch (Exception e) {
- for (Long projectId: projectIds)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ } catch (URISyntaxException e) {
throw new RuntimeException(e);
} finally {
client.close();
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportWorkspace.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportWorkspace.java
index d2325bd851..40aa6ace1a 100644
--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportWorkspace.java
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportWorkspace.java
@@ -18,6 +18,8 @@ public class ImportWorkspace implements Serializable {
ImportServer server;
private String workspace;
+
+ private boolean includeForks;
@Editable(order=100, name="Bitbucket Workspace", description="Select workspace to import from")
@ChoiceProvider("getWorkspaceChoices")
@@ -37,4 +39,13 @@ public class ImportWorkspace implements Serializable {
return setting.server.listWorkspaces();
}
+ @Editable(order=200, description="Whether or not to include forked repositories")
+ public boolean isIncludeForks() {
+ return includeForks;
+ }
+
+ public void setIncludeForks(boolean includeForks) {
+ this.includeForks = includeForks;
+ }
+
}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaIssueImporter.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaIssueImporter.java
index cbe4ad3509..ea80825acd 100644
--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaIssueImporter.java
+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaIssueImporter.java
@@ -75,7 +75,7 @@ public class GiteaIssueImporter implements IssueImporter {
}
@Override
- public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ public String doImport(Project project, boolean dryRun, TaskLogger logger) {
ImportServer server = serverStep.getSetting();
String giteaRepo = repositoryStep.getSetting().getRepository();
IssueImportOption option = optionStep.getSetting();
@@ -83,7 +83,7 @@ public class GiteaIssueImporter implements IssueImporter {
logger.log("Importing issues from repository " + giteaRepo + "...");
Map<String, Optional<User>> users = new HashMap<>();
- return server.importIssues(giteaRepo, project, retainIssueNumbers, option, users, dryRun, logger)
+ return server.importIssues(giteaRepo, project, option, users, dryRun, logger)
.toHtml("Issues imported successfully");
}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaProjectImporter.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaProjectImporter.java
index 5dd2f9b6d4..169be153ba 100644
--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaProjectImporter.java
+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaProjectImporter.java
@@ -61,7 +61,8 @@ public class GiteaProjectImporter implements ProjectImporter {
protected ImportRepositories newSetting() {
ImportRepositories repositories = new ImportRepositories();
String organization = organizationStep.getSetting().getOrganization();
- for (String repository: serverStep.getSetting().listRepositories(organization)) {
+ for (String repository: serverStep.getSetting().listRepositories(
+ organization, organizationStep.getSetting().isIncludeForks())) {
ProjectMapping projectMapping = new ProjectMapping();
projectMapping.setGiteaRepo(repository);
projectMapping.setOneDevProject(repository);
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportOrganization.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportOrganization.java
index 3f754db584..144e5bff58 100644
--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportOrganization.java
+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportOrganization.java
@@ -16,6 +16,8 @@ public class ImportOrganization implements Serializable {
ImportServer server;
private String organization;
+
+ private boolean includeForks;
@Editable(order=100, name="Gitea Organization", description="Select organization to import from. "
+ "Leave empty to import from repositories under current account")
@@ -35,4 +37,13 @@ public class ImportOrganization implements Serializable {
return setting.server.listOrganizations();
}
+ @Editable(order=200, description="Whether or not to include forked repositories")
+ public boolean isIncludeForks() {
+ return includeForks;
+ }
+
+ public void setIncludeForks(boolean includeForks) {
+ this.includeForks = includeForks;
+ }
+
}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepositories.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepositories.java
index 058e3397a3..2a7166129c 100644
--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepositories.java
+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepositories.java
@@ -12,6 +12,7 @@ import org.apache.shiro.authz.UnauthorizedException;
import io.onedev.server.OneDev;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.model.Project;
+import io.onedev.server.security.SecurityUtils;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@@ -41,21 +42,10 @@ public class ImportRepositories implements Serializable, Validatable {
boolean isValid = true;
ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
for (int i=0; i<projectMappings.size(); i++) {
- String errorMessage = null;
try {
Project project = projectManager.setup(projectMappings.get(i).getOneDevProject());
- if (!project.isNew())
- errorMessage = "Project already exists";
- } catch (UnauthorizedException e) {
- errorMessage = e.getMessage();
- }
- if (errorMessage != null) {
- context.buildConstraintViolationWithTemplate(errorMessage)
- .addPropertyNode("projectMappings")
- .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
- .inIterable().atIndex(i).addConstraintViolation();
- isValid = false;
- } else {
+ if (!project.isNew() && !SecurityUtils.canManage(project))
+ throw new UnauthorizedException("Project management permission is required");
for (int j=0; j<projectMappings.size(); j++) {
if (j != i && projectMappings.get(j).getOneDevProject().equals(projectMappings.get(i).getOneDevProject())) {
context.buildConstraintViolationWithTemplate("Duplicate project")
@@ -66,6 +56,12 @@ public class ImportRepositories implements Serializable, Validatable {
break;
}
}
+ } catch (UnauthorizedException e) {
+ context.buildConstraintViolationWithTemplate(e.getMessage())
+ .addPropertyNode("projectMappings")
+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
+ .inIterable().atIndex(i).addConstraintViolation();
+ isValid = false;
}
}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepository.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepository.java
index 547b4ff760..76255e43b6 100644
--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepository.java
+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepository.java
@@ -33,7 +33,12 @@ public class ImportRepository extends ImportOrganization {
BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);
ImportRepository setting = (ImportRepository) editor.getModelObject();
String organization = (String) EditContext.get().getInputValue("organization");
- return setting.server.listRepositories(organization);
+ return setting.server.listRepositories(organization, true);
+ }
+
+ @Override
+ public boolean isIncludeForks() {
+ return super.isIncludeForks();
}
}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportServer.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportServer.java
index 4bf44a848d..caba90347e 100644
--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportServer.java
+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportServer.java
@@ -20,6 +20,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Nullable;
import javax.validation.ConstraintValidatorContext;
+import javax.validation.constraints.NotEmpty;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Invocation;
@@ -29,14 +30,12 @@ import javax.ws.rs.core.Response;
import org.apache.http.client.utils.URIBuilder;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.client.oauth2.OAuth2ClientSupport;
-import javax.validation.constraints.NotEmpty;
import org.joda.time.format.ISODateTimeFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.unbescape.html.HtmlEscape;
import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Preconditions;
import edu.emory.mathcs.backport.java.util.Collections;
import io.onedev.commons.bootstrap.SensitiveMasker;
@@ -63,7 +62,6 @@ import io.onedev.server.model.support.administration.GlobalIssueSetting;
import io.onedev.server.model.support.inputspec.InputSpec;
import io.onedev.server.model.support.issue.field.spec.FieldSpec;
import io.onedev.server.persistence.dao.Dao;
-import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.DateUtils;
import io.onedev.server.util.JerseyUtils;
import io.onedev.server.util.JerseyUtils.PageDataConsumer;
@@ -142,7 +140,7 @@ public class ImportServer implements Serializable, Validatable {
return organizations;
}
- List<String> listRepositories(@Nullable String organization) {
+ List<String> listRepositories(@Nullable String organization, boolean includeForks) {
Client client = newClient();
try {
String apiEndpoint;
@@ -163,8 +161,10 @@ public class ImportServer implements Serializable, Validatable {
JsonNode ownerNode = repoNode.get("owner");
String ownerName = ownerNode.get("login").asText();
JsonNode emailNode = ownerNode.get("email");
- if (organization != null || emailNode != null && StringUtils.isNotBlank(emailNode.asText()))
+ if ((organization != null || emailNode != null && StringUtils.isNotBlank(emailNode.asText()))
+ && (includeForks || !repoNode.get("fork").asBoolean())) {
repositories.add(ownerName + "/" + repoName);
+ }
}
Collections.sort(repositories);
return repositories;
@@ -228,8 +228,9 @@ public class ImportServer implements Serializable, Validatable {
return userOpt.orElse(null);
}
- ImportResult importIssues(String giteaRepo, Project oneDevProject, boolean useExistingIssueNumbers,
- IssueImportOption option, Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {
+ ImportResult importIssues(String giteaRepo, Project oneDevProject, IssueImportOption option,
+ Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {
+ IssueManager issueManager = OneDev.getInstance(IssueManager.class);
Client client = newClient();
try {
Set<String> nonExistentMilestones = new HashSet<>();
@@ -282,12 +283,13 @@ public class ImportServer implements Serializable, Validatable {
issue.setNumberScope(oneDevProject.getForkRoot());
- Long oldNumber = issueNode.get("number").asLong();
Long newNumber;
- if (dryRun || useExistingIssueNumbers)
+ Long oldNumber = issueNode.get("number").asLong();
+ if (dryRun || (issueManager.find(oneDevProject, oldNumber) == null && !issueNumberMappings.containsValue(oldNumber)))
newNumber = oldNumber;
else
- newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);
+ newNumber = issueManager.getNextNumber(oneDevProject);
+
issue.setNumber(newNumber);
issueNumberMappings.put(oldNumber, newNumber);
@@ -458,7 +460,7 @@ public class ImportServer implements Serializable, Validatable {
if (issue.getDescription() != null)
issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));
- OneDev.getInstance(IssueManager.class).save(issue);
+ issueManager.save(issue);
for (IssueSchedule schedule: issue.getSchedules())
dao.persist(schedule);
for (IssueField field: issue.getFields())
@@ -480,78 +482,86 @@ public class ImportServer implements Serializable, Validatable {
return result;
} finally {
+ if (!dryRun)
+ issueManager.resetNextNumber(oneDevProject);
client.close();
}
}
- String importProjects(ImportRepositories repositories, ProjectImportOption option, boolean dryRun, TaskLogger logger) {
- Collection<Long> projectIds = new ArrayList<>();
+ String importProjects(ImportRepositories repositories, ProjectImportOption option,
+ boolean dryRun, TaskLogger logger) {
Client client = newClient();
try {
Map<String, Optional<User>> users = new HashMap<>();
ImportResult result = new ImportResult();
for (ProjectMapping projectMapping: repositories.getProjectMappings()) {
- logger.log("Cloning code from repository " + projectMapping.getGiteaRepo() + "...");
-
String apiEndpoint = getApiEndpoint("/repos/" + projectMapping.getGiteaRepo());
JsonNode repoNode = JerseyUtils.get(client, apiEndpoint, logger);
ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
Project project = projectManager.setup(projectMapping.getOneDevProject());
- Preconditions.checkState(project.isNew());
project.setDescription(repoNode.get("description").asText(null));
project.setIssueManagement(repoNode.get("has_issues").asBoolean());
boolean isPrivate = repoNode.get("private").asBoolean();
if (!isPrivate && option.getPublicRole() != null)
project.setDefaultRole(option.getPublicRole());
-
- URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());
- builder.setUserInfo("git", getAccessToken());
-
- SensitiveMasker.push(new SensitiveMasker() {
- @Override
- public String mask(String text) {
- return StringUtils.replace(text, getAccessToken(), "******");
- }
+ if (project.isNew() || project.getDefaultBranch() == null) {
+ logger.log("Cloning code from repository " + projectMapping.getGiteaRepo() + "...");
- });
- try {
- if (dryRun) {
- new LsRemoteCommand(builder.build().toString()).refs("HEAD").quiet(true).run();
- } else {
- projectManager.clone(project, builder.build().toString());
- projectIds.add(project.getId());
+ URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());
+ builder.setUserInfo("git", getAccessToken());
+
+ SensitiveMasker.push(new SensitiveMasker() {
+
+ @Override
+ public String mask(String text) {
+ return StringUtils.replace(text, getAccessToken(), "******");
+ }
+
+ });
+ try {
+ if (dryRun) {
+ new LsRemoteCommand(builder.build().toString()).refs("HEAD").quiet(true).run();
+ } else {
+ if (project.isNew())
+ projectManager.create(project);
+ projectManager.clone(project, builder.build().toString());
+ }
+ } finally {
+ SensitiveMasker.pop();
}
- } finally {
- SensitiveMasker.pop();
+ } else {
+ logger.warning("Skipping code clone as the project already has code");
}
if (option.getIssueImportOption() != null) {
- List<Milestone> milestones = new ArrayList<>();
logger.log("Importing milestones from repository " + projectMapping.getGiteaRepo() + "...");
apiEndpoint = getApiEndpoint("/repos/" + projectMapping.getGiteaRepo() + "/milestones?state=all");
for (JsonNode milestoneNode: list(client, apiEndpoint, logger)) {
- Milestone milestone = new Milestone();
- milestone.setName(milestoneNode.get("title").asText());
- milestone.setDescription(milestoneNode.get("description").asText(null));
- milestone.setProject(project);
- String dueDateString = milestoneNode.get("due_on").asText(null);
- if (dueDateString != null)
- milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());
- if (milestoneNode.get("state").asText().equals("closed"))
- milestone.setClosed(true);
-
- milestones.add(milestone);
- project.getMilestones().add(milestone);
-
- if (!dryRun)
- OneDev.getInstance(MilestoneManager.class).save(milestone);
+ String milestoneName = milestoneNode.get("title").asText();
+ Milestone milestone = project.getMilestone(milestoneName);
+ if (milestone == null) {
+ milestone = new Milestone();
+ milestone.setName(milestoneName);
+ milestone.setDescription(milestoneNode.get("description").asText(null));
+ milestone.setProject(project);
+ String dueDateString = milestoneNode.get("due_on").asText(null);
+ if (dueDateString != null)
+ milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());
+ if (milestoneNode.get("state").asText().equals("closed"))
+ milestone.setClosed(true);
+
+ project.getMilestones().add(milestone);
+
+ if (!dryRun)
+ OneDev.getInstance(MilestoneManager.class).save(milestone);
+ }
}
logger.log("Importing issues from repository " + projectMapping.getGiteaRepo() + "...");
ImportResult currentResult = importIssues(projectMapping.getGiteaRepo(),
- project, true, option.getIssueImportOption(), users, dryRun, logger);
+ project, option.getIssueImportOption(), users, dryRun, logger);
result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);
result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);
@@ -560,9 +570,7 @@ public class ImportServer implements Serializable, Validatable {
}
return result.toHtml("Repositories imported successfully");
- } catch (Exception e) {
- for (Long projectId: projectIds)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ } catch (URISyntaxException e) {
throw new RuntimeException(e);
} finally {
client.close();
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java
index 9770065321..bbc8486b6f 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java
@@ -75,7 +75,7 @@ public class GitHubIssueImporter implements IssueImporter {
}
@Override
- public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ public String doImport(Project project, boolean dryRun, TaskLogger logger) {
logger.log("Importing issues from repository " + repositoryStep.getSetting().getRepository() + "...");
Map<String, Optional<User>> users = new HashMap<>();
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepositories.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepositories.java
index f4be5f810a..c3f523bb14 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepositories.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepositories.java
@@ -42,21 +42,10 @@ public class ImportRepositories implements Serializable, Validatable {
boolean isValid = true;
ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
for (int i=0; i<projectMappings.size(); i++) {
- String errorMessage = null;
try {
Project project = projectManager.setup(projectMappings.get(i).getOneDevProject());
if (!project.isNew() && !SecurityUtils.canManage(project))
throw new UnauthorizedException("Project management permission is required");
- } catch (UnauthorizedException e) {
- errorMessage = e.getMessage();
- }
- if (errorMessage != null) {
- context.buildConstraintViolationWithTemplate(errorMessage)
- .addPropertyNode("projectMappings")
- .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
- .inIterable().atIndex(i).addConstraintViolation();
- isValid = false;
- } else {
for (int j=0; j<projectMappings.size(); j++) {
if (j != i && projectMappings.get(j).getOneDevProject().equals(projectMappings.get(i).getOneDevProject())) {
context.buildConstraintViolationWithTemplate("Duplicate project")
@@ -67,6 +56,12 @@ public class ImportRepositories implements Serializable, Validatable {
break;
}
}
+ } catch (UnauthorizedException e) {
+ context.buildConstraintViolationWithTemplate(e.getMessage())
+ .addPropertyNode("projectMappings")
+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
+ .inIterable().atIndex(i).addConstraintViolation();
+ isValid = false;
}
}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportServer.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportServer.java
index beb044eb68..2e6e7730f1 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportServer.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportServer.java
@@ -59,7 +59,6 @@ import io.onedev.server.model.support.administration.GlobalIssueSetting;
import io.onedev.server.model.support.inputspec.InputSpec;
import io.onedev.server.model.support.issue.field.spec.FieldSpec;
import io.onedev.server.persistence.dao.Dao;
-import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.JerseyUtils;
import io.onedev.server.util.JerseyUtils.PageDataConsumer;
import io.onedev.server.util.Pair;
@@ -280,10 +279,11 @@ public class ImportServer implements Serializable, Validatable {
Long newNumber;
Long oldNumber = issueNode.get("number").asLong();
- if (dryRun || issueManager.find(oneDevProject, oldNumber) == null)
+ if (dryRun || (issueManager.find(oneDevProject, oldNumber) == null && !issueNumberMappings.containsValue(oldNumber)))
newNumber = oldNumber;
else
newNumber = issueManager.getNextNumber(oneDevProject);
+
issue.setNumber(newNumber);
issueNumberMappings.put(oldNumber, newNumber);
@@ -536,7 +536,6 @@ public class ImportServer implements Serializable, Validatable {
}
String importProjects(ImportRepositories repositories, ProjectImportOption option, boolean dryRun, TaskLogger logger) {
- Collection<Long> projectIdsToDelete = new ArrayList<>();
Client client = newClient();
try {
Map<String, Optional<User>> users = new HashMap<>();
@@ -570,11 +569,9 @@ public class ImportServer implements Serializable, Validatable {
try {
if (dryRun) {
new LsRemoteCommand(builder.build().toString()).refs("HEAD").quiet(true).run();
- } else if (project.isNew()) {
- projectManager.create(project);
- projectManager.clone(project, builder.build().toString());
- projectIdsToDelete.add(project.getId());
} else {
+ if (project.isNew())
+ projectManager.create(project);
projectManager.clone(project, builder.build().toString());
}
} finally {
@@ -585,7 +582,6 @@ public class ImportServer implements Serializable, Validatable {
}
if (option.getIssueImportOption() != null) {
- List<Milestone> milestones = new ArrayList<>();
logger.log("Importing milestones from repository " + projectMapping.getGitHubRepo() + "...");
apiEndpoint = getApiEndpoint("/repos/" + projectMapping.getGitHubRepo() + "/milestones?state=all");
for (JsonNode milestoneNode: list(client, apiEndpoint, logger)) {
@@ -602,7 +598,6 @@ public class ImportServer implements Serializable, Validatable {
if (milestoneNode.get("state").asText().equals("closed"))
milestone.setClosed(true);
- milestones.add(milestone);
project.getMilestones().add(milestone);
if (!dryRun)
@@ -622,9 +617,7 @@ public class ImportServer implements Serializable, Validatable {
}
return result.toHtml("Repositories imported successfully");
- } catch (Exception e) {
- for (Long projectId: projectIdsToDelete)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ } catch (URISyntaxException e) {
throw new RuntimeException(e);
} finally {
client.close();
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.java
index f04abca0d5..6f8c10b146 100644
--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.java
+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.java
@@ -75,13 +75,13 @@ public class GitLabIssueImporter implements IssueImporter {
}
@Override
- public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ public String doImport(Project project, boolean dryRun, TaskLogger logger) {
ImportServer server = serverStep.getSetting();
String gitLabProject = projectStep.getSetting().getProject();
IssueImportOption option = optionStep.getSetting();
logger.log("Importing issues from project " + gitLabProject + "...");
Map<String, Optional<User>> users = new HashMap<>();
- return server.importIssues(gitLabProject, project, retainIssueNumbers, option, users, dryRun, logger)
+ return server.importIssues(gitLabProject, project, option, users, dryRun, logger)
.toHtml("Issues imported successfully");
}
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.java
index ee1b96cb81..e14f6d3630 100644
--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.java
+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.java
@@ -61,7 +61,8 @@ public class GitLabProjectImporter implements ProjectImporter {
protected ImportProjects newSetting() {
ImportProjects projects = new ImportProjects();
String groupId = groupStep.getSetting().getGroupId();
- for (String project: serverStep.getSetting().listProjects(groupId)) {
+ for (String project: serverStep.getSetting().listProjects(
+ groupId, groupStep.getSetting().isIncludeForks())) {
ProjectMapping projectMapping = new ProjectMapping();
projectMapping.setGitLabProject(project);
projectMapping.setOneDevProject(project);
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportGroup.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportGroup.java
index fa9a17a605..60847b3914 100644
--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportGroup.java
+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportGroup.java
@@ -17,6 +17,8 @@ public class ImportGroup implements Serializable {
private String groupId;
+ private boolean includeForks;
+
@Editable(order=100, name="GitLab Group", description="Specify group to import from. "
+ "Leave empty to import from projects under current account")
@ChoiceProvider("getGroupChoices")
@@ -35,4 +37,13 @@ public class ImportGroup implements Serializable {
return setting.server.listGroups();
}
+ @Editable(order=200, description="Whether or not to include forked repositories")
+ public boolean isIncludeForks() {
+ return includeForks;
+ }
+
+ public void setIncludeForks(boolean includeForks) {
+ this.includeForks = includeForks;
+ }
+
}
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProject.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProject.java
index 3840dc67e5..3a2b8b0b58 100644
--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProject.java
+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProject.java
@@ -28,12 +28,17 @@ public class ImportProject extends ImportGroup {
this.project = project;
}
+ @Override
+ public boolean isIncludeForks() {
+ return super.isIncludeForks();
+ }
+
@SuppressWarnings("unused")
private static List<String> getProjectChoices() {
BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);
ImportProject project = (ImportProject) editor.getModelObject();
String groupId = (String) EditContext.get().getInputValue("groupId");
- return project.server.listProjects(groupId);
+ return project.server.listProjects(groupId, true);
}
}
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProjects.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProjects.java
index ab70b4b918..216aeae0aa 100644
--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProjects.java
+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProjects.java
@@ -12,6 +12,7 @@ import org.apache.shiro.authz.UnauthorizedException;
import io.onedev.server.OneDev;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.model.Project;
+import io.onedev.server.security.SecurityUtils;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@@ -41,21 +42,10 @@ public class ImportProjects implements Serializable, Validatable {
boolean isValid = true;
ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
for (int i=0; i<projectMappings.size(); i++) {
- String errorMessage = null;
try {
Project project = projectManager.setup(projectMappings.get(i).getOneDevProject());
- if (!project.isNew())
- errorMessage = "Project already exists";
- } catch (UnauthorizedException e) {
- errorMessage = e.getMessage();
- }
- if (errorMessage != null) {
- context.buildConstraintViolationWithTemplate(errorMessage)
- .addPropertyNode("projectMappings")
- .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
- .inIterable().atIndex(i).addConstraintViolation();
- isValid = false;
- } else {
+ if (!project.isNew() && !SecurityUtils.canManage(project))
+ throw new UnauthorizedException("Project management permission is required");
for (int j=0; j<projectMappings.size(); j++) {
if (j != i && projectMappings.get(j).getOneDevProject().equals(projectMappings.get(i).getOneDevProject())) {
context.buildConstraintViolationWithTemplate("Duplicate project")
@@ -66,6 +56,12 @@ public class ImportProjects implements Serializable, Validatable {
break;
}
}
+ } catch (UnauthorizedException e) {
+ context.buildConstraintViolationWithTemplate(e.getMessage())
+ .addPropertyNode("projectMappings")
+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
+ .inIterable().atIndex(i).addConstraintViolation();
+ isValid = false;
}
}
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportServer.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportServer.java
index c9b84804b1..bc856773af 100644
--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportServer.java
+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportServer.java
@@ -21,6 +21,7 @@ import java.util.regex.Pattern;
import javax.annotation.Nullable;
import javax.validation.ConstraintValidatorContext;
+import javax.validation.constraints.NotEmpty;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Invocation;
@@ -30,14 +31,12 @@ import javax.ws.rs.core.Response;
import org.apache.http.client.utils.URIBuilder;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.client.oauth2.OAuth2ClientSupport;
-import javax.validation.constraints.NotEmpty;
import org.joda.time.format.ISODateTimeFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.unbescape.html.HtmlEscape;
import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Preconditions;
import edu.emory.mathcs.backport.java.util.Collections;
import io.onedev.commons.bootstrap.SensitiveMasker;
@@ -65,7 +64,6 @@ import io.onedev.server.model.support.administration.GlobalIssueSetting;
import io.onedev.server.model.support.inputspec.InputSpec;
import io.onedev.server.model.support.issue.field.spec.FieldSpec;
import io.onedev.server.persistence.dao.Dao;
-import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.AttachmentTooLargeException;
import io.onedev.server.util.CollectionUtils;
import io.onedev.server.util.DateUtils;
@@ -144,7 +142,7 @@ public class ImportServer implements Serializable, Validatable {
return CollectionUtils.sortByValue(groups);
}
- List<String> listProjects(@Nullable String groupId) {
+ List<String> listProjects(@Nullable String groupId, boolean includeForks) {
List<String> projects = new ArrayList<>();
Client client = newClient();
try {
@@ -162,8 +160,10 @@ public class ImportServer implements Serializable, Validatable {
});
for (JsonNode projectNode: projectNodes) {
- if (groupId != null || projectNode.get("namespace").get("kind").asText().equals("user"))
+ if ((groupId != null || projectNode.get("namespace").get("kind").asText().equals("user"))
+ && (includeForks || projectNode.get("forked_from_project") == null)) {
projects.add(projectNode.get("path_with_namespace").asText());
+ }
}
} finally {
client.close();
@@ -266,19 +266,15 @@ public class ImportServer implements Serializable, Validatable {
}
String importProjects(ImportProjects projects, ProjectImportOption option, boolean dryRun, TaskLogger logger) {
- Collection<Long> projectIds = new ArrayList<>();
Client client = newClient();
try {
Map<String, Optional<User>> users = new HashMap<>();
ImportResult result = new ImportResult();
for (ProjectMapping projectMapping: projects.getProjectMappings()) {
- logger.log("Cloning code from project " + projectMapping.getGitLabProject() + "...");
-
String apiEndpoint = getApiEndpoint("/projects/" + projectMapping.getGitLabProject().replace("/", "%2F"));
JsonNode projectNode = JerseyUtils.get(client, apiEndpoint, logger);
ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
Project project = projectManager.setup(projectMapping.getOneDevProject());
- Preconditions.checkState(project.isNew());
project.setDescription(projectNode.get("description").asText(null));
project.setIssueManagement(projectNode.get("issues_enabled").asBoolean());
@@ -286,26 +282,32 @@ public class ImportServer implements Serializable, Validatable {
if (!visibility.equals("private") && option.getPublicRole() != null)
project.setDefaultRole(option.getPublicRole());
- URIBuilder builder = new URIBuilder(projectNode.get("http_url_to_repo").asText());
- builder.setUserInfo("git", getAccessToken());
-
- SensitiveMasker.push(new SensitiveMasker() {
-
- @Override
- public String mask(String text) {
- return StringUtils.replace(text, getAccessToken(), "******");
- }
+ if (project.isNew() || project.getDefaultBranch() == null) {
+ logger.log("Cloning code from project " + projectMapping.getGitLabProject() + "...");
+ URIBuilder builder = new URIBuilder(projectNode.get("http_url_to_repo").asText());
+ builder.setUserInfo("git", getAccessToken());
- });
- try {
- if (dryRun) {
- new LsRemoteCommand(builder.build().toString()).refs("HEAD").quiet(true).run();
- } else {
- projectManager.clone(project, builder.build().toString());
- projectIds.add(project.getId());
+ SensitiveMasker.push(new SensitiveMasker() {
+
+ @Override
+ public String mask(String text) {
+ return StringUtils.replace(text, getAccessToken(), "******");
+ }
+
+ });
+ try {
+ if (dryRun) {
+ new LsRemoteCommand(builder.build().toString()).refs("HEAD").quiet(true).run();
+ } else {
+ if (project.isNew())
+ projectManager.create(project);
+ projectManager.clone(project, builder.build().toString());
+ }
+ } finally {
+ SensitiveMasker.pop();
}
- } finally {
- SensitiveMasker.pop();
+ } else {
+ logger.warning("Skipping code clone as the project already has code");
}
if (option.getIssueImportOption() != null) {
@@ -322,15 +324,17 @@ public class ImportServer implements Serializable, Validatable {
}
for (Milestone milestone: milestones) {
- milestone.setProject(project);
- project.getMilestones().add(milestone);
- if (!dryRun)
- OneDev.getInstance(MilestoneManager.class).save(milestone);
+ if (project.getMilestone(milestone.getName()) == null) {
+ milestone.setProject(project);
+ project.getMilestones().add(milestone);
+ if (!dryRun)
+ OneDev.getInstance(MilestoneManager.class).save(milestone);
+ }
}
logger.log("Importing issues from project " + projectMapping.getGitLabProject() + "...");
ImportResult currentResult = importIssues(projectMapping.getGitLabProject(),
- project, true, option.getIssueImportOption(), users, dryRun, logger);
+ project, option.getIssueImportOption(), users, dryRun, logger);
result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);
result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);
@@ -340,18 +344,16 @@ public class ImportServer implements Serializable, Validatable {
}
return result.toHtml("Projects imported successfully");
- } catch (Exception e) {
- for (Long projectId: projectIds)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ } catch (URISyntaxException e) {
throw new RuntimeException(e);
} finally {
client.close();
}
}
- ImportResult importIssues(String gitLabProject, Project oneDevProject,
- boolean useExistingIssueNumbers, IssueImportOption option,
+ ImportResult importIssues(String gitLabProject, Project oneDevProject, IssueImportOption option,
Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {
+ IssueManager issueManager = OneDev.getInstance(IssueManager.class);
Client client = newClient();
try {
Set<String> nonExistentMilestones = new HashSet<>();
@@ -444,12 +446,13 @@ public class ImportServer implements Serializable, Validatable {
issue.setNumberScope(oneDevProject.getForkRoot());
- Long oldNumber = issueNode.get("iid").asLong();
Long newNumber;
- if (dryRun || useExistingIssueNumbers)
+ Long oldNumber = issueNode.get("iid").asLong();
+ if (dryRun || (issueManager.find(oneDevProject, oldNumber) == null && !issueNumberMappings.containsValue(oldNumber)))
newNumber = oldNumber;
else
- newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);
+ newNumber = issueManager.getNextNumber(oneDevProject);
+
issue.setNumber(newNumber);
issueNumberMappings.put(oldNumber, newNumber);
@@ -674,7 +677,7 @@ public class ImportServer implements Serializable, Validatable {
if (issue.getDescription() != null)
issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));
- OneDev.getInstance(IssueManager.class).save(issue);
+ issueManager.save(issue);
for (IssueSchedule schedule: issue.getSchedules())
dao.persist(schedule);
for (IssueField field: issue.getFields())
@@ -695,6 +698,8 @@ public class ImportServer implements Serializable, Validatable {
return result;
} finally {
+ if (!dryRun)
+ issueManager.resetNextNumber(oneDevProject);
client.close();
}
}
diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProjects.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProjects.java
index 07e7f5789e..118f675d64 100644
--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProjects.java
+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProjects.java
@@ -12,6 +12,7 @@ import org.apache.shiro.authz.UnauthorizedException;
import io.onedev.server.OneDev;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.model.Project;
+import io.onedev.server.security.SecurityUtils;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@@ -41,21 +42,10 @@ public class ImportProjects implements Serializable, Validatable {
boolean isValid = true;
ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
for (int i=0; i<projectMappings.size(); i++) {
- String errorMessage = null;
try {
Project project = projectManager.setup(projectMappings.get(i).getOneDevProject());
- if (!project.isNew())
- errorMessage = "Project already exists";
- } catch (UnauthorizedException e) {
- errorMessage = e.getMessage();
- }
- if (errorMessage != null) {
- context.buildConstraintViolationWithTemplate(errorMessage)
- .addPropertyNode("projectMappings")
- .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
- .inIterable().atIndex(i).addConstraintViolation();
- isValid = false;
- } else {
+ if (!project.isNew() && !SecurityUtils.canManage(project))
+ throw new UnauthorizedException("Project management permission is required");
for (int j=0; j<projectMappings.size(); j++) {
if (j != i && projectMappings.get(j).getOneDevProject().equals(projectMappings.get(i).getOneDevProject())) {
context.buildConstraintViolationWithTemplate("Duplicate project")
@@ -66,6 +56,12 @@ public class ImportProjects implements Serializable, Validatable {
break;
}
}
+ } catch (UnauthorizedException e) {
+ context.buildConstraintViolationWithTemplate(e.getMessage())
+ .addPropertyNode("projectMappings")
+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
+ .inIterable().atIndex(i).addConstraintViolation();
+ isValid = false;
}
}
diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportServer.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportServer.java
index 38bcf7dc3e..f66fd0e303 100644
--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportServer.java
+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportServer.java
@@ -43,7 +43,6 @@ import org.unbescape.html.HtmlEscape;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import io.onedev.commons.utils.ExplicitException;
@@ -67,7 +66,6 @@ import io.onedev.server.model.support.administration.GlobalIssueSetting;
import io.onedev.server.model.support.inputspec.InputSpec;
import io.onedev.server.model.support.issue.field.spec.FieldSpec;
import io.onedev.server.persistence.dao.Dao;
-import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.DateUtils;
import io.onedev.server.util.JerseyUtils;
import io.onedev.server.util.JerseyUtils.PageDataConsumer;
@@ -333,8 +331,6 @@ public class ImportServer implements Serializable, Validatable {
}
String importProjects(ImportProjects projects, ImportOption option, boolean dryRun, TaskLogger logger) {
- Collection<Long> projectIds = new ArrayList<>();
-
Map<String, JsonNode> projectNodes = getProjectNodes(logger);
Client client = newClient();
@@ -355,17 +351,14 @@ public class ImportServer implements Serializable, Validatable {
ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
Project project = projectManager.setup(projectMapping.getOneDevProject());
- Preconditions.checkState(project.isNew());
project.setDescription(projectNode.get("description").asText(null));
- if (!dryRun) {
- OneDev.getInstance(ProjectManager.class).create(project);
- projectIds.add(project.getId());
- }
+ if (!dryRun && project.isNew())
+ projectManager.create(project);
logger.log("Importing issues from project " + jiraProject + "...");
- ImportResult currentResult = importIssues(projectNode, project, true, option, users, dryRun, logger);
+ ImportResult currentResult = importIssues(projectNode, project, option, users, dryRun, logger);
result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
result.errorAttachments.addAll(currentResult.errorAttachments);
result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);
@@ -375,17 +368,12 @@ public class ImportServer implements Serializable, Validatable {
}
return result.toHtml("Projects imported successfully");
- } catch (Exception e) {
- for (Long projectId: projectIds)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
- throw new RuntimeException(e);
} finally {
client.close();
}
}
- String importIssues(Project project, String jiraProject, ImportOption option,
- boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ String importIssues(Project project, String jiraProject, ImportOption option, boolean dryRun, TaskLogger logger) {
Map<String, JsonNode> projectNodes = getProjectNodes(logger);
Client client = newClient();
@@ -394,17 +382,16 @@ public class ImportServer implements Serializable, Validatable {
JsonNode projectNode = projectNodes.get(jiraProject);
if (projectNode == null)
throw new ExplicitException("Unable to find project: " + jiraProject);
- ImportResult result = importIssues(projectNode, project, retainIssueNumbers,
- option, users, dryRun, logger);
+ ImportResult result = importIssues(projectNode, project, option, users, dryRun, logger);
return result.toHtml("Issues imported successfully");
} finally {
client.close();
}
}
- private ImportResult importIssues(JsonNode jiraProject, Project oneDevProject,
- boolean retainIssueNumbers, ImportOption option,
+ private ImportResult importIssues(JsonNode jiraProject, Project oneDevProject, ImportOption option,
Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {
+ IssueManager issueManager = OneDev.getInstance(IssueManager.class);
Client client = newClient();
try {
Set<String> unmappedIssueStatuses = new HashSet<>();
@@ -485,12 +472,13 @@ public class ImportServer implements Serializable, Validatable {
issue.setNumberScope(oneDevProject.getForkRoot());
- Long oldNumber = Long.valueOf(StringUtils.substringAfterLast(issueNode.get("key").asText(), "-"));
Long newNumber;
- if (dryRun || retainIssueNumbers)
+ Long oldNumber = Long.valueOf(StringUtils.substringAfterLast(issueNode.get("key").asText(), "-"));
+ if (dryRun || (issueManager.find(oneDevProject, oldNumber) == null && !issueNumberMappings.containsValue(oldNumber)))
newNumber = oldNumber;
else
- newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);
+ newNumber = issueManager.getNextNumber(oneDevProject);
+
issue.setNumber(newNumber);
issueNumberMappings.put(oldNumber, newNumber);
@@ -833,7 +821,7 @@ public class ImportServer implements Serializable, Validatable {
if (issue.getDescription() != null)
issue.setDescription(migrator.migratePrefixed(issue.getDescription(), jiraProjectKey + "-"));
- OneDev.getInstance(IssueManager.class).save(issue);
+ issueManager.save(issue);
for (IssueSchedule schedule: issue.getSchedules())
dao.persist(schedule);
for (IssueField field: issue.getFields())
@@ -857,6 +845,8 @@ public class ImportServer implements Serializable, Validatable {
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
} finally {
+ if (!dryRun)
+ issueManager.resetNextNumber(oneDevProject);
client.close();
}
}
diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraIssueImporter.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraIssueImporter.java
index 118433f96f..9ae0cd58d8 100644
--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraIssueImporter.java
+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraIssueImporter.java
@@ -71,11 +71,11 @@ public class JiraIssueImporter implements IssueImporter {
}
@Override
- public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ public String doImport(Project project, boolean dryRun, TaskLogger logger) {
ImportServer server = serverStep.getSetting();
String jiraProject = projectStep.getSetting().getProject();
ImportOption option = optionStep.getSetting();
- return server.importIssues(project, jiraProject, option, retainIssueNumbers, dryRun, logger);
+ return server.importIssues(project, jiraProject, option, dryRun, logger);
}
@Override
diff --git a/server-plugin/server-plugin-import-url/src/main/java/io/onedev/server/plugin/imports/url/ImportServer.java b/server-plugin/server-plugin-import-url/src/main/java/io/onedev/server/plugin/imports/url/ImportServer.java
index 0b153a1864..12eb786bd2 100644
--- a/server-plugin/server-plugin-import-url/src/main/java/io/onedev/server/plugin/imports/url/ImportServer.java
+++ b/server-plugin/server-plugin-import-url/src/main/java/io/onedev/server/plugin/imports/url/ImportServer.java
@@ -1,15 +1,14 @@
package io.onedev.server.plugin.imports.url;
import java.io.Serializable;
+import java.net.URISyntaxException;
import javax.annotation.Nullable;
import javax.validation.ConstraintValidatorContext;
+import javax.validation.constraints.NotEmpty;
import org.apache.http.client.utils.URIBuilder;
import org.apache.shiro.authz.UnauthorizedException;
-import javax.validation.constraints.NotEmpty;
-
-import com.google.common.base.Preconditions;
import io.onedev.commons.bootstrap.SensitiveMasker;
import io.onedev.commons.utils.ExplicitException;
@@ -19,7 +18,7 @@ import io.onedev.server.OneDev;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.git.command.LsRemoteCommand;
import io.onedev.server.model.Project;
-import io.onedev.server.storage.StorageManager;
+import io.onedev.server.security.SecurityUtils;
import io.onedev.server.util.EditContext;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
@@ -86,11 +85,7 @@ public class ImportServer implements Serializable, Validatable {
}
String importProject(boolean dryRun, TaskLogger logger) {
- Long projectId = null;
-
try {
- logger.log("Cloning code from " + getUrl() + "...");
-
String projectPath = getProject();
if (projectPath == null)
projectPath = deriveProjectPath(getUrl());
@@ -98,38 +93,41 @@ public class ImportServer implements Serializable, Validatable {
throw new ExplicitException("Invalid url: " + getUrl());
Project project = getProjectManager().setup(projectPath);
- Preconditions.checkState(project.isNew());
-
- URIBuilder builder = new URIBuilder(getUrl());
- if (authentication != null)
- builder.setUserInfo(authentication.getUserName(), authentication.getPassword());
-
- SensitiveMasker.push(new SensitiveMasker() {
-
- @Override
- public String mask(String text) {
- if (authentication != null)
- return StringUtils.replace(text, authentication.getPassword(), "******");
- else
- return text;
- }
+
+ if (project.isNew() || project.getDefaultBranch() == null) {
+ logger.log("Cloning code from " + getUrl() + "...");
- });
- try {
- if (dryRun) {
- new LsRemoteCommand(builder.build().toString()).refs("HEAD").quiet(true).run();
- } else {
- getProjectManager().clone(project, builder.build().toString());
- projectId = project.getId();
+ URIBuilder builder = new URIBuilder(getUrl());
+ if (authentication != null)
+ builder.setUserInfo(authentication.getUserName(), authentication.getPassword());
+
+ SensitiveMasker.push(new SensitiveMasker() {
+
+ @Override
+ public String mask(String text) {
+ if (authentication != null)
+ return StringUtils.replace(text, authentication.getPassword(), "******");
+ else
+ return text;
+ }
+
+ });
+ try {
+ if (dryRun) {
+ new LsRemoteCommand(builder.build().toString()).refs("HEAD").quiet(true).run();
+ } else {
+ if (project.isNew())
+ getProjectManager().create(project);
+ getProjectManager().clone(project, builder.build().toString());
+ }
+ } finally {
+ SensitiveMasker.pop();
}
- } finally {
- SensitiveMasker.pop();
+ return "project imported successfully";
+ } else {
+ throw new ExplicitException("Project already has code");
}
-
- return "project imported successfully";
- } catch (Exception e) {
- if (projectId != null)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ } catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
@@ -151,16 +149,12 @@ public class ImportServer implements Serializable, Validatable {
}
if (getProject() != null) {
- String errorMessage = null;
try {
Project project = getProjectManager().setup(getProject());
- if (!project.isNew())
- errorMessage = "Project already exists";
+ if (!project.isNew() && !SecurityUtils.canManage(project))
+ throw new UnauthorizedException("Project management permission is required");
} catch (UnauthorizedException e) {
- errorMessage = e.getMessage();
- }
- if (errorMessage != null) {
- context.buildConstraintViolationWithTemplate(errorMessage)
+ context.buildConstraintViolationWithTemplate(e.getMessage())
.addPropertyNode("project").addConstraintViolation();
isValid = false;
}
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProjects.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProjects.java
index 41a498f888..85e28e5e81 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProjects.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProjects.java
@@ -12,6 +12,7 @@ import org.apache.shiro.authz.UnauthorizedException;
import io.onedev.server.OneDev;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.model.Project;
+import io.onedev.server.security.SecurityUtils;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@@ -43,21 +44,10 @@ public class ImportProjects implements Serializable, Validatable {
boolean isValid = true;
ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
for (int i=0; i<projectMappings.size(); i++) {
- String errorMessage = null;
try {
Project project = projectManager.setup(projectMappings.get(i).getOneDevProject());
- if (!project.isNew())
- errorMessage = "Project already exists";
- } catch (UnauthorizedException e) {
- errorMessage = e.getMessage();
- }
- if (errorMessage != null) {
- context.buildConstraintViolationWithTemplate(errorMessage)
- .addPropertyNode("projectMappings")
- .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
- .inIterable().atIndex(i).addConstraintViolation();
- isValid = false;
- } else {
+ if (!project.isNew() && !SecurityUtils.canManage(project))
+ throw new UnauthorizedException("Project management permission is required");
for (int j=0; j<projectMappings.size(); j++) {
if (j != i && projectMappings.get(j).getOneDevProject().equals(projectMappings.get(i).getOneDevProject())) {
context.buildConstraintViolationWithTemplate("Duplicate project")
@@ -68,6 +58,12 @@ public class ImportProjects implements Serializable, Validatable {
break;
}
}
+ } catch (UnauthorizedException e) {
+ context.buildConstraintViolationWithTemplate(e.getMessage())
+ .addPropertyNode("projectMappings")
+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
+ .inIterable().atIndex(i).addConstraintViolation();
+ isValid = false;
}
}
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java
index 553c162de7..f2490e050b 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java
@@ -21,6 +21,7 @@ import java.util.stream.Collectors;
import javax.annotation.Nullable;
import javax.validation.ConstraintValidatorContext;
+import javax.validation.constraints.NotEmpty;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Invocation;
@@ -30,16 +31,13 @@ import javax.ws.rs.core.Response;
import org.apache.http.client.utils.URIBuilder;
import org.apache.wicket.MetaDataKey;
import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
-import javax.validation.constraints.NotEmpty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.unbescape.html.HtmlEscape;
import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
-import io.onedev.commons.utils.ExceptionUtils;
import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.StringUtils;
import io.onedev.commons.utils.TaskLogger;
@@ -75,11 +73,10 @@ import io.onedev.server.model.support.issue.field.spec.UserChoiceField;
import io.onedev.server.model.support.issue.field.spec.WorkingPeriodField;
import io.onedev.server.persistence.dao.Dao;
import io.onedev.server.security.SecurityUtils;
-import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.DateUtils;
import io.onedev.server.util.JerseyUtils;
-import io.onedev.server.util.Pair;
import io.onedev.server.util.JerseyUtils.PageDataConsumer;
+import io.onedev.server.util.Pair;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@@ -296,8 +293,9 @@ public class ImportServer implements Serializable, Validatable {
return option;
}
- private ImportResult importIssues(String youTrackProjectId, Project oneDevProject, boolean retainIssueNumbers,
+ private ImportResult importIssues(String youTrackProjectId, Project oneDevProject,
ImportOption option, boolean dryRun, TaskLogger logger) {
+ IssueManager issueManager = OneDev.getInstance(IssueManager.class);
Client client = newClient();
try {
String apiEndpoint = getApiEndpoint("/admin/projects/" + youTrackProjectId + "?fields=shortName");
@@ -458,12 +456,14 @@ public class ImportServer implements Serializable, Validatable {
Issue issue = new Issue();
String readableId = issueNode.get("idReadable").asText();
- Long oldNumber = issueNode.get("numberInProject").asLong();
+
Long newNumber;
- if (dryRun || retainIssueNumbers)
+ Long oldNumber = issueNode.get("numberInProject").asLong();
+ if (dryRun || (issueManager.find(oneDevProject, oldNumber) == null && !issueNumberMappings.containsValue(oldNumber)))
newNumber = oldNumber;
else
- newNumber = getIssueManager().getNextNumber(oneDevProject);
+ newNumber = issueManager.getNextNumber(oneDevProject);
+
issue.setNumber(newNumber);
issueNumberMappings.put(oldNumber, newNumber);
issueMappings.put(oldNumber, issue);
@@ -1007,7 +1007,7 @@ public class ImportServer implements Serializable, Validatable {
if (issue.getDescription() != null)
issue.setDescription(migrator.migratePrefixed(issue.getDescription(), youTrackProjectShortName + "-"));
- getIssueManager().save(issue);
+ issueManager.save(issue);
for (IssueSchedule schedule: issue.getSchedules())
dao.persist(schedule);
for (IssueField field: issue.getFields())
@@ -1046,14 +1046,12 @@ public class ImportServer implements Serializable, Validatable {
return result;
} finally {
+ if (!dryRun)
+ issueManager.resetNextNumber(oneDevProject);
client.close();
}
}
- private IssueManager getIssueManager() {
- return OneDev.getInstance(IssueManager.class);
- }
-
private GlobalIssueSetting getIssueSetting() {
return OneDev.getInstance(SettingManager.class).getIssueSetting();
}
@@ -1074,8 +1072,6 @@ public class ImportServer implements Serializable, Validatable {
String importProjects(ImportProjects projects, ImportOption option, boolean dryRun, TaskLogger logger) {
Map<String, String> youTrackProjectIds = new HashMap<>();
Map<String, String> youTrackProjectDescriptions = new HashMap<>();
- Collection<Long> projectIds = new ArrayList<>();
-
Client client = newClient();
try {
String apiEndpoint = getApiEndpoint("/admin/projects?fields=id,name,description");
@@ -1096,17 +1092,14 @@ public class ImportServer implements Serializable, Validatable {
ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
Project project = projectManager.setup(projectMapping.getOneDevProject());
- Preconditions.checkState(project.isNew());
project.setDescription(youTrackProjectDescriptions.get(projectMapping.getYouTrackProject()));
project.setIssueManagement(true);
- if (!dryRun) {
+ if (!dryRun && project.isNew())
projectManager.create(project);
- projectIds.add(project.getId());
- }
ImportResult currentResult = importIssues(youTrackProjectId, project,
- true, option, dryRun, logger);
+ option, dryRun, logger);
result.mismatchedIssueFields.putAll(currentResult.mismatchedIssueFields);
result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);
@@ -1117,10 +1110,6 @@ public class ImportServer implements Serializable, Validatable {
}
return result.toHtml("Projects imported successfully");
- } catch (Exception e) {
- for (Long projectId: projectIds)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
- throw ExceptionUtils.unchecked(e);
} finally {
client.close();
}
@@ -1154,7 +1143,7 @@ public class ImportServer implements Serializable, Validatable {
}
String importIssues(Project project, String youTrackProject, ImportOption option,
- boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ boolean dryRun, TaskLogger logger) {
logger.log("Importing issues from '" + youTrackProject + "'...");
Client client = newClient();
try {
@@ -1162,7 +1151,7 @@ public class ImportServer implements Serializable, Validatable {
for (JsonNode projectNode: list(client, apiEndpoint, logger)) {
if (youTrackProject.equals(projectNode.get("name").asText())) {
ImportResult result = importIssues(projectNode.get("id").asText(),
- project, retainIssueNumbers, option, dryRun, logger);
+ project, option, dryRun, logger);
return result.toHtml("Issues imported successfully");
}
}
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java
index 6b0a1514ef..a7bf1ae896 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java
@@ -72,12 +72,12 @@ public class YouTrackIssueImporter implements IssueImporter {
}
@Override
- public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ public String doImport(Project project, boolean dryRun, TaskLogger logger) {
ImportServer server = serverStep.getSetting();
String youTrackProject = projectStep.getSetting().getProject();
ImportOption option = optionStep.getSetting();
- return server.importIssues(project, youTrackProject, option, retainIssueNumbers, dryRun, logger);
+ return server.importIssues(project, youTrackProject, option, dryRun, logger);
}
@Override