The Unified and Autonomous Development Platform
Fix issue #322 - Import project from GitLab
Robin Shenaad79192422e
diff --git a/pom.xml b/pom.xmlindex 0dd0511002..9737559cb3 100644--- a/pom.xml+++ b/pom.xml@@ -204,6 +204,11 @@<artifactId>jersey-container-servlet</artifactId><version>${jersey.version}</version></dependency>+ <dependency>+ <groupId>org.glassfish.jersey.security</groupId>+ <artifactId>oauth2-client</artifactId>+ <version>${jersey.version}</version>+ </dependency><dependency><groupId>org.glassfish.jersey.ext</groupId><artifactId>jersey-bean-validation</artifactId>diff --git a/server-core/pom.xml b/server-core/pom.xmlindex 2040bc0702..1c573d4c97 100644--- a/server-core/pom.xml+++ b/server-core/pom.xml@@ -228,6 +228,10 @@<groupId>org.glassfish.jersey.ext</groupId><artifactId>jersey-bean-validation</artifactId></dependency>+ <dependency>+ <groupId>org.glassfish.jersey.security</groupId>+ <artifactId>oauth2-client</artifactId>+ </dependency><dependency><groupId>org.hibernate</groupId><artifactId>hibernate-validator</artifactId>diff --git a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.javaindex 9b2fce36ee..5d058a62d6 100644--- a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java+++ b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java@@ -313,7 +313,19 @@ public abstract class IssueListPanel extends Panel {@Overrideprotected List<MenuItem> getMenuItems(FloatingPanel dropdown) {Collection<IssueImporter<? extends Serializable, ? extends Serializable>> importers = new ArrayList<>();- for (IssueImporterContribution contribution: OneDev.getExtensions(IssueImporterContribution.class))++ List<IssueImporterContribution> contributions =+ new ArrayList<>(OneDev.getExtensions(IssueImporterContribution.class));+ Collections.sort(contributions, new Comparator<IssueImporterContribution>() {++ @Override+ public int compare(IssueImporterContribution o1, IssueImporterContribution o2) {+ return o1.getOrder() - o2.getOrder();+ }++ });++ for (IssueImporterContribution contribution: contributions)importers.addAll(contribution.getImporters());List<MenuItem> menuItems = new ArrayList<>();diff --git a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.html b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.htmlindex ea30107be4..e48e91c935 100644--- a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.html+++ b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.html@@ -23,7 +23,7 @@<wicket:svg href="sort" class="icon"></wicket:svg> Order By</a><a wicket:id="importProjects" class="import-projects text-gray">- <wicket:svg href="import" class="icon"></wicket:svg> Import Projects...+ <wicket:svg href="import" class="icon"></wicket:svg> Import...</a></div><div wicket:id="body">diff --git a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.javaindex af8f223050..83cfbb8b61 100644--- a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java+++ b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java@@ -3,6 +3,8 @@ package io.onedev.server.web.component.project.list;import java.io.Serializable;import java.util.ArrayList;import java.util.Collection;+import java.util.Collections;+import java.util.Comparator;import java.util.Iterator;import java.util.List;@@ -245,7 +247,18 @@ public class ProjectListPanel extends Panel {@Overrideprotected List<MenuItem> getMenuItems(FloatingPanel dropdown) {Collection<ProjectImporter<? extends Serializable, ? extends Serializable>> importers = new ArrayList<>();- for (ProjectImporterContribution contribution: OneDev.getExtensions(ProjectImporterContribution.class))+ List<ProjectImporterContribution> contributions =+ new ArrayList<>(OneDev.getExtensions(ProjectImporterContribution.class));+ Collections.sort(contributions, new Comparator<ProjectImporterContribution>() {++ @Override+ public int compare(ProjectImporterContribution o1, ProjectImporterContribution o2) {+ return o1.getOrder() - o2.getOrder();+ }++ });++ for (ProjectImporterContribution contribution: contributions)importers.addAll(contribution.getImporters());List<MenuItem> menuItems = new ArrayList<>();diff --git a/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskButton.java b/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskButton.javaindex 725617e17e..9b75a308d8 100644--- a/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskButton.java+++ b/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskButton.java@@ -130,7 +130,7 @@ public abstract class TaskButton extends AjaxButton {});}- protected void onCompleted(AjaxRequestTarget target) {+ protected void onCompleted(AjaxRequestTarget target, boolean successful) {}protected void onCancelled(AjaxRequestTarget target) {@@ -143,13 +143,12 @@ public abstract class TaskButton extends AjaxButton {ExecutorService executorService = OneDev.getInstance(ExecutorService.class);List<JobLogEntryEx> messages = new ArrayList<>();messages.add(new JobLogEntryEx(new JobLogEntry(new Date(), "Please wait...")));- TaskFuture future = getTaskFutures().put(path, new TaskFuture(executorService.submit(new Callable<String>() {+ TaskFuture future = getTaskFutures().put(path, new TaskFuture(executorService.submit(new Callable<TaskResult>() {@Override- public String call() throws Exception {- String result;+ public TaskResult call() throws Exception {try {- result = String.format(+ String feedback = String.format("<div class='task-result text-break alert alert-light-info'>%s</div>",runTask(new SimpleLogger() {@@ -160,22 +159,24 @@ public abstract class TaskButton extends AjaxButton {}}- }));+ }));+ return new TaskResult(true, feedback);} catch (Exception e) {logger.error("Error " + title, e);String suggestedSolution = ExceptionUtils.suggestSolution(e);if (suggestedSolution != null)logger.warn("!!! " + suggestedSolution);+ String feedback;if (e.getMessage() != null)- result = e.getMessage();+ feedback = e.getMessage();else- result = "Error " + title;- result = String.format(+ feedback = "Error " + title;+ feedback = String.format("<div class='task-result text-break alert alert-light-danger'>%s</div>",- HtmlEscape.escapeHtml5(result));- result = StringUtils.replace(result, "\n", "<br>");+ HtmlEscape.escapeHtml5(feedback));+ feedback = StringUtils.replace(feedback, "\n", "<br>");+ return new TaskResult(false, feedback);}- return result;}}), messages));@@ -188,14 +189,18 @@ public abstract class TaskButton extends AjaxButton {@Overrideprotected void onClosed() {super.onClosed();- Future<?> future = getTaskFutures().remove(path);+ TaskFuture future = getTaskFutures().remove(path);AjaxRequestTarget target = RequestCycle.get().find(AjaxRequestTarget.class);if (future != null && !future.isDone()) {future.cancel(true);onCancelled(target);} else {- onCompleted(target);+ try {+ onCompleted(target, future != null && future.get().successful);+ } catch (InterruptedException | ExecutionException e) {+ throw new RuntimeException(e);+ }}}@@ -223,7 +228,7 @@ public abstract class TaskButton extends AjaxButton {}@Override- protected String getResult() {+ protected TaskResult getResult() {TaskFuture future = getTaskFutures().get(path);if (future != null && future.isDone() && !future.isCancelled()) {try {@@ -299,15 +304,15 @@ public abstract class TaskButton extends AjaxButton {}- private static class TaskFuture implements Future<String> {+ private static class TaskFuture implements Future<TaskResult> {- private final Future<String> wrapped;+ private final Future<TaskResult> wrapped;private final List<JobLogEntryEx> logEntries;private volatile Date lastActive = new Date();- public TaskFuture(Future<String> wrapped, List<JobLogEntryEx> logEntries) {+ public TaskFuture(Future<TaskResult> wrapped, List<JobLogEntryEx> logEntries) {this.wrapped = wrapped;this.logEntries = logEntries;}@@ -332,12 +337,12 @@ public abstract class TaskButton extends AjaxButton {}@Override- public String get() throws InterruptedException, ExecutionException {+ public TaskResult get() throws InterruptedException, ExecutionException {return wrapped.get();}@Override- public String get(long timeout, TimeUnit unit)+ public TaskResult get(long timeout, TimeUnit unit)throws InterruptedException, ExecutionException, TimeoutException {return get(timeout, unit);}@@ -352,4 +357,5 @@ public abstract class TaskButton extends AjaxButton {}}+}diff --git a/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskFeedbackPanel.java b/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskFeedbackPanel.javaindex 235e2df899..5d036c1a81 100644--- a/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskFeedbackPanel.java+++ b/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskFeedbackPanel.java@@ -86,7 +86,7 @@ abstract class TaskFeedbackPanel extends Panel {@Overrideprotected String load() {- return getResult();+ return getResult().getFeedback();}}) {@@ -149,6 +149,6 @@ abstract class TaskFeedbackPanel extends Panel {protected abstract List<JobLogEntryEx> getLogEntries();@Nullable- protected abstract String getResult();+ protected abstract TaskResult getResult();}diff --git a/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskResult.java b/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskResult.javanew file mode 100644index 0000000000..400a1c9b42--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskResult.java@@ -0,0 +1,26 @@+package io.onedev.server.web.component.taskbutton;++import java.io.Serializable;++public class TaskResult implements Serializable {++ private static final long serialVersionUID = 1L;++ final boolean successful;++ private final String feedback;++ public TaskResult(boolean successful, String feedback) {+ this.successful = successful;+ this.feedback = feedback;+ }++ public boolean isSuccessful() {+ return successful;+ }++ public String getFeedback() {+ return feedback;+ }++}\ No newline at end of filediff --git a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.javaindex 2bac5f24a6..d71a89653d 100644--- a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java+++ b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java@@ -118,15 +118,17 @@ public class ProjectImportPage<T extends Serializable, S extends Serializable> eform.add(new TaskButton("import") {@Override- protected void onCompleted(AjaxRequestTarget target) {- super.onCompleted(target);-- EntitySort sort = new EntitySort();- sort.setField(Project.NAME_UPDATE_DATE);- sort.setDirection(Direction.DESCENDING);- ProjectQuery query = new ProjectQuery(null, Lists.newArrayList(sort));- PageParameters params = ProjectListPage.paramsOf(query.toString(), 0, 0);- throw new RestartResponseException(ProjectListPage.class, params);+ protected void onCompleted(AjaxRequestTarget target, boolean successful) {+ super.onCompleted(target, successful);++ if (successful) {+ EntitySort sort = new EntitySort();+ sort.setField(Project.NAME_UPDATE_DATE);+ sort.setDirection(Direction.DESCENDING);+ ProjectQuery query = new ProjectQuery(null, Lists.newArrayList(sort));+ PageParameters params = ProjectListPage.paramsOf(query.toString(), 0, 0);+ throw new RestartResponseException(ProjectListPage.class, params);+ }}@Overridediff --git a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImporterContribution.java b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImporterContribution.javaindex 54b4f8a94a..520c6612ac 100644--- a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImporterContribution.java+++ b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImporterContribution.java@@ -10,4 +10,6 @@ public interface ProjectImporterContribution {Collection<ProjectImporter<? extends Serializable, ? extends Serializable>> getImporters();+ int getOrder();+}diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.javaindex eac8486f0f..3978cda020 100644--- a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java@@ -122,16 +122,18 @@ public class IssueImportPage<T extends Serializable, S extends Serializable> extform.add(new TaskButton("import") {@Override- protected void onCompleted(AjaxRequestTarget target) {- super.onCompleted(target);+ protected void onCompleted(AjaxRequestTarget target, boolean successful) {+ super.onCompleted(target, successful);- EntitySort sort = new EntitySort();- sort.setField(Issue.NAME_NUMBER);- sort.setDirection(Direction.DESCENDING);- IssueQuery query = new IssueQuery(null, Lists.newArrayList(sort));-- PageParameters params = ProjectIssueListPage.paramsOf(getProject(), query.toString(), 0);- throw new RestartResponseException(ProjectIssueListPage.class, params);+ if (successful) {+ EntitySort sort = new EntitySort();+ sort.setField(Issue.NAME_NUMBER);+ sort.setDirection(Direction.DESCENDING);+ IssueQuery query = new IssueQuery(null, Lists.newArrayList(sort));++ PageParameters params = ProjectIssueListPage.paramsOf(getProject(), query.toString(), 0);+ throw new RestartResponseException(ProjectIssueListPage.class, params);+ }}@Override@@ -213,7 +215,7 @@ public class IssueImportPage<T extends Serializable, S extends Serializable> ext@Overrideprotected Component newProjectTitle(String componentId) {- return new Label(componentId, "Import Issues frmo " + importer.getName());+ return new Label(componentId, "Import Issues from " + importer.getName());}}diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporterContribution.java b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporterContribution.javaindex 45aa00bad3..b8edb1f561 100644--- a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporterContribution.java+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporterContribution.java@@ -10,4 +10,6 @@ public interface IssueImporterContribution {Collection<IssueImporter<? extends Serializable, ? extends Serializable>> getImporters();+ int getOrder();+}diff --git a/server-plugin/pom.xml b/server-plugin/pom.xmlindex 3f73f35220..bd0eb90f8f 100644--- a/server-plugin/pom.xml+++ b/server-plugin/pom.xml@@ -30,5 +30,6 @@<module>server-plugin-report-checkstyle</module><module>server-plugin-import-github</module><module>server-plugin-import-youtrack</module>+ <module>server-plugin-import-gitlab</module></modules></project>\ No newline at end of filediff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportResult.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportResult.javaindex bd5f191bfe..b3bb792d98 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportResult.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportResult.java@@ -1,12 +1,17 @@package io.onedev.server.plugin.imports.github;+import java.util.ArrayList;+import java.util.Collection;import java.util.HashSet;+import java.util.List;import java.util.Set;import org.unbescape.html.HtmlEscape;public class GitHubImportResult {+ private static final int MAX_DISPLAY_ENTRIES = 100;+Set<String> nonExistentLogins = new HashSet<>();Set<String> unmappedIssueLabels = new HashSet<>();@@ -15,6 +20,15 @@ public class GitHubImportResult {boolean issuesImported = false;+ private String getEntryFeedback(String entryDescription, Collection<String> entries) {+ if (entries.size() > MAX_DISPLAY_ENTRIES) {+ List<String> entriesToDisplay = new ArrayList<>(entries).subList(0, MAX_DISPLAY_ENTRIES);+ return "<li> " + entryDescription + ": " + HtmlEscape.escapeHtml5(entriesToDisplay.toString()) + " and more";+ } else {+ return "<li> " + entryDescription + ": " + HtmlEscape.escapeHtml5(entries.toString());+ }+ }+public String toHtml(String leadingText) {StringBuilder feedback = new StringBuilder(leadingText);@@ -28,17 +42,15 @@ public class GitHubImportResult {if (hasNotice)feedback.append("<br><br><b>NOTE:</b><ul>");- if (!nonExistentMilestones.isEmpty()) {- feedback.append("<li> Non existent milestones: "- + HtmlEscape.escapeHtml5(nonExistentMilestones.toString()));- }+ if (!nonExistentMilestones.isEmpty())+ feedback.append(getEntryFeedback("Non existent milestones", nonExistentMilestones));if (!unmappedIssueLabels.isEmpty()) {- feedback.append("<li> GitHub issue labels not mapped to OneDev custom field: "- + HtmlEscape.escapeHtml5(unmappedIssueLabels.toString()));+ feedback.append(getEntryFeedback("GitHub issue labels not mapped to OneDev custom field",+ unmappedIssueLabels));}if (!nonExistentLogins.isEmpty()) {- feedback.append("<li> GitHub logins without public email or public email can not be mapped to OneDev account: "- + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));+ feedback.append(getEntryFeedback("GitHub logins without public email or public email can not be mapped to OneDev account",+ nonExistentLogins));}if (issuesImported)diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportUtils.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportUtils.javaindex 8d4aea7fa6..42007acc17 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportUtils.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportUtils.java@@ -269,6 +269,17 @@ public class GitHubImportUtils {if (!currentUnmappedLabels.isEmpty())extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));+ Set<String> fieldAndValues = new HashSet<>();+ for (IssueField field: issue.getFields()) {+ String fieldAndValue = field.getName() + "::" + field.getValue();+ if (!fieldAndValues.add(fieldAndValue)) {+ String errorMessage = String.format(+ "Duplicate issue field mapping (issue: #%d, field: %s)",+ gitHubRepo + "#" + oldNumber, fieldAndValue);+ throw new ExplicitException(errorMessage);+ }+ }+if (!extraIssueInfo.isEmpty()) {StringBuilder builder = new StringBuilder("|");for (String key: extraIssueInfo.keySet())diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportOption.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportOption.javaindex 255039723c..122e785746 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportOption.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportOption.java@@ -26,7 +26,8 @@ public class GitHubIssueImportOption implements Serializable {private List<IssueLabelMapping> issueLabelMappings = new ArrayList<>();- @Editable(order=300, description="Specify which issue state to use for closed GitHub issues")+ @Editable(order=300, description="Specify which issue state to use for closed GitHub issues.<br>"+ + "<b>NOTE: </b> You may customize OneDev issue states in case there is no appropriate option here")@ChoiceProvider("getCloseStateChoices")@NotEmptypublic String getClosedIssueState() {@@ -49,8 +50,8 @@ public class GitHubIssueImportOption implements Serializable {return choices;}- @Editable(order=350, description="Specify a multi-value user field to "- + "hold assignees information")+ @Editable(order=350, description="Specify a multi-value user field to hold assignees information."+ + "<b>NOTE: </b> You may customize OneDev issue fields in case there is no appropriate option here")@ChoiceProvider("getAssigneesIssueFieldChoices")@NotEmptypublic String getAssigneesIssueField() {@@ -72,8 +73,8 @@ public class GitHubIssueImportOption implements Serializable {}@Editable(order=400, description="Specify how to map GitHub issue labels to OneDev custom "- + "fields. Only multi-valued enum field can be used here. Unmapped labels will be "- + "reflected in issue description")+ + "fields. Unmapped labels will be reflected in issue description.<br>"+ + "<b>NOTE: </b> You may customize OneDev issue fields in case there is no appropriate option here")public List<IssueLabelMapping> getIssueLabelMappings() {return issueLabelMappings;}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.javaindex 58de5a7bc9..b820c737c2 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java@@ -24,11 +24,7 @@ public class GitHubIssueImporter extends IssueImporter<GitHubIssueImportSource,@Overridepublic GitHubIssueImportOption getImportOption(GitHubIssueImportSource importSource, SimpleLogger logger) {- GitHubIssueImportOption importOption;- if (importSource.isPrepopulateImportOptions())- importOption = buildImportOption(importSource, importSource.getRepository(), logger);- else- importOption = new GitHubIssueImportOption();+ GitHubIssueImportOption importOption = buildImportOption(importSource, importSource.getRepository(), logger);return importOption;}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.javaindex 21be1145de..1f49c2c0b1 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java@@ -27,6 +27,11 @@ public class GitHubPluginModule extends AbstractPluginModule {public Collection<ProjectImporter<? extends Serializable, ? extends Serializable>> getImporters() {return Lists.newArrayList(new GitHubProjectImporter());}++ @Override+ public int getOrder() {+ return 100;+ }});@@ -36,6 +41,11 @@ public class GitHubPluginModule extends AbstractPluginModule {public Collection<IssueImporter<? extends Serializable, ? extends Serializable>> getImporters() {return Lists.newArrayList(new GitHubIssueImporter());}++ @Override+ public int getOrder() {+ return 100;+ }});}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportOption.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportOption.javaindex d4e144a0aa..1c76c42bad 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportOption.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportOption.java@@ -63,7 +63,7 @@ public class GitHubProjectImportOption extends GitHubIssueImportOption implementfor (int i=0; i<projectMappings.size(); i++) {if (projectManager.find(projectMappings.get(i).getOneDevProject()) != null) {context.buildConstraintViolationWithTemplate("Project name already used")- .addPropertyNode("imports")+ .addPropertyNode("projectMappings").addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT).inIterable().atIndex(i).addConstraintViolation();isValid = false;@@ -71,7 +71,7 @@ public class GitHubProjectImportOption extends GitHubIssueImportOption implementfor (int j=0; j<projectMappings.size(); j++) {if (j != i && projectMappings.get(j).getOneDevProject().equals(projectMappings.get(i).getOneDevProject())) {context.buildConstraintViolationWithTemplate("Duplicate project name")- .addPropertyNode("imports")+ .addPropertyNode("projectMappings").addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT).inIterable().atIndex(i).addConstraintViolation();isValid = false;diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportSource.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportSource.javaindex 1ca2ea3a80..dd225660ca 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportSource.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportSource.java@@ -33,8 +33,6 @@ public class GitHubProjectImportSource implements Serializable, Validatable {private String accessToken;- private boolean prepopulateImportOptions = true;-@Editable(order=10, name="GitHub API URL", description="Specify GitHub API url, for instance <tt>https://api.github.com</tt>")@NotEmptypublic String getApiUrl() {@@ -57,16 +55,6 @@ public class GitHubProjectImportSource implements Serializable, Validatable {this.accessToken = accessToken;}- @Editable(order=1000, description="If checked, import options will be pre-populated based on all accessible "- + "repositories and its settings")- public boolean isPrepopulateImportOptions() {- return prepopulateImportOptions;- }-- public void setPrepopulateImportOptions(boolean prepopulateImportOptions) {- this.prepopulateImportOptions = prepopulateImportOptions;- }-public static String getApiEndpoint(String apiUrl, String apiPath) {return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");}@@ -84,7 +72,12 @@ public class GitHubProjectImportSource implements Serializable, Validatable {WebTarget target = client.target(apiEndpoint);Invocation.Builder builder = target.request();try (Response response = builder.get()) {- if (response.getStatus() == 401) {+ if (!response.getMediaType().toString().startsWith("application/json")) {+ context.disableDefaultConstraintViolation();+ context.buildConstraintViolationWithTemplate("This does not seem like a GitHub api url")+ .addPropertyNode(PROP_API_URL).addConstraintViolation();+ return false;+ } else if (response.getStatus() == 401) {context.disableDefaultConstraintViolation();String errorMessage = "Authentication failed";context.buildConstraintViolationWithTemplate(errorMessage)diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.javaindex 7b543a2c5c..cf37bec44e 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java@@ -10,10 +10,10 @@ import static io.onedev.server.plugin.imports.github.GitHubImportUtils.newClientimport java.util.ArrayList;import java.util.Collection;import java.util.HashMap;+import java.util.List;import java.util.Map;import java.util.Optional;-import javax.annotation.Nullable;import javax.ws.rs.client.Client;import org.apache.http.client.utils.URIBuilder;@@ -24,7 +24,6 @@ import com.fasterxml.jackson.databind.JsonNode;import io.onedev.server.OneDev;import io.onedev.server.entitymanager.MilestoneManager;import io.onedev.server.entitymanager.ProjectManager;-import io.onedev.server.entitymanager.UserManager;import io.onedev.server.model.Milestone;import io.onedev.server.model.Project;import io.onedev.server.model.User;@@ -44,45 +43,27 @@ public class GitHubProjectImporter extends ProjectImporter<GitHubProjectImportSo@Overridepublic GitHubProjectImportOption getImportOption(GitHubProjectImportSource importSource, SimpleLogger logger) {GitHubProjectImportOption importOption = new GitHubProjectImportOption();- if (importSource.isPrepopulateImportOptions()) {- Client client = newClient(importSource);- try {- String apiEndpoint = importSource.getApiEndpoint("/user/repos");- for (JsonNode repoNode: list(client, apiEndpoint, logger)) {- String repoName = repoNode.get("name").asText();- String ownerName = repoNode.get("owner").get("login").asText();- ProjectMapping projectMapping = new ProjectMapping();- projectMapping.setGitHubRepo(ownerName + "/" + repoName);- projectMapping.setOneDevProject(ownerName + "-" + repoName);- importOption.getProjectMappings().add(projectMapping);- }- GitHubIssueImportOption issueImportOption = buildImportOption(importSource, null, logger);- importOption.setAssigneesIssueField(issueImportOption.getAssigneesIssueField());- importOption.setClosedIssueState(issueImportOption.getClosedIssueState());- importOption.setIssueLabelMappings(issueImportOption.getIssueLabelMappings());- } finally {- client.close();- }+ Client client = newClient(importSource);+ try {+ String apiEndpoint = importSource.getApiEndpoint("/user/repos");+ for (JsonNode repoNode: list(client, apiEndpoint, logger)) {+ String repoName = repoNode.get("name").asText();+ String ownerName = repoNode.get("owner").get("login").asText();+ ProjectMapping projectMapping = new ProjectMapping();+ projectMapping.setGitHubRepo(ownerName + "/" + repoName);+ projectMapping.setOneDevProject(ownerName + "-" + repoName);+ importOption.getProjectMappings().add(projectMapping);+ }+ GitHubIssueImportOption issueImportOption = buildImportOption(importSource, null, logger);+ importOption.setAssigneesIssueField(issueImportOption.getAssigneesIssueField());+ importOption.setClosedIssueState(issueImportOption.getClosedIssueState());+ importOption.setIssueLabelMappings(issueImportOption.getIssueLabelMappings());+ } finally {+ client.close();}return importOption;}- @Nullable- private User getUser(Client client, GitHubProjectImportSource importSource,- Map<String, Optional<User>> users, String login, SimpleLogger logger) {- Optional<User> userOpt = users.get(login);- if (userOpt == null) {- String apiEndpoint = importSource.getApiEndpoint("/users/" + login);- String email = get(client, apiEndpoint, logger).get("email").asText(null);- if (email != null)- userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByEmail(email));- else- userOpt = Optional.empty();- users.put(login, userOpt);- }- return userOpt.orElse(null);- }-@Overridepublic String doImport(GitHubProjectImportSource importSource, GitHubProjectImportOption importOption,boolean dryRun, SimpleLogger logger) {@@ -115,7 +96,7 @@ public class GitHubProjectImporter extends ProjectImporter<GitHubProjectImportSo}if (projectMapping.isImportIssues()) {- Map<Long, Milestone> milestones = new HashMap<>();+ List<Milestone> milestones = new ArrayList<>();logger.log("Importing milestones from repository " + projectMapping.getGitHubRepo() + "...");apiEndpoint = importSource.getApiEndpoint("/repos/" + projectMapping.getGitHubRepo() + "/milestones?state=all");for (JsonNode milestoneNode: list(client, apiEndpoint, logger)) {@@ -129,7 +110,7 @@ public class GitHubProjectImporter extends ProjectImporter<GitHubProjectImportSoif (milestoneNode.get("state").asText().equals("closed"))milestone.setClosed(true);- milestones.put(milestoneNode.get("number").asLong(), milestone);+ milestones.add(milestone);project.getMilestones().add(milestone);if (!dryRun)diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/IssueLabelMapping.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/IssueLabelMapping.javaindex da556df637..c08ca7eaa8 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/IssueLabelMapping.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/IssueLabelMapping.java@@ -49,7 +49,7 @@ public class IssueLabelMapping implements Serializable {List<String> choices = new ArrayList<>();GlobalIssueSetting issueSetting = OneDev.getInstance(SettingManager.class).getIssueSetting();for (FieldSpec field: issueSetting.getFieldSpecs()) {- if (field.getType().equals(InputSpec.ENUMERATION) && field.isAllowMultiple()) {+ if (field.getType().equals(InputSpec.ENUMERATION)) {for (String value: field.getPossibleValues())choices.add(field.getName() + "::" + value);}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectMapping.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectMapping.javaindex cb2b4393f0..477202bdf1 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectMapping.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectMapping.java@@ -20,8 +20,8 @@ public class ProjectMapping implements Serializable {private boolean importIssues = true;- @Editable(order=100, name="GitHub Repository", description="Specify GitHub repository to import "- + "code and issues (optional) from")+ @Editable(order=100, name="GitHub Repository", description="Specify GitHub repository in form of "+ + "<tt>organization/repository</tt>")@NotEmptypublic String getGitHubRepo() {return gitHubRepo;diff --git a/server-plugin/server-plugin-import-gitlab/pom.xml b/server-plugin/server-plugin-import-gitlab/pom.xmlnew file mode 100644index 0000000000..93ce6c3b38--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/pom.xml@@ -0,0 +1,13 @@+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">+ <modelVersion>4.0.0</modelVersion>+ <artifactId>server-plugin-import-gitlab</artifactId>+ <parent>+ <groupId>io.onedev</groupId>+ <artifactId>server-plugin</artifactId>+ <version>4.8.1</version>+ </parent>+ <properties>+ <moduleClass>io.onedev.server.plugin.imports.gitlab.GitLabPluginModule</moduleClass>+ </properties>+</project>diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabImportResult.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabImportResult.javanew file mode 100644index 0000000000..1b0d6deb9b--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabImportResult.java@@ -0,0 +1,67 @@+package io.onedev.server.plugin.imports.gitlab;++import java.util.ArrayList;+import java.util.Collection;+import java.util.HashSet;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Set;++import org.unbescape.html.HtmlEscape;++public class GitLabImportResult {++ private static final int MAX_DISPLAY_ENTRIES = 100;++ Set<String> nonExistentLogins = new HashSet<>();++ Set<String> unmappedIssueLabels = new HashSet<>();++ Set<String> nonExistentMilestones = new HashSet<>();++ Set<String> tooLargeAttachments = new LinkedHashSet<>();++ Set<String> errorAttachments = new LinkedHashSet<>();++ private String getEntryFeedback(String entryDescription, Collection<String> entries) {+ if (entries.size() > MAX_DISPLAY_ENTRIES) {+ List<String> entriesToDisplay = new ArrayList<>(entries).subList(0, MAX_DISPLAY_ENTRIES);+ return "<li> " + entryDescription + ": " + HtmlEscape.escapeHtml5(entriesToDisplay.toString()) + " and more";+ } else {+ return "<li> " + entryDescription + ": " + HtmlEscape.escapeHtml5(entries.toString());+ }+ }++ public String toHtml(String leadingText) {+ StringBuilder feedback = new StringBuilder(leadingText);++ boolean hasNotice = false;++ if (!nonExistentMilestones.isEmpty() || !unmappedIssueLabels.isEmpty()+ || !nonExistentLogins.isEmpty() || !tooLargeAttachments.isEmpty()) {+ hasNotice = true;+ }++ if (hasNotice)+ feedback.append("<br><br><b>NOTE:</b><ul>");++ if (!nonExistentMilestones.isEmpty())+ feedback.append(getEntryFeedback("Non existent milestones", nonExistentLogins));+ if (!unmappedIssueLabels.isEmpty())+ feedback.append(getEntryFeedback("GitLab issue labels not mapped to OneDev custom field", unmappedIssueLabels));+ if (!nonExistentLogins.isEmpty()) {+ feedback.append(getEntryFeedback("GitLab logins without email or email can not be mapped to OneDev account",+ nonExistentLogins));+ }+ if (!tooLargeAttachments.isEmpty())+ feedback.append(getEntryFeedback("Too large attachments", tooLargeAttachments));+ if (!errorAttachments.isEmpty())+ feedback.append(getEntryFeedback("Failed to download attachments", errorAttachments));++ if (hasNotice)+ feedback.append("</ul>");++ return feedback.toString();++ }+}diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabImportUtils.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabImportUtils.javanew file mode 100644index 0000000000..c5cd4c7de0--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabImportUtils.java@@ -0,0 +1,523 @@+package io.onedev.server.plugin.imports.gitlab;++import java.io.IOException;+import java.io.InputStream;+import java.net.URI;+import java.net.URISyntaxException;+import java.util.ArrayList;+import java.util.HashMap;+import java.util.HashSet;+import java.util.LinkedHashMap;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.regex.Matcher;+import java.util.regex.Pattern;++import javax.annotation.Nullable;+import javax.ws.rs.client.Client;+import javax.ws.rs.client.ClientBuilder;+import javax.ws.rs.client.Invocation;+import javax.ws.rs.client.WebTarget;+import javax.ws.rs.core.Response;++import org.apache.commons.lang3.StringUtils;+import org.apache.http.client.utils.URIBuilder;+import org.glassfish.jersey.client.ClientProperties;+import org.glassfish.jersey.client.oauth2.OAuth2ClientSupport;+import org.joda.time.format.ISODateTimeFormat;+import org.unbescape.html.HtmlEscape;++import com.fasterxml.jackson.databind.JsonNode;++import io.onedev.commons.utils.ExplicitException;+import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.IssueManager;+import io.onedev.server.entitymanager.SettingManager;+import io.onedev.server.entitymanager.UserManager;+import io.onedev.server.model.Issue;+import io.onedev.server.model.IssueComment;+import io.onedev.server.model.IssueField;+import io.onedev.server.model.Milestone;+import io.onedev.server.model.Project;+import io.onedev.server.model.User;+import io.onedev.server.model.support.LastUpdate;+import io.onedev.server.model.support.administration.GlobalIssueSetting;+import io.onedev.server.model.support.inputspec.InputSpec;+import io.onedev.server.model.support.issue.field.spec.FieldSpec;+import io.onedev.server.persistence.dao.Dao;+import io.onedev.server.util.AttachmentTooLargeException;+import io.onedev.server.util.DateUtils;+import io.onedev.server.util.JerseyUtils;+import io.onedev.server.util.Pair;+import io.onedev.server.util.ReferenceMigrator;+import io.onedev.server.util.SimpleLogger;++public class GitLabImportUtils {++ static final String NAME = "GitLab";++ static final int PER_PAGE = 100;++ private static final Pattern PATTERN_ATTACHMENT = Pattern.compile("\\[(.+?)\\]\\s*\\((/uploads/.+?)\\)");++ static GitLabIssueImportOption buildImportOption(GitLabProjectImportSource importSource,+ @Nullable String gitLabProject, SimpleLogger logger) {+ GitLabIssueImportOption importOption = new GitLabIssueImportOption();+ Client client = newClient(importSource);+ try {+ List<String> gitLabProjects = new ArrayList<>();+ if (gitLabProject == null) {+ String apiEndpoint = importSource.getApiEndpoint("/projects?membership=true");+ for (JsonNode projectNode: list(client, apiEndpoint, logger))+ gitLabProjects.add(projectNode.get("path_with_namespace").asText());+ } else {+ gitLabProjects.add(gitLabProject);+ }++ Set<String> labels = new LinkedHashSet<>();+ for (String each: gitLabProjects) {+ String apiEndpoint = importSource.getApiEndpoint("/projects/" + each.replace("/", "%2F") + "/labels");+ for (JsonNode labelNode: list(client, apiEndpoint, logger))+ labels.add(labelNode.get("name").asText());+ }++ for (String label: labels) {+ IssueLabelMapping mapping = new IssueLabelMapping();+ mapping.setGitLabIssueLabel(label);+ importOption.getIssueLabelMappings().add(mapping);+ }+ } finally {+ client.close();+ }+ return importOption;+ }++ @Nullable+ static User getUser(Client client, GitLabProjectImportSource importSource, Map<String, Optional<User>> users,+ String userId, SimpleLogger logger) {+ Optional<User> userOpt = users.get(userId);+ if (userOpt == null) {+ String apiEndpoint = importSource.getApiEndpoint("/users/" + userId);+ JsonNode userNode = get(client, apiEndpoint, logger);+ String email = null;+ if (userNode.hasNonNull("email"))+ email = userNode.get("email").asText(null);+ if (email == null && userNode.hasNonNull("public_email"))+ email = userNode.get("public_email").asText(null);+ if (email != null)+ userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByEmail(email));+ else+ userOpt = Optional.empty();+ users.put(userId, userOpt);+ }+ return userOpt.orElse(null);+ }++ static GitLabImportResult importIssues(GitLabProjectImportSource importSource, String gitLabProject, Project oneDevProject,+ boolean useExistingIssueNumbers, GitLabIssueImportOption importOption, Map<String, Optional<User>> users,+ boolean dryRun, SimpleLogger logger) {+ Client client = newClient(importSource);+ try {+ Set<String> nonExistentMilestones = new HashSet<>();+ Set<String> nonExistentLogins = new HashSet<>();+ Set<String> unmappedIssueLabels = new HashSet<>();+ Set<String> tooLargeAttachments = new LinkedHashSet<>();+ Set<String> errorAttachments = new HashSet<>();++ Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();+ Map<String, Milestone> milestoneMappings = new HashMap<>();++ for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);+ if (fieldSpec == null)+ throw new ExplicitException("No field spec found: " + oneDevFieldName);+ labelMappings.put(mapping.getGitLabIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));+ }++ for (Milestone milestone: oneDevProject.getMilestones())+ milestoneMappings.put(milestone.getName(), milestone);++ String initialIssueState = getIssueSetting().getInitialStateSpec().getName();++ List<Issue> issues = new ArrayList<>();++ Map<Long, Long> issueNumberMappings = new HashMap<>();++ AtomicInteger numOfImportedIssues = new AtomicInteger(0);+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {++ @Nullable+ private String processAttachments(String issueUUID, String issueFQN, String markdown,+ String attachmentRootUrl, Set<String> tooLargeAttachments) {+ StringBuffer buffer = new StringBuffer();+ Matcher matcher = PATTERN_ATTACHMENT.matcher(markdown);+ while (matcher.find()) {+ String attachmentUrl = attachmentRootUrl + matcher.group(2);+ String attachmentName = StringUtils.substringAfterLast(attachmentUrl, "/");+ WebTarget target = client.target(attachmentUrl);+ Invocation.Builder builder = target.request();+ try (Response response = builder.get()) {+ String errorMessage = JerseyUtils.checkStatus(attachmentUrl, response);+ if (errorMessage != null) {+ logger.error("Error downloading attachment: " + errorMessage);+ errorAttachments.add(attachmentUrl);+ } else {+ try (InputStream is = response.readEntity(InputStream.class)) {+ String oneDevAttachmentName = oneDevProject.saveAttachment(issueUUID, attachmentName, is);+ String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);+ matcher.appendReplacement(buffer, "[" + matcher.group(1) + "](" + oneDevAttachmentUrl + ")");+ } catch (AttachmentTooLargeException ex) {+ tooLargeAttachments.add(issueFQN + ":" + matcher.group(2));+ } catch (IOException e) {+ logger.error("Error downloading attachment", e);+ errorAttachments.add(attachmentUrl);+ }+ }+ }+ }+ matcher.appendTail(buffer);++ return buffer.toString();+ }++ private String joinAsMultilineHtml(List<String> values) {+ List<String> escapedValues = new ArrayList<>();+ for (String value: values)+ escapedValues.add(HtmlEscape.escapeHtml5(value));+ return StringUtils.join(escapedValues, "<br>");+ }++ @Override+ public void consume(List<JsonNode> pageData) throws InterruptedException {+ for (JsonNode issueNode: pageData) {+ if (Thread.interrupted())+ throw new InterruptedException();++ Map<String, String> extraIssueInfo = new LinkedHashMap<>();++ Issue issue = new Issue();+ issue.setProject(oneDevProject);+ issue.setTitle(issueNode.get("title").asText());+ issue.setDescription(issueNode.get("description").asText(null));++ issue.setNumberScope(oneDevProject);++ Long oldNumber = issueNode.get("iid").asLong();+ Long newNumber;+ if (dryRun || useExistingIssueNumbers)+ newNumber = oldNumber;+ else+ newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);+ issue.setNumber(newNumber);+ issueNumberMappings.put(oldNumber, newNumber);++ String issueFQN = gitLabProject + "#" + oldNumber;++ if (issueNode.get("state").asText().equals("closed"))+ issue.setState(importOption.getClosedIssueState());+ else+ issue.setState(initialIssueState);++ if (issueNode.hasNonNull("milestone")) {+ String milestoneName = issueNode.get("milestone").get("title").asText();+ Milestone milestone = milestoneMappings.get(milestoneName);+ if (milestone != null) {+ issue.setMilestone(milestone);+ } else {+ extraIssueInfo.put("Milestone", milestoneName);+ nonExistentMilestones.add(milestoneName);+ }+ }++ JsonNode authorNode = issueNode.get("author");+ User user = getUser(client, importSource, users, authorNode.get("id").asText(), logger);+ if (user != null) {+ issue.setSubmitter(user);+ } else {+ issue.setSubmitterName(authorNode.get("name").asText());+ nonExistentLogins.add(authorNode.get("username").asText());+ }++ issue.setSubmitDate(ISODateTimeFormat.dateTime()+ .parseDateTime(issueNode.get("created_at").asText())+ .toDate());++ LastUpdate lastUpdate = new LastUpdate();+ lastUpdate.setActivity("Opened");+ lastUpdate.setDate(issue.getSubmitDate());+ lastUpdate.setUser(issue.getSubmitter());+ lastUpdate.setUserName(issue.getSubmitterName());+ issue.setLastUpdate(lastUpdate);++ List<JsonNode> assigneeNodes = new ArrayList<>();+ if (issueNode.hasNonNull("assignees")) {+ for (JsonNode assigneeNode: issueNode.get("assignees"))+ assigneeNodes.add(assigneeNode);+ } else {+ assigneeNodes.add(issueNode.get("assignee"));+ }++ for (JsonNode assigneeNode: assigneeNodes) {+ IssueField assigneeField = new IssueField();+ assigneeField.setIssue(issue);+ assigneeField.setName(importOption.getAssigneesIssueField());+ assigneeField.setType(InputSpec.USER);++ user = getUser(client, importSource, users, assigneeNode.get("id").asText(), logger);+ if (user != null) {+ assigneeField.setValue(user.getName());+ } else {+ String assigneeLogin = assigneeNode.get("username").asText();+ assigneeField.setValue(assigneeLogin);+ nonExistentLogins.add(assigneeLogin);+ }+ issue.getFields().add(assigneeField);+ }++ if (importOption.getDueDateIssueField() != null) {+ String dueDate = issueNode.get("due_date").asText(null);+ if (dueDate != null) {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(importOption.getDueDateIssueField());+ issueField.setType(InputSpec.DATE);+ issueField.setValue(dueDate);+ issue.getFields().add(issueField);+ }+ }++ JsonNode timeStatsNode = issueNode.get("time_stats");+ if (importOption.getEstimatedTimeIssueField() != null) {+ int value = timeStatsNode.get("time_estimate").asInt();+ if (value != 0) {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(importOption.getEstimatedTimeIssueField());+ issueField.setType(InputSpec.WORKING_PERIOD);+ issueField.setValue(DateUtils.formatWorkingPeriod(value/60));+ issue.getFields().add(issueField);+ }+ }+ if (importOption.getSpentTimeIssueField() != null) {+ int value = timeStatsNode.get("total_time_spent").asInt();+ if (value != 0) {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(importOption.getSpentTimeIssueField());+ issueField.setType(InputSpec.WORKING_PERIOD);+ issueField.setValue(DateUtils.formatWorkingPeriod(value/60));+ issue.getFields().add(issueField);+ }+ }++ List<String> currentUnmappedLabels = new ArrayList<>();+ for (JsonNode labelNode: issueNode.get("labels")) {+ String labelName = labelNode.asText();+ Pair<FieldSpec, String> mapped = labelMappings.get(labelName);+ if (mapped != null) {+ IssueField tagField = new IssueField();+ tagField.setIssue(issue);+ tagField.setName(mapped.getFirst().getName());+ tagField.setType(InputSpec.ENUMERATION);+ tagField.setValue(mapped.getSecond());+ tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));+ issue.getFields().add(tagField);+ } else {+ currentUnmappedLabels.add(HtmlEscape.escapeHtml5(labelName));+ unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));+ }+ }++ if (!currentUnmappedLabels.isEmpty())+ extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));++ String webUrl = issueNode.get("web_url").asText();+ String attachmentRootUrl = StringUtils.substringBeforeLast(webUrl, "/-");+ if (!dryRun && issue.getDescription() != null) {+ issue.setDescription(processAttachments(issue.getUUID(), issueFQN,+ issue.getDescription(), attachmentRootUrl, tooLargeAttachments));+ }++ String apiEndpoint = importSource.getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F")+ + "/issues/" + oldNumber + "/notes?sort=asc");+ for (JsonNode noteNode: list(client, apiEndpoint, logger)) {+ if (!noteNode.get("system").asBoolean()) {+ String commentContent = noteNode.get("body").asText(null);+ if (commentContent != null) {+ if (!dryRun) {+ commentContent = processAttachments(issue.getUUID(), issueFQN,+ commentContent, attachmentRootUrl, tooLargeAttachments);+ }++ IssueComment comment = new IssueComment();+ comment.setIssue(issue);+ comment.setContent(commentContent);+ comment.setDate(ISODateTimeFormat.dateTime()+ .parseDateTime(noteNode.get("created_at").asText())+ .toDate());++ authorNode = noteNode.get("author");+ user = getUser(client, importSource, users, authorNode.get("id").asText(), logger);+ if (user != null) {+ comment.setUser(user);+ } else {+ comment.setUserName(authorNode.get("name").asText());+ nonExistentLogins.add(authorNode.get("username").asText());+ }+ issue.getComments().add(comment);+ }+ }+ }++ issue.setCommentCount(issue.getComments().size());++ Set<String> fieldAndValues = new HashSet<>();+ for (IssueField field: issue.getFields()) {+ String fieldAndValue = field.getName() + "::" + field.getValue();+ if (!fieldAndValues.add(fieldAndValue)) {+ String errorMessage = String.format(+ "Duplicate issue field mapping (issue: #%d, field: %s)",+ issueFQN, fieldAndValue);+ throw new ExplicitException(errorMessage);+ }+ }++ if (!extraIssueInfo.isEmpty()) {+ StringBuilder builder = new StringBuilder("|");+ for (String key: extraIssueInfo.keySet())+ builder.append(key).append("|");+ builder.append("\n|");+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));+ builder.append("\n|");+ for (String value: extraIssueInfo.values())+ builder.append(value).append("|");++ if (issue.getDescription() != null)+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());+ else+ issue.setDescription(builder.toString());+ }+ issues.add(issue);+ }+ logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");+ }++ };++ String apiEndpoint = importSource.getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F") + "/issues?sort=asc");+ list(client, apiEndpoint, pageDataConsumer, logger);++ if (!dryRun) {+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);+ for (Issue issue: issues) {+ if (issue.getDescription() != null)+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));++ OneDev.getInstance(IssueManager.class).save(issue);+ for (IssueField field: issue.getFields())+ OneDev.getInstance(Dao.class).persist(field);+ for (IssueComment comment: issue.getComments()) {+ comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));+ OneDev.getInstance(Dao.class).persist(comment);+ }+ }+ }++ GitLabImportResult result = new GitLabImportResult();+ result.nonExistentLogins.addAll(nonExistentLogins);+ result.nonExistentMilestones.addAll(nonExistentMilestones);+ result.unmappedIssueLabels.addAll(unmappedIssueLabels);+ result.tooLargeAttachments.addAll(tooLargeAttachments);+ result.errorAttachments.addAll(errorAttachments);++ return result;+ } finally {+ client.close();+ }+ }++ static GlobalIssueSetting getIssueSetting() {+ return OneDev.getInstance(SettingManager.class).getIssueSetting();+ }++ static List<JsonNode> list(Client client, String apiEndpoint, SimpleLogger logger) {+ List<JsonNode> result = new ArrayList<>();+ list(client, apiEndpoint, new PageDataConsumer() {++ @Override+ public void consume(List<JsonNode> pageData) {+ result.addAll(pageData);+ }++ }, logger);+ return result;+ }++ static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,+ SimpleLogger logger) {+ URI uri;+ try {+ uri = new URIBuilder(apiEndpoint)+ .addParameter("per_page", String.valueOf(PER_PAGE)).build();+ } catch (URISyntaxException e) {+ throw new RuntimeException(e);+ }++ int page = 1;+ while (true) {+ try {+ URIBuilder builder = new URIBuilder(uri);+ builder.addParameter("page", String.valueOf(page));+ List<JsonNode> pageData = new ArrayList<>();+ for (JsonNode each: get(client, builder.build().toString(), logger))+ pageData.add(each);+ pageDataConsumer.consume(pageData);+ if (pageData.size() < PER_PAGE)+ break;+ page++;+ } catch (URISyntaxException|InterruptedException e) {+ throw new RuntimeException(e);+ }+ }+ }++ static JsonNode get(Client client, String apiEndpoint, SimpleLogger logger) {+ WebTarget target = client.target(apiEndpoint);+ Invocation.Builder builder = target.request();+ while (true) {+ try (Response response = builder.get()) {+ int status = response.getStatus();+ if (status != 200) {+ String errorMessage = response.readEntity(String.class);+ if (StringUtils.isNotBlank(errorMessage)) {+ throw new ExplicitException(String.format("Http request failed (url: %s, status code: %d, error message: %s)",+ apiEndpoint, status, errorMessage));+ } else {+ throw new ExplicitException(String.format("Http request failed (status: %s)", status));+ }+ }+ return response.readEntity(JsonNode.class);+ }+ }+ }++ static Client newClient(GitLabProjectImportSource importSource) {+ Client client = ClientBuilder.newClient();+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);+ client.register(OAuth2ClientSupport.feature(importSource.getAccessToken()));+ return client;+ }++ static interface PageDataConsumer {++ void consume(List<JsonNode> pageData) throws InterruptedException;++ }++}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImportOption.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImportOption.javanew file mode 100644index 0000000000..3886ab8266--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImportOption.java@@ -0,0 +1,145 @@+package io.onedev.server.plugin.imports.gitlab;++import java.io.Serializable;+import java.util.ArrayList;+import java.util.List;+import java.util.stream.Collectors;++import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.SettingManager;+import io.onedev.server.model.support.administration.GlobalIssueSetting;+import io.onedev.server.model.support.inputspec.InputSpec;+import io.onedev.server.model.support.issue.field.spec.FieldSpec;+import io.onedev.server.web.editable.annotation.ChoiceProvider;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public class GitLabIssueImportOption implements Serializable {++ private static final long serialVersionUID = 1L;++ private String closedIssueState;++ private String assigneesIssueField;++ private String dueDateIssueField;++ private String estimatedTimeIssueField;++ private String spentTimeIssueField;++ private List<IssueLabelMapping> issueLabelMappings = new ArrayList<>();++ @Editable(order=300, description="Specify which issue state to use for closed GitLab issues.<br>"+ + "<b>NOTE: </b> You may customize OneDev issue states in case there is no appropriate option here")+ @ChoiceProvider("getCloseStateChoices")+ @NotEmpty+ public String getClosedIssueState() {+ return closedIssueState;+ }++ public void setClosedIssueState(String closedIssueState) {+ this.closedIssueState = closedIssueState;+ }++ private static GlobalIssueSetting getIssueSetting() {+ return OneDev.getInstance(SettingManager.class).getIssueSetting();+ }++ @SuppressWarnings("unused")+ private static List<String> getCloseStateChoices() {+ List<String> choices = getIssueSetting().getStateSpecs().stream()+ .map(it->it.getName()).collect(Collectors.toList());+ choices.remove(0);+ return choices;+ }++ @Editable(order=350, description="Specify a multi-value user field to hold assignees information.<br>"+ + "<b>NOTE: </b> You may customize OneDev issue fields in case there is no appropriate option here")+ @ChoiceProvider("getAssigneesIssueFieldChoices")+ @NotEmpty+ public String getAssigneesIssueField() {+ return assigneesIssueField;+ }++ public void setAssigneesIssueField(String assigneesIssueField) {+ this.assigneesIssueField = assigneesIssueField;+ }++ @SuppressWarnings("unused")+ private static List<String> getAssigneesIssueFieldChoices() {+ List<String> choices = new ArrayList<>();+ for (FieldSpec field: getIssueSetting().getFieldSpecs()) {+ if (field.getType().equals(InputSpec.USER) && field.isAllowMultiple())+ choices.add(field.getName());+ }+ return choices;+ }++ @Editable(order=360, description="Optionally specify a date field to hold due date information.<br>"+ + "<b>NOTE: </b> You may customize OneDev issue fields in case there is no appropriate option here")+ @ChoiceProvider("getDueDateIssueFieldChoices")+ public String getDueDateIssueField() {+ return dueDateIssueField;+ }++ public void setDueDateIssueField(String dueDateIssueField) {+ this.dueDateIssueField = dueDateIssueField;+ }++ @SuppressWarnings("unused")+ private static List<String> getDueDateIssueFieldChoices() {+ List<String> choices = new ArrayList<>();+ for (FieldSpec field: getIssueSetting().getFieldSpecs()) {+ if (field.getType().equals(InputSpec.DATE))+ choices.add(field.getName());+ }+ return choices;+ }++ @Editable(order=370, description="Optionally specify a working period field to hold estimated time infomration.<br>"+ + "<b>NOTE: </b> You may customize OneDev issue fields in case there is no appropriate option here")+ @ChoiceProvider("getWorkingPeriodIssueFieldChoices")+ public String getEstimatedTimeIssueField() {+ return estimatedTimeIssueField;+ }++ public void setEstimatedTimeIssueField(String estimatedTimeIssueField) {+ this.estimatedTimeIssueField = estimatedTimeIssueField;+ }++ @Editable(order=380, description="Optionally specify a working period field to hold spent time infomration.<br>"+ + "<b>NOTE: </b> You may customize OneDev issue fields in case there is no appropriate option here")+ @ChoiceProvider("getWorkingPeriodIssueFieldChoices")+ public String getSpentTimeIssueField() {+ return spentTimeIssueField;+ }++ public void setSpentTimeIssueField(String spentTimeIssueField) {+ this.spentTimeIssueField = spentTimeIssueField;+ }++ @SuppressWarnings("unused")+ private static List<String> getWorkingPeriodIssueFieldChoices() {+ List<String> choices = new ArrayList<>();+ for (FieldSpec field: getIssueSetting().getFieldSpecs()) {+ if (field.getType().equals(InputSpec.WORKING_PERIOD))+ choices.add(field.getName());+ }+ return choices;+ }++ @Editable(order=400, description="Specify how to map GitLab issue labels to OneDev custom "+ + "fields. Unmapped labels will be reflected in issue description.<br>"+ + "<b>NOTE: </b> You may customize OneDev issue fields in case there is no appropriate option here")+ public List<IssueLabelMapping> getIssueLabelMappings() {+ return issueLabelMappings;+ }++ public void setIssueLabelMappings(List<IssueLabelMapping> issueLabelMappings) {+ this.issueLabelMappings = issueLabelMappings;+ }++}diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImportSource.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImportSource.javanew file mode 100644index 0000000000..24e760a68d--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImportSource.java@@ -0,0 +1,86 @@+package io.onedev.server.plugin.imports.gitlab;++import static io.onedev.server.plugin.imports.gitlab.GitLabImportUtils.get;+import static io.onedev.server.plugin.imports.gitlab.GitLabImportUtils.list;++import java.util.ArrayList;+import java.util.Collections;+import java.util.List;++import javax.ws.rs.client.Client;+import javax.ws.rs.client.ClientBuilder;++import org.glassfish.jersey.client.oauth2.OAuth2ClientSupport;+import org.hibernate.validator.constraints.NotEmpty;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import com.fasterxml.jackson.databind.JsonNode;++import io.onedev.commons.utils.StringUtils;+import io.onedev.server.buildspec.job.log.StyleBuilder;+import io.onedev.server.util.EditContext;+import io.onedev.server.util.SimpleLogger;+import io.onedev.server.util.validation.annotation.ClassValidating;+import io.onedev.server.web.editable.annotation.ChoiceProvider;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+@ClassValidating+public class GitLabIssueImportSource extends GitLabProjectImportSource {++ private static final long serialVersionUID = 1L;++ private static final Logger logger = LoggerFactory.getLogger(GitLabIssueImportSource.class);++ private String project;++ @Editable(order=400, name="GitLab Project", description="Choose GitLab project to "+ + "import issues from. Available projects will be populated when above options "+ + "are specified")+ @ChoiceProvider("getProjectChoices")+ @NotEmpty+ public String getProject() {+ return project;+ }++ public void setProject(String project) {+ this.project = project;+ }++ @SuppressWarnings("unused")+ private static List<String> getProjectChoices() {+ List<String> choices = new ArrayList<>();++ String apiUrl = (String) EditContext.get().getInputValue(PROP_API_URL);+ String accessToken = (String) EditContext.get().getInputValue(PROP_ACCESS_TOKEN);++ if (StringUtils.isNotBlank(apiUrl) && StringUtils.isNotBlank(accessToken)) {+ Client client = ClientBuilder.newClient();+ client.register(OAuth2ClientSupport.feature(accessToken));+ try {+ String apiEndpoint = getApiEndpoint(apiUrl, "/user");+ SimpleLogger logger = new SimpleLogger() {++ @Override+ public void log(String message, StyleBuilder styleBuilder) {+ GitLabIssueImportSource.logger.info(message);+ }++ };+ get(client, apiEndpoint, logger);++ apiEndpoint = getApiEndpoint(apiUrl, "/projects?membership=true");+ for (JsonNode projectNode: list(client, apiEndpoint, logger))+ choices.add(projectNode.get("path_with_namespace").asText());+ } catch (Exception e) {+ } finally {+ client.close();+ }+ }++ Collections.sort(choices);+ return choices;+ }++}diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.javanew file mode 100644index 0000000000..6509547b37--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.java@@ -0,0 +1,39 @@+package io.onedev.server.plugin.imports.gitlab;++import static io.onedev.server.plugin.imports.gitlab.GitLabImportUtils.NAME;+import static io.onedev.server.plugin.imports.gitlab.GitLabImportUtils.buildImportOption;+import static io.onedev.server.plugin.imports.gitlab.GitLabImportUtils.importIssues;++import java.util.HashMap;+import java.util.Map;+import java.util.Optional;++import io.onedev.server.model.Project;+import io.onedev.server.model.User;+import io.onedev.server.util.SimpleLogger;+import io.onedev.server.web.page.project.issues.imports.IssueImporter;++public class GitLabIssueImporter extends IssueImporter<GitLabIssueImportSource, GitLabIssueImportOption> {++ private static final long serialVersionUID = 1L;++ @Override+ public String getName() {+ return NAME;+ }++ @Override+ public GitLabIssueImportOption getImportOption(GitLabIssueImportSource importSource, SimpleLogger logger) {+ return buildImportOption(importSource, importSource.getProject(), logger);+ }++ @Override+ public String doImport(Project project, GitLabIssueImportSource importSource, GitLabIssueImportOption importOption,+ boolean dryRun, SimpleLogger logger) {+ logger.log("Importing issues from project " + importSource.getProject() + "...");+ Map<String, Optional<User>> users = new HashMap<>();+ return importIssues(importSource, importSource.getProject(), project, false, importOption, users, dryRun, logger)+ .toHtml("Issues imported successfully");+ }++}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabPluginModule.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabPluginModule.javanew file mode 100644index 0000000000..aa3dbd53e7--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabPluginModule.java@@ -0,0 +1,53 @@+package io.onedev.server.plugin.imports.gitlab;++import java.io.Serializable;+import java.util.Collection;++import com.google.common.collect.Lists;++import io.onedev.commons.launcher.loader.AbstractPluginModule;+import io.onedev.server.web.page.project.imports.ProjectImporter;+import io.onedev.server.web.page.project.imports.ProjectImporterContribution;+import io.onedev.server.web.page.project.issues.imports.IssueImporter;+import io.onedev.server.web.page.project.issues.imports.IssueImporterContribution;++/**+ * NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.+ *+ */+public class GitLabPluginModule extends AbstractPluginModule {++ @Override+ protected void configure() {+ super.configure();++ contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {++ @Override+ public Collection<ProjectImporter<? extends Serializable, ? extends Serializable>> getImporters() {+ return Lists.newArrayList(new GitLabProjectImporter());+ }++ @Override+ public int getOrder() {+ return 200;+ }++ });++ contribute(IssueImporterContribution.class, new IssueImporterContribution() {++ @Override+ public Collection<IssueImporter<? extends Serializable, ? extends Serializable>> getImporters() {+ return Lists.newArrayList(new GitLabIssueImporter());+ }++ @Override+ public int getOrder() {+ return 200;+ }++ });+ }++}diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImportOption.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImportOption.javanew file mode 100644index 0000000000..c82bf68fb8--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImportOption.java@@ -0,0 +1,89 @@+package io.onedev.server.plugin.imports.gitlab;++import java.util.ArrayList;+import java.util.List;++import javax.annotation.Nullable;+import javax.validation.ConstraintValidatorContext;+import javax.validation.constraints.Size;++import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.entitymanager.RoleManager;+import io.onedev.server.model.Role;+import io.onedev.server.util.validation.Validatable;+import io.onedev.server.util.validation.annotation.ClassValidating;+import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.RoleChoice;++@Editable+@ClassValidating+public class GitLabProjectImportOption extends GitLabIssueImportOption implements Validatable {++ private static final long serialVersionUID = 1L;++ private List<ProjectMapping> projectMappings = new ArrayList<>();++ private String publicRoleName;++ @Editable(order=200, name="Public Role", description="If specified, all public and internal projects imported from GitLab "+ + "will use this as default role. Private projects are not affected")+ @RoleChoice+ @Nullable+ public String getPublicRoleName() {+ return publicRoleName;+ }++ public void setPublicRoleName(String publicRoleName) {+ this.publicRoleName = publicRoleName;+ }++ @Nullable+ public Role getPublicRole() {+ if (publicRoleName != null)+ return OneDev.getInstance(RoleManager.class).find(publicRoleName);+ else+ return null;+ }++ @Editable(order=100, name="Projects to Import")+ @Size(min=1, max=10000, message="No projects to import")+ public List<ProjectMapping> getProjectMappings() {+ return projectMappings;+ }++ public void setProjectMappings(List<ProjectMapping> projectMappings) {+ this.projectMappings = projectMappings;+ }++ @Override+ public boolean isValid(ConstraintValidatorContext context) {+ boolean isValid = true;+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);+ for (int i=0; i<projectMappings.size(); i++) {+ if (projectManager.find(projectMappings.get(i).getOneDevProject()) != null) {+ context.buildConstraintViolationWithTemplate("Project name already used")+ .addPropertyNode("projectMappings")+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)+ .inIterable().atIndex(i).addConstraintViolation();+ isValid = false;+ } else {+ for (int j=0; j<projectMappings.size(); j++) {+ if (j != i && projectMappings.get(j).getOneDevProject().equals(projectMappings.get(i).getOneDevProject())) {+ context.buildConstraintViolationWithTemplate("Duplicate project name")+ .addPropertyNode("projectMappings")+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)+ .inIterable().atIndex(i).addConstraintViolation();+ isValid = false;+ break;+ }+ }+ }+ }++ if (!isValid)+ context.disableDefaultConstraintViolation();+ return isValid;+ }++}diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImportSource.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImportSource.javanew file mode 100644index 0000000000..ea32e75093--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImportSource.java@@ -0,0 +1,110 @@+package io.onedev.server.plugin.imports.gitlab;++import java.io.Serializable;++import javax.validation.ConstraintValidatorContext;+import javax.ws.rs.client.Client;+import javax.ws.rs.client.ClientBuilder;+import javax.ws.rs.client.Invocation;+import javax.ws.rs.client.WebTarget;+import javax.ws.rs.core.Response;++import org.glassfish.jersey.client.oauth2.OAuth2ClientSupport;+import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.commons.utils.StringUtils;+import io.onedev.server.util.JerseyUtils;+import io.onedev.server.util.validation.Validatable;+import io.onedev.server.util.validation.annotation.ClassValidating;+import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.Password;++@Editable+@ClassValidating+public class GitLabProjectImportSource implements Serializable, Validatable {++ private static final long serialVersionUID = 1L;++ static final String PROP_API_URL = "apiUrl";++ static final String PROP_ACCESS_TOKEN = "accessToken";++ private String apiUrl;++ private String accessToken;++ @Editable(order=10, name="GitLab API URL", description="Specify GitLab API url, for instance <tt>https://gitlab.example.com/api/v4</tt>")+ @NotEmpty+ public String getApiUrl() {+ return apiUrl;+ }++ public void setApiUrl(String apiUrl) {+ this.apiUrl = apiUrl;+ }++ @Editable(order=100, name="GitLab Personal Access Token", description="GitLab personal access token should be generated with "+ + "scope <b>read_api</b>, <b>read_user</b> and <b>read_repository</b>")+ @Password+ @NotEmpty+ public String getAccessToken() {+ return accessToken;+ }++ public void setAccessToken(String accessToken) {+ this.accessToken = accessToken;+ }++ public static String getApiEndpoint(String apiUrl, String apiPath) {+ return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");+ }++ public String getApiEndpoint(String apiPath) {+ return getApiEndpoint(apiUrl, apiPath);+ }++ @Override+ public boolean isValid(ConstraintValidatorContext context) {+ Client client = ClientBuilder.newClient();+ client.register(OAuth2ClientSupport.feature(accessToken));+ try {+ String apiEndpoint = getApiEndpoint("/user");+ WebTarget target = client.target(apiEndpoint);+ Invocation.Builder builder = target.request();+ try (Response response = builder.get()) {+ if (!response.getMediaType().toString().startsWith("application/json")) {+ context.disableDefaultConstraintViolation();+ context.buildConstraintViolationWithTemplate("This does not seem like a GitLab api url")+ .addPropertyNode(PROP_API_URL).addConstraintViolation();+ return false;+ } else if (response.getStatus() == 401) {+ context.disableDefaultConstraintViolation();+ String errorMessage = "Authentication failed";+ context.buildConstraintViolationWithTemplate(errorMessage)+ .addPropertyNode(PROP_ACCESS_TOKEN).addConstraintViolation();+ return false;+ } else {+ String errorMessage = JerseyUtils.checkStatus(apiEndpoint, response);+ if (errorMessage != null) {+ context.disableDefaultConstraintViolation();+ context.buildConstraintViolationWithTemplate(errorMessage)+ .addPropertyNode(PROP_API_URL).addConstraintViolation();+ return false;+ }+ }+ }+ } catch (Exception e) {+ context.disableDefaultConstraintViolation();+ String errorMessage = "Error connecting api service";+ if (e.getMessage() != null)+ errorMessage += ": " + e.getMessage();+ context.buildConstraintViolationWithTemplate(errorMessage)+ .addPropertyNode(PROP_API_URL).addConstraintViolation();+ return false;+ } finally {+ client.close();+ }+ return true;+ }++}diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.javanew file mode 100644index 0000000000..1377e1ae1e--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.java@@ -0,0 +1,169 @@+package io.onedev.server.plugin.imports.gitlab;++import static io.onedev.server.plugin.imports.gitlab.GitLabImportUtils.NAME;+import static io.onedev.server.plugin.imports.gitlab.GitLabImportUtils.buildImportOption;+import static io.onedev.server.plugin.imports.gitlab.GitLabImportUtils.get;+import static io.onedev.server.plugin.imports.gitlab.GitLabImportUtils.importIssues;+import static io.onedev.server.plugin.imports.gitlab.GitLabImportUtils.list;+import static io.onedev.server.plugin.imports.gitlab.GitLabImportUtils.newClient;++import java.util.ArrayList;+import java.util.Collection;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.Optional;++import javax.ws.rs.client.Client;++import org.apache.http.client.utils.URIBuilder;+import org.joda.time.format.ISODateTimeFormat;++import com.fasterxml.jackson.databind.JsonNode;++import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.MilestoneManager;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.model.Milestone;+import io.onedev.server.model.Project;+import io.onedev.server.model.User;+import io.onedev.server.storage.StorageManager;+import io.onedev.server.util.SimpleLogger;+import io.onedev.server.web.page.project.imports.ProjectImporter;++public class GitLabProjectImporter extends ProjectImporter<GitLabProjectImportSource, GitLabProjectImportOption> {++ private static final long serialVersionUID = 1L;++ @Override+ public String getName() {+ return NAME;+ }++ @Override+ public GitLabProjectImportOption getImportOption(GitLabProjectImportSource importSource, SimpleLogger logger) {+ GitLabProjectImportOption importOption = new GitLabProjectImportOption();+ Client client = newClient(importSource);+ try {+ String apiEndpoint = importSource.getApiEndpoint("/projects?membership=true");+ for (JsonNode projectNode: list(client, apiEndpoint, logger)) {+ String pathWithNamespace = projectNode.get("path_with_namespace").asText();+ ProjectMapping projectMapping = new ProjectMapping();+ projectMapping.setGitLabProject(pathWithNamespace);+ projectMapping.setOneDevProject(pathWithNamespace.replace('/', '-'));+ importOption.getProjectMappings().add(projectMapping);+ }+ GitLabIssueImportOption issueImportOption = buildImportOption(importSource, null, logger);+ importOption.setAssigneesIssueField(issueImportOption.getAssigneesIssueField());+ importOption.setClosedIssueState(issueImportOption.getClosedIssueState());+ importOption.setIssueLabelMappings(issueImportOption.getIssueLabelMappings());+ } finally {+ client.close();+ }+ return importOption;+ }++ private List<Milestone> getMilestones(GitLabProjectImportSource importSource,+ String groupId, SimpleLogger logger) {+ Client client = newClient(importSource);+ try {+ List<Milestone> milestones = new ArrayList<>();+ String apiEndpoint = importSource.getApiEndpoint("/groups/" + groupId + "/milestones");+ for (JsonNode milestoneNode: list(client, apiEndpoint, logger))+ milestones.add(getMilestone(milestoneNode));+ apiEndpoint = importSource.getApiEndpoint("/groups/" + groupId);+ JsonNode groupNode = get(client, apiEndpoint, logger);+ JsonNode parentIdNode = groupNode.get("parent_id");+ if (parentIdNode != null && parentIdNode.asText(null) != null)+ milestones.addAll(getMilestones(importSource, parentIdNode.asText(), logger));+ return milestones;+ } finally {+ client.close();+ }+ }++ private Milestone getMilestone(JsonNode milestoneNode) {+ Milestone milestone = new Milestone();+ milestone.setName(milestoneNode.get("title").asText());+ milestone.setDescription(milestoneNode.get("description").asText(null));+ String dueDateString = milestoneNode.get("due_date").asText(null);+ if (dueDateString != null)+ milestone.setDueDate(ISODateTimeFormat.date().parseDateTime(dueDateString).toDate());+ if (milestoneNode.get("state").asText().equals("closed"))+ milestone.setClosed(true);+ return milestone;+ }++ @Override+ public String doImport(GitLabProjectImportSource importSource, GitLabProjectImportOption importOption,+ boolean dryRun, SimpleLogger logger) {+ Collection<Long> projectIds = new ArrayList<>();+ Client client = newClient(importSource);+ try {+ Map<String, Optional<User>> users = new HashMap<>();+ GitLabImportResult result = new GitLabImportResult();+ for (ProjectMapping projectMapping: importOption.getProjectMappings()) {+ logger.log("Cloning code from project " + projectMapping.getGitLabProject() + "...");++ String apiEndpoint = importSource.getApiEndpoint("/projects/" + projectMapping.getGitLabProject().replace("/", "%2F"));+ JsonNode projectNode = get(client, apiEndpoint, logger);+ Project project = new Project();+ project.setName(projectMapping.getOneDevProject());+ project.setDescription(projectNode.get("description").asText(null));+ project.setIssueManagementEnabled(projectNode.get("issues_enabled").asBoolean());++ String visibility = projectNode.get("visibility").asText();+ if (!visibility.equals("private") && importOption.getPublicRole() != null)+ project.setDefaultRole(importOption.getPublicRole());++ URIBuilder builder = new URIBuilder(projectNode.get("http_url_to_repo").asText());+ if (!visibility.equals("public"))+ builder.setUserInfo("git", importSource.getAccessToken());++ if (!dryRun) {+ OneDev.getInstance(ProjectManager.class).clone(project, builder.build().toString());+ projectIds.add(project.getId());+ }++ if (projectMapping.isImportIssues()) {+ List<Milestone> milestones = new ArrayList<>();+ logger.log("Importing milestones from project " + projectMapping.getGitLabProject() + "...");+ apiEndpoint = importSource.getApiEndpoint("/projects/"+ + projectMapping.getGitLabProject().replace("/", "%2F") + "/milestones");+ for (JsonNode milestoneNode: list(client, apiEndpoint, logger))+ milestones.add(getMilestone(milestoneNode));+ JsonNode namespaceNode = projectNode.get("namespace");+ if (namespaceNode.get("kind").asText().equals("group")) {+ String groupId = namespaceNode.get("id").asText();+ milestones.addAll(getMilestones(importSource, groupId, logger));+ }++ for (Milestone milestone: milestones) {+ milestone.setProject(project);+ project.getMilestones().add(milestone);+ if (!dryRun)+ OneDev.getInstance(MilestoneManager.class).save(milestone);+ }++ logger.log("Importing issues from project " + projectMapping.getGitLabProject() + "...");+ GitLabImportResult currentResult = importIssues(importSource, projectMapping.getGitLabProject(),+ project, true, importOption, users, dryRun, logger);+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);+ result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);+ result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);+ result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);+ result.errorAttachments.addAll(currentResult.errorAttachments);+ }+ }++ return result.toHtml("Projects imported successfully");+ } catch (Exception e) {+ for (Long projectId: projectIds)+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);+ throw new RuntimeException(e);+ } finally {+ client.close();+ }+ }++}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/IssueLabelMapping.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/IssueLabelMapping.javanew file mode 100644index 0000000000..59a2d64af4--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/IssueLabelMapping.java@@ -0,0 +1,60 @@+package io.onedev.server.plugin.imports.gitlab;++import java.io.Serializable;+import java.util.ArrayList;+import java.util.List;++import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.SettingManager;+import io.onedev.server.model.support.administration.GlobalIssueSetting;+import io.onedev.server.model.support.inputspec.InputSpec;+import io.onedev.server.model.support.issue.field.spec.FieldSpec;+import io.onedev.server.web.editable.annotation.ChoiceProvider;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public class IssueLabelMapping implements Serializable {++ private static final long serialVersionUID = 1L;++ private String gitLabIssueLabel;++ private String oneDevIssueField;++ @Editable(order=100, name="GitLab Issue Label")+ @NotEmpty+ public String getGitLabIssueLabel() {+ return gitLabIssueLabel;+ }++ public void setGitLabIssueLabel(String gitLabIssueLabel) {+ this.gitLabIssueLabel = gitLabIssueLabel;+ }++ @Editable(order=200, name="OneDev Issue Field", description="Specify a multi-value enum custom field")+ @ChoiceProvider("getOneDevIssueFieldChoices")+ @NotEmpty+ public String getOneDevIssueField() {+ return oneDevIssueField;+ }++ public void setOneDevIssueField(String oneDevIssueField) {+ this.oneDevIssueField = oneDevIssueField;+ }++ @SuppressWarnings("unused")+ private static List<String> getOneDevIssueFieldChoices() {+ List<String> choices = new ArrayList<>();+ GlobalIssueSetting issueSetting = OneDev.getInstance(SettingManager.class).getIssueSetting();+ for (FieldSpec field: issueSetting.getFieldSpecs()) {+ if (field.getType().equals(InputSpec.ENUMERATION)) {+ for (String value: field.getPossibleValues())+ choices.add(field.getName() + "::" + value);+ }+ }+ return choices;+ }++}diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ProjectMapping.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ProjectMapping.javanew file mode 100644index 0000000000..7c5d46f56f--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ProjectMapping.java@@ -0,0 +1,55 @@+package io.onedev.server.plugin.imports.gitlab;++import java.io.Serializable;++import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.server.util.validation.annotation.ProjectName;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public class ProjectMapping implements Serializable {++ private static final long serialVersionUID = 1L;++ public static final String PROP_ONEDEV_PROJECT = "oneDevProject";++ private String gitLabProejct;++ private String oneDevProject;++ private boolean importIssues = true;++ @Editable(order=100, name="GitLab Project", description="Specify GitLab project in form of "+ + "<tt>group/subgroup/project</tt>")+ @NotEmpty+ public String getGitLabProject() {+ return gitLabProejct;+ }++ public void setGitLabProject(String gitLabProject) {+ this.gitLabProejct = gitLabProject;+ }++ @Editable(order=200, name="OneDev Project", description="Specify OneDev project to be created as "+ + "result of importing")+ @ProjectName+ @NotEmpty+ public String getOneDevProject() {+ return oneDevProject;+ }++ public void setOneDevProject(String oneDevProject) {+ this.oneDevProject = oneDevProject;+ }++ @Editable(order=400)+ public boolean isImportIssues() {+ return importIssues;+ }++ public void setImportIssues(boolean importIssues) {+ this.importIssues = importIssues;+ }++}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/IssueTagMapping.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/IssueTagMapping.javaindex baad93b1b2..2e52b7f52b 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/IssueTagMapping.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/IssueTagMapping.java@@ -49,7 +49,7 @@ public class IssueTagMapping implements Serializable {List<String> choices = new ArrayList<>();GlobalIssueSetting issueSetting = OneDev.getInstance(SettingManager.class).getIssueSetting();for (FieldSpec field: issueSetting.getFieldSpecs()) {- if (field.getType().equals(InputSpec.ENUMERATION) && field.isAllowMultiple()) {+ if (field.getType().equals(InputSpec.ENUMERATION)) {for (String value: field.getPossibleValues())choices.add(field.getName() + "::" + value);}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportResult.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportResult.javaindex 872833c197..ef6357e125 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportResult.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportResult.java@@ -1,7 +1,10 @@package io.onedev.server.plugin.imports.youtrack;+import java.util.ArrayList;+import java.util.Collection;import java.util.LinkedHashMap;import java.util.LinkedHashSet;+import java.util.List;import java.util.Map;import java.util.Set;@@ -9,6 +12,8 @@ import org.unbescape.html.HtmlEscape;public class YouTrackImportResult {+ private static final int MAX_DISPLAY_ENTRIES = 100;+Set<String> nonExistentLogins = new LinkedHashSet<>();Set<String> unmappedIssueTags = new LinkedHashSet<>();@@ -21,6 +26,15 @@ public class YouTrackImportResult {Set<String> tooLargeAttachments = new LinkedHashSet<>();+ private String getEntryFeedback(String entryDescription, Collection<String> entries) {+ if (entries.size() > MAX_DISPLAY_ENTRIES) {+ List<String> entriesToDisplay = new ArrayList<>(entries).subList(0, MAX_DISPLAY_ENTRIES);+ return "<li> " + entryDescription + ": " + HtmlEscape.escapeHtml5(entriesToDisplay.toString()) + " and more";+ } else {+ return "<li> " + entryDescription + ": " + HtmlEscape.escapeHtml5(entries.toString());+ }+ }+public String toHtml(String leadingText) {StringBuilder feedback = new StringBuilder(HtmlEscape.escapeHtml5(leadingText));@@ -36,35 +50,38 @@ public class YouTrackImportResult {feedback.append("<br><br><b>NOTE:</b><ul>");if (!unmappedIssueStates.isEmpty()) {- feedback.append("<li> Unmapped YouTrack issue states (using OneDev initial state): "- + HtmlEscape.escapeHtml5(unmappedIssueStates.toString()));- }- if (!unmappedIssueFields.isEmpty()) {- feedback.append("<li> Unmapped YouTrack issue fields: "- + HtmlEscape.escapeHtml5(unmappedIssueFields.toString()));+ feedback.append(getEntryFeedback("Unmapped YouTrack issue states (using OneDev initial state)",+ unmappedIssueStates));}+ if (!unmappedIssueFields.isEmpty())+ feedback.append(getEntryFeedback("Unmapped YouTrack issue fields", unmappedIssueFields));if (!mismatchedIssueFields.isEmpty()) {feedback.append("<li> YouTrack issue fields mapped to wrong type of OneDev issue field: ");feedback.append("<ul>");++ int displayedEntries = 0;for (Map.Entry<String, String> entry: mismatchedIssueFields.entrySet()) {feedback.append("<li>").append(HtmlEscape.escapeHtml5(entry.getKey())).append(" : ").append(HtmlEscape.escapeHtml5(entry.getValue()));+ if (displayedEntries++ >= MAX_DISPLAY_ENTRIES)+ break;}+ if (mismatchedIssueFields.size() > MAX_DISPLAY_ENTRIES)+ feedback.append("<li> And more...");feedback.append("</ul>");}if (!unmappedIssueTags.isEmpty()) {- feedback.append("<li> YouTrack issue tags not mapped to OneDev custom field: "- + HtmlEscape.escapeHtml5(unmappedIssueTags.toString()));+ feedback.append(getEntryFeedback("YouTrack issue tags not mapped to OneDev custom field",+ unmappedIssueTags));}if (!nonExistentLogins.isEmpty()) {- feedback.append("<li> YouTrack logins without email or email can not be mapped to OneDev account: "- + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));+ feedback.append(getEntryFeedback("YouTrack logins without email or email can not be mapped to OneDev account",+ nonExistentLogins));}if (!tooLargeAttachments.isEmpty()) {- feedback.append("<li> Too large attachments: "- + HtmlEscape.escapeHtml5(tooLargeAttachments.toString()));+ feedback.append(getEntryFeedback("Too large attachments", tooLargeAttachments));}if (hasNotes)diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportUtils.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportUtils.javaindex cf2c9cb68e..fd848510b1 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportUtils.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportUtils.java@@ -7,6 +7,7 @@ import java.net.URISyntaxException;import java.util.ArrayList;import java.util.Date;import java.util.HashMap;+import java.util.HashSet;import java.util.LinkedHashMap;import java.util.LinkedHashSet;import java.util.List;@@ -756,6 +757,17 @@ public class YouTrackImportUtils {issue.setCommentCount(issue.getComments().size());+ Set<String> fieldAndValues = new HashSet<>();+ for (IssueField field: issue.getFields()) {+ String fieldAndValue = field.getName() + "::" + field.getValue();+ if (!fieldAndValues.add(fieldAndValue)) {+ String errorMessage = String.format(+ "Duplicate issue field mapping (issue: #%d, field: %s)",+ readableId, fieldAndValue);+ throw new ExplicitException(errorMessage);+ }+ }+if (!extraIssueInfo.isEmpty()) {StringBuilder builder = new StringBuilder("|");for (String key: extraIssueInfo.keySet())diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportOption.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportOption.javaindex 327e3a5c76..080239e9d4 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportOption.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportOption.java@@ -18,7 +18,8 @@ public class YouTrackIssueImportOption implements Serializable {private List<IssueTagMapping> issueTagMappings = new ArrayList<>();@Editable(order=200, description="Specify how to map YouTrack issue state to OneDev issue state. "- + "Unmapped states will use the initial state in OneDev")+ + "Unmapped states will use the initial state in OneDev.<br>"+ + "<b>NOTE: </b> You may customize OneDev issue states in case there is no appropriate option here")public List<IssueStateMapping> getIssueStateMappings() {return issueStateMappings;}@@ -27,10 +28,14 @@ public class YouTrackIssueImportOption implements Serializable {this.issueStateMappings = issueStateMappings;}- @Editable(order=300, description="Specify how to map YouTrack issue custom fields to OneDev issue "- + "custom fields. Unmapped fields will be reflected in issue description.<br>"- + "<b>Note: </b>Enum field needs to be mapped in form of <tt><Field Name>::<Field Value></tt>, "- + "for instance <tt>Priority::Critical</tt>")+ @Editable(order=300, description="Specify how to map YouTrack issue fields to OneDev. Unmapped fields will "+ + "be reflected in issue description.<br>"+ + "<b>Note: </b>"+ + "<ul>"+ + "<li>Enum field needs to be mapped in form of <tt><Field Name>::<Field Value></tt>, "+ + "for instance <tt>Priority::Critical</tt>"+ + "<li>You may customize OneDev issue fields in case there is no appropriate option here"+ + "</ul>")public List<IssueFieldMapping> getIssueFieldMappings() {return issueFieldMappings;}@@ -40,8 +45,8 @@ public class YouTrackIssueImportOption implements Serializable {}@Editable(order=300, description="Specify how to map YouTrack issue tags to OneDev issue custom "- + "fields. Only multi-valued enum field can be used here. Unmapped tags will be "- + "reflected in issue description")+ + "fields. Unmapped tags will be reflected in issue description.<br>"+ + "<b>NOTE: </b> You may customize OneDev issue fields in case there is no appropriate option here")public List<IssueTagMapping> getIssueTagMappings() {return issueTagMappings;}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.javaindex cc7bbdc6df..5e0c0273fd 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java@@ -1,6 +1,10 @@package io.onedev.server.plugin.imports.youtrack;-import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.*;+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.NAME;+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.buildImportOption;+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.importIssues;+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.list;+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.newClient;import javax.ws.rs.client.Client;@@ -37,12 +41,8 @@ public class YouTrackIssueImporter extends IssueImporter<YouTrackIssueImportSour@Overridepublic YouTrackIssueImportOption getImportOption(YouTrackIssueImportSource importSource, SimpleLogger logger) {- if (importSource.isPrepopulateImportOptions()) {- String youTrackProjectId = getYouTrackProjectId(importSource, importSource.getProject(), logger);- return buildImportOption(importSource, youTrackProjectId, logger);- } else {- return new YouTrackIssueImportOption();- }+ String youTrackProjectId = getYouTrackProjectId(importSource, importSource.getProject(), logger);+ return buildImportOption(importSource, youTrackProjectId, logger);}@Overridediff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.javaindex 4f7170b203..a789f9da0d 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java@@ -6,8 +6,6 @@ import java.util.Collection;import com.google.common.collect.Lists;import io.onedev.commons.launcher.loader.AbstractPluginModule;-import io.onedev.server.web.page.project.imports.ProjectImporter;-import io.onedev.server.web.page.project.imports.ProjectImporterContribution;import io.onedev.server.web.page.project.issues.imports.IssueImporter;import io.onedev.server.web.page.project.issues.imports.IssueImporterContribution;@@ -21,22 +19,18 @@ public class YouTrackPluginModule extends AbstractPluginModule {protected void configure() {super.configure();- // put your guice bindings here- contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {-- @Override- public Collection<ProjectImporter<? extends Serializable, ? extends Serializable>> getImporters() {- return Lists.newArrayList(new YouTrackProjectImporter());- }-- });-+ // put your guice bindings herecontribute(IssueImporterContribution.class, new IssueImporterContribution() {@Overridepublic Collection<IssueImporter<? extends Serializable, ? extends Serializable>> getImporters() {return Lists.newArrayList(new YouTrackIssueImporter());}++ @Override+ public int getOrder() {+ return 300;+ }});}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportOption.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportOption.javaindex 14e4397376..8ab178712b 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportOption.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportOption.java@@ -37,7 +37,7 @@ public class YouTrackProjectImportOption extends YouTrackIssueImportOption implefor (int i=0; i<projecMappings.size(); i++) {if (projectManager.find(projecMappings.get(i).getOneDevProject()) != null) {context.buildConstraintViolationWithTemplate("Project name already used")- .addPropertyNode("imports")+ .addPropertyNode("projectMappings").addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT).inIterable().atIndex(i).addConstraintViolation();isValid = false;@@ -45,7 +45,7 @@ public class YouTrackProjectImportOption extends YouTrackIssueImportOption implefor (int j=0; j<projecMappings.size(); j++) {if (j != i && projecMappings.get(j).getOneDevProject().equals(projecMappings.get(i).getOneDevProject())) {context.buildConstraintViolationWithTemplate("Duplicate project name")- .addPropertyNode("imports")+ .addPropertyNode("projectMappings").addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT).inIterable().atIndex(i).addConstraintViolation();isValid = false;diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportSource.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportSource.javaindex 8722357070..5968f61249 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportSource.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportSource.java@@ -39,8 +39,6 @@ public class YouTrackProjectImportSource implements Serializable, Validatable {private String password;- private boolean prepopulateImportOptions = true;-@Editable(order=10, name="YouTrack API URL", description="Specify url of YouTrack API. For instance <tt>http://localhost:8080/api</tt>")@NotEmptypublic String getApiUrl() {@@ -77,16 +75,6 @@ public class YouTrackProjectImportSource implements Serializable, Validatable {this.password = password;}- @Editable(order=1000, description="If checked, import options will be pre-populated based on all accessible "- + "projects and its settings")- public boolean isPrepopulateImportOptions() {- return prepopulateImportOptions;- }-- public void setPrepopulateImportOptions(boolean prepopulateImportOptions) {- this.prepopulateImportOptions = prepopulateImportOptions;- }-static String getApiEndpoint(String apiUrl, String apiPath) {return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.javadeleted file mode 100644index 963c0f5e14..0000000000--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java+++ /dev/null@@ -1,116 +0,0 @@-package io.onedev.server.plugin.imports.youtrack;--import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.NAME;-import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.buildImportOption;-import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.importIssues;-import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.list;-import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.newClient;--import java.util.ArrayList;-import java.util.Collection;-import java.util.HashMap;-import java.util.Map;--import javax.ws.rs.client.Client;--import com.fasterxml.jackson.databind.JsonNode;--import io.onedev.commons.utils.ExceptionUtils;-import io.onedev.commons.utils.ExplicitException;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.ProjectManager;-import io.onedev.server.model.Project;-import io.onedev.server.storage.StorageManager;-import io.onedev.server.util.SimpleLogger;-import io.onedev.server.web.page.project.imports.ProjectImporter;--public class YouTrackProjectImporter extends ProjectImporter<YouTrackProjectImportSource, YouTrackProjectImportOption> {-- private static final long serialVersionUID = 1L;-- @Override- public String getName() {- return NAME;- }-- @Override- public YouTrackProjectImportOption getImportOption(YouTrackProjectImportSource importSource, SimpleLogger logger) {- YouTrackProjectImportOption importOption = new YouTrackProjectImportOption();- if (importSource.isPrepopulateImportOptions()) {- Client client = newClient(importSource);- try {- String apiEndpoint = importSource.getApiEndpoint("/admin/projects?fields=name");- for (JsonNode projectNode: list(client, apiEndpoint, logger)) {- ProjectMapping projectMapping = new ProjectMapping();- projectMapping.setYouTrackProject(projectNode.get("name").asText());- projectMapping.setOneDevProject(projectMapping.getYouTrackProject().replace(' ', '-'));- importOption.getProjectMappings().add(projectMapping);- }- YouTrackIssueImportOption issueImportOption = buildImportOption(importSource, null, logger);- importOption.setIssueFieldMappings(issueImportOption.getIssueFieldMappings());- importOption.setIssueStateMappings(issueImportOption.getIssueStateMappings());- importOption.setIssueTagMappings(issueImportOption.getIssueTagMappings());- } finally {- client.close();- }- }- return importOption;- }-- @Override- public String doImport(YouTrackProjectImportSource importSource, YouTrackProjectImportOption importOption,- boolean dryRun, SimpleLogger logger) {-- Map<String, String> youTrackProjectIds = new HashMap<>();- Map<String, String> youTrackProjectDescriptions = new HashMap<>();- Collection<Long> projectIds = new ArrayList<>();-- Client client = newClient(importSource);- try {- String apiEndpoint = importSource.getApiEndpoint("/admin/projects?fields=id,name,description");- for (JsonNode projectNode: list(client, apiEndpoint, logger)) {- String projectName = projectNode.get("name").asText();- youTrackProjectIds.put(projectName, projectNode.get("id").asText());- youTrackProjectDescriptions.put(projectName, projectNode.get("description").asText(null));- }-- YouTrackImportResult result = new YouTrackImportResult();-- for (ProjectMapping projectMapping: importOption.getProjectMappings()) {- logger.log("Importing project '" + projectMapping.getYouTrackProject() + "'...");-- String youTrackProjectId = youTrackProjectIds.get(projectMapping.getYouTrackProject());- if (youTrackProjectId == null)- throw new ExplicitException("Unable to find YouTrack project: " + projectMapping.getYouTrackProject());-- Project project = new Project();- project.setName(projectMapping.getOneDevProject());- project.setDescription(youTrackProjectDescriptions.get(projectMapping.getYouTrackProject()));- project.setIssueManagementEnabled(true);-- if (!dryRun) {- OneDev.getInstance(ProjectManager.class).create(project);- projectIds.add(project.getId());- }-- YouTrackImportResult currentResult = importIssues(importSource, youTrackProjectId, project,- true, importOption, dryRun, logger);- result.mismatchedIssueFields.putAll(currentResult.mismatchedIssueFields);- result.nonExistentLogins.addAll(currentResult.nonExistentLogins);- result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);- result.unmappedIssueFields.addAll(currentResult.unmappedIssueFields);- result.unmappedIssueStates.addAll(currentResult.unmappedIssueStates);- result.unmappedIssueTags.addAll(currentResult.unmappedIssueTags);- }-- return result.toHtml("Projects imported successfully");- } catch (Exception e) {- for (Long projectId: projectIds)- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);- throw ExceptionUtils.unchecked(e);- } finally {- client.close();- }- }--}\ No newline at end of filediff --git a/server-product/pom.xml b/server-product/pom.xmlindex 8e747c2926..6c808bf700 100644--- a/server-product/pom.xml+++ b/server-product/pom.xml@@ -75,6 +75,11 @@<artifactId>server-plugin-import-github</artifactId><version>${project.version}</version></dependency>+ <dependency>+ <groupId>io.onedev</groupId>+ <artifactId>server-plugin-import-gitlab</artifactId>+ <version>${project.version}</version>+ </dependency><dependency><groupId>io.onedev</groupId><artifactId>server-plugin-import-youtrack</artifactId>diff --git a/server-product/src/test/java/io/onedev/server/product/Test.java b/server-product/src/test/java/io/onedev/server/product/Test.javaindex c8a93e0c98..e061411771 100644--- a/server-product/src/test/java/io/onedev/server/product/Test.java+++ b/server-product/src/test/java/io/onedev/server/product/Test.java@@ -1,11 +1,25 @@package io.onedev.server.product;import java.util.concurrent.ExecutionException;+import java.util.regex.Matcher;+import java.util.regex.Pattern;public class Test {@org.junit.Testpublic void test() throws InterruptedException, ExecutionException {+ Pattern PATTERN_ATTACHMENT = Pattern.compile("\\[(.+?)\\]\\s*\\((/uploads/.+?)\\)");++ StringBuffer buffer = new StringBuffer();+ Matcher matcher = PATTERN_ATTACHMENT.matcher("hello [kb.txt](/uploads/kb.txt)");+ while (matcher.find()) {+ System.out.println(matcher.group(1));+ System.out.println(matcher.group(2));+ matcher.appendReplacement(buffer, "[kb.txt](hello.txt)");+ }+ matcher.appendTail(buffer);++ System.out.println(buffer);}}\ No newline at end of file