Skip to content

The Unified and Autonomous Development Platform

Fix issue #342 - Able to import YouTrack/GitHub issues into existing project

Robin Shen469dee7765a3
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/IssueManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/IssueManager.java
index 4a051ca331..51bcb34809 100644
--- a/server-core/src/main/java/io/onedev/server/entitymanager/IssueManager.java
+++ b/server-core/src/main/java/io/onedev/server/entitymanager/IssueManager.java
@@ -32,6 +32,8 @@ public interface IssueManager extends EntityManager<Issue> {
void open(Issue issue);
+ Long getNextNumber(Project numberScope);
+
List<Issue> query(@Nullable Project project, EntityQuery<Issue> issueQuery,
int firstResult, int maxResults, boolean loadFields);
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultIssueManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultIssueManager.java
index 5237f0692f..2695592784 100644
--- a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultIssueManager.java
+++ b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultIssueManager.java
@@ -164,10 +164,7 @@ public class DefaultIssueManager extends BaseEntityManager<Issue> implements Iss
public void open(Issue issue) {
Preconditions.checkArgument(issue.isNew());
issue.setNumberScope(issue.getProject().getForkRoot());
- Query<?> query = getSession().createQuery(String.format("select max(%s) from Issue where %s=:numberScope",
- Issue.PROP_NUMBER, Issue.PROP_NUMBER_SCOPE));
- query.setParameter("numberScope", issue.getNumberScope());
- issue.setNumber(getNextNumber(issue.getNumberScope(), query));
+ issue.setNumber(getNextNumber(issue.getNumberScope()));
IssueOpened event = new IssueOpened(issue);
issue.setLastUpdate(event.getLastUpdate());
@@ -178,6 +175,14 @@ public class DefaultIssueManager extends BaseEntityManager<Issue> implements Iss
listenerRegistry.post(event);
}
+
+ @Sessional
+ public Long getNextNumber(Project numberScope) {
+ Query<?> query = getSession().createQuery(String.format("select max(%s) from Issue where %s=:numberScope",
+ Issue.PROP_NUMBER, Issue.PROP_NUMBER_SCOPE));
+ query.setParameter(Issue.PROP_NUMBER_SCOPE, numberScope);
+ return getNextNumber(numberScope, query);
+ }
@Transactional
@Override
diff --git a/server-core/src/main/java/io/onedev/server/model/Project.java b/server-core/src/main/java/io/onedev/server/model/Project.java
index 5e792d5f11..ee78dea58a 100644
--- a/server-core/src/main/java/io/onedev/server/model/Project.java
+++ b/server-core/src/main/java/io/onedev/server/model/Project.java
@@ -1040,7 +1040,7 @@ public class Project extends AbstractEntity implements NameAware {
this.codeComments = codeComments;
}
- @Editable(order=300, name="Issue management", description="Whether or not to provide issue management for the project")
+ @Editable(order=300, name="Issue management", description="Whether or not to enable issue management for the project")
public boolean isIssueManagementEnabled() {
return issueManagementEnabled;
}
diff --git a/server-core/src/main/java/io/onedev/server/util/JerseyUtils.java b/server-core/src/main/java/io/onedev/server/util/JerseyUtils.java
index 2fdcd7d008..db017c9084 100644
--- a/server-core/src/main/java/io/onedev/server/util/JerseyUtils.java
+++ b/server-core/src/main/java/io/onedev/server/util/JerseyUtils.java
@@ -8,15 +8,15 @@ import io.onedev.commons.utils.StringUtils;
public class JerseyUtils {
@Nullable
- public static String checkStatus(Response response) {
+ public static String checkStatus(String url, Response response) {
int status = response.getStatus();
if (status != 200) {
String errorMessage = response.readEntity(String.class);
if (StringUtils.isNotBlank(errorMessage)) {
- return String.format("Http request failed (status code: %d, error message: %s)",
- status, errorMessage);
+ return String.format("Http request failed (url: %s, status code: %d, error message: %s)",
+ url, status, errorMessage);
} else {
- return String.format("Http request failed (status code: %d)", status);
+ return String.format("Http request failed (url: %s, status code: %d)", url, status);
}
} else {
return null;
diff --git a/server-core/src/main/java/io/onedev/server/util/ReferenceMigrator.java b/server-core/src/main/java/io/onedev/server/util/ReferenceMigrator.java
new file mode 100644
index 0000000000..92609c67be
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/util/ReferenceMigrator.java
@@ -0,0 +1,59 @@
+package io.onedev.server.util;
+
+import java.util.Map;
+
+import io.onedev.commons.utils.WordUtils;
+
+public class ReferenceMigrator {
+
+ private final String referenceType;
+
+ private final Map<Long, Long> numberMappings;
+
+ public ReferenceMigrator(Class<?> referenceClass, Map<Long, Long> numberMappings) {
+ referenceType = WordUtils.uncamel(referenceClass.getSimpleName()).toLowerCase();
+ this.numberMappings = numberMappings;
+ }
+
+ public String migratePrefixed(String content, String prefix) {
+ StringBuilder builder = new StringBuilder();
+ int start = 0;
+ while (true) {
+ int index = content.indexOf(prefix, start);
+ if (index == -1) {
+ builder.append(content.substring(start));
+ break;
+ } else {
+ builder.append(content.substring(start, index));
+ start = index + prefix.length();
+ if (index > 0 && Character.isLetterOrDigit(content.charAt(index-1))) {
+ builder.append(prefix);
+ } else {
+ StringBuilder numberBuilder = new StringBuilder();
+ while (start < content.length()) {
+ char ch = content.charAt(start);
+ if (Character.isDigit(ch)) {
+ numberBuilder.append(ch);
+ start++;
+ } else {
+ if (Character.isAlphabetic(ch))
+ numberBuilder.setLength(0);
+ break;
+ }
+ }
+ if (numberBuilder.length() != 0) {
+ Long newNumber = numberMappings.get(Long.valueOf(numberBuilder.toString()));
+ if (newNumber != null)
+ builder.append(referenceType).append(" #").append(newNumber);
+ else
+ builder.append(content.substring(index, start));
+ } else {
+ builder.append(content.substring(index, start));
+ }
+ }
+ }
+ }
+ return builder.toString();
+ }
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/web/OneUrlMapper.java b/server-core/src/main/java/io/onedev/server/web/OneUrlMapper.java
index c8253172a0..071d30c9d9 100644
--- a/server-core/src/main/java/io/onedev/server/web/OneUrlMapper.java
+++ b/server-core/src/main/java/io/onedev/server/web/OneUrlMapper.java
@@ -80,6 +80,7 @@ import io.onedev.server.web.page.project.issues.detail.IssueActivitiesPage;
import io.onedev.server.web.page.project.issues.detail.IssueBuildsPage;
import io.onedev.server.web.page.project.issues.detail.IssueCommitsPage;
import io.onedev.server.web.page.project.issues.detail.IssuePullRequestsPage;
+import io.onedev.server.web.page.project.issues.imports.IssueImportPage;
import io.onedev.server.web.page.project.issues.list.ProjectIssueListPage;
import io.onedev.server.web.page.project.issues.milestones.MilestoneDetailPage;
import io.onedev.server.web.page.project.issues.milestones.MilestoneEditPage;
@@ -258,6 +259,7 @@ public class OneUrlMapper extends CompoundRequestMapper {
add(new DynamicPathPageMapper("projects/${project}/issues/${issue}/pull-requests", IssuePullRequestsPage.class));
add(new DynamicPathPageMapper("projects/${project}/issues/${issue}/builds", IssueBuildsPage.class));
add(new DynamicPathPageMapper("projects/${project}/issues/new", NewIssuePage.class));
+ add(new DynamicPathPageMapper("projects/${project}/issues/import/${importer}", IssueImportPage.class));
add(new DynamicPathPageMapper("projects/${project}/milestones", MilestoneListPage.class));
add(new DynamicPathPageMapper("projects/${project}/milestones/${milestone}", MilestoneDetailPage.class));
add(new DynamicPathPageMapper("projects/${project}/milestones/${milestone}/edit", MilestoneEditPage.class));
diff --git a/server-core/src/main/java/io/onedev/server/web/asset/icon/import.svg b/server-core/src/main/java/io/onedev/server/web/asset/icon/import.svg
new file mode 100644
index 0000000000..a7ea11fd46
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/web/asset/icon/import.svg
@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1624065252808" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="1212" xmlns:xlink="http://www.w3.org/1999/xlink" width="200" height="200"><defs><style type="text/css"></style></defs><path d="M481.28 715.946667a32 32 0 0 0 22.613333 9.386666h16.213334a32.853333 32.853333 0 0 0 22.613333-9.386666l218.88-219.306667a21.333333 21.333333 0 0 0 0-30.293333l-30.293333-30.293334a20.906667 20.906667 0 0 0-29.866667 0L554.666667 583.253333V149.333333a21.333333 21.333333 0 0 0-21.333334-21.333333h-42.666666a21.333333 21.333333 0 0 0-21.333334 21.333333v433.92l-146.773333-147.2a21.333333 21.333333 0 0 0-30.293333 0l-29.866667 30.293334a21.333333 21.333333 0 0 0 0 30.293333z m30.72-90.026667zM896 810.666667v-149.333334a21.333333 21.333333 0 0 0-21.333333-21.333333h-42.666667a21.333333 21.333333 0 0 0-21.333333 21.333333V810.666667H213.333333v-149.333334a21.333333 21.333333 0 0 0-21.333333-21.333333h-42.666667a21.333333 21.333333 0 0 0-21.333333 21.333333V810.666667a85.333333 85.333333 0 0 0 85.333333 85.333333h597.333334a85.333333 85.333333 0 0 0 85.333333-85.333333z" p-id="1213"></path></svg>
\ No newline at end of file
diff --git a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.html b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.html
index 055d0556b0..39da962fc4 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.html
+++ b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.html
@@ -28,9 +28,12 @@
<a wicket:id="listFields" class="list-fields text-gray mr-4">
<wicket:svg href="select" class="icon"></wicket:svg> Display Fields
</a>
- <a wicket:id="orderBy" class="order-by text-gray">
+ <a wicket:id="orderBy" class="order-by text-gray mr-4">
<wicket:svg href="sort" class="icon"></wicket:svg> Order By
</a>
+ <a wicket:id="importIssues" class="import-issues text-gray">
+ <wicket:svg href="import" class="icon"></wicket:svg> Import Issues
+ </a>
</div>
<div wicket:id="body" class="body">
<div wicket:id="feedback"></div>
diff --git a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java
index 3014d6e769..1cb156520e 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java
+++ b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java
@@ -1,5 +1,6 @@
package io.onedev.server.web.component.issue.list;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
@@ -85,6 +86,8 @@ import io.onedev.server.web.component.issue.fieldvalues.FieldValuesPanel;
import io.onedev.server.web.component.link.ActionablePageLink;
import io.onedev.server.web.component.link.DropdownLink;
import io.onedev.server.web.component.link.copytoclipboard.CopyToClipboardLink;
+import io.onedev.server.web.component.menu.MenuItem;
+import io.onedev.server.web.component.menu.MenuLink;
import io.onedev.server.web.component.modal.ModalLink;
import io.onedev.server.web.component.modal.ModalPanel;
import io.onedev.server.web.component.orderedit.OrderEditPanel;
@@ -97,6 +100,10 @@ import io.onedev.server.web.component.user.ident.Mode;
import io.onedev.server.web.component.user.ident.UserIdentPanel;
import io.onedev.server.web.page.project.issues.create.NewIssuePage;
import io.onedev.server.web.page.project.issues.detail.IssueActivitiesPage;
+import io.onedev.server.web.page.project.issues.imports.IssueImportPage;
+import io.onedev.server.web.page.project.issues.imports.IssueImporter;
+import io.onedev.server.web.page.project.issues.imports.IssueImporterContribution;
+import io.onedev.server.web.page.project.issues.list.ProjectIssueListPage;
import io.onedev.server.web.util.Cursor;
import io.onedev.server.web.util.LoadableDetachableDataProvider;
import io.onedev.server.web.util.PagingHistorySupport;
@@ -298,6 +305,43 @@ public abstract class IssueListPanel extends Panel {
});
+ add(new MenuLink("importIssues") {
+
+ @Override
+ protected List<MenuItem> getMenuItems(FloatingPanel dropdown) {
+ Collection<IssueImporter<? extends Serializable, ? extends Serializable>> importers = new ArrayList<>();
+ for (IssueImporterContribution contribution: OneDev.getExtensions(IssueImporterContribution.class))
+ importers.addAll(contribution.getImporters());
+
+ List<MenuItem> menuItems = new ArrayList<>();
+ for (IssueImporter<? extends Serializable, ? extends Serializable> importer: importers) {
+ menuItems.add(new MenuItem() {
+
+ @Override
+ public String getLabel() {
+ return "From " + importer.getName();
+ }
+
+ @Override
+ public WebMarkupContainer newLink(String id) {
+ String url = urlFor(getPage().getClass(), getPage().getPageParameters()).toString();
+ return new BookmarkablePageLink<Void>(id, IssueImportPage.class,
+ IssueImportPage.paramsOf(getProject(), importer.getName(), url));
+ }
+
+ });
+ }
+ return menuItems;
+ }
+
+ @Override
+ protected void onConfigure() {
+ super.onConfigure();
+ setVisible(getPage() instanceof ProjectIssueListPage);
+ }
+
+ });
+
queryInput = new TextField<String>("input", queryStringModel);
queryInput.add(new IssueQueryBehavior(new AbstractReadOnlyModel<Project>() {
diff --git a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.html b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.html
index 22798e2398..490cc3c459 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.html
+++ b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.html
@@ -19,9 +19,12 @@
<a wicket:id="saveQuery" class="save-query text-gray mr-4">
<wicket:svg href="save" class="icon"></wicket:svg> Save Query
</a>
- <a wicket:id="orderBy" class="order-by text-gray">
+ <a wicket:id="orderBy" class="order-by text-gray mr-4">
<wicket:svg href="sort" class="icon"></wicket:svg> Order By
</a>
+ <a wicket:id="importProjects" class="import-projects text-gray">
+ <wicket:svg href="import" class="icon"></wicket:svg> Import Projects
+ </a>
</div>
<div wicket:id="body">
<div wicket:id="feedback"></div>
diff --git a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java
index d7e323b920..af8f223050 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java
+++ b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java
@@ -240,6 +240,36 @@ public class ProjectListPanel extends Panel {
});
+ add(new MenuLink("importProjects") {
+
+ @Override
+ protected List<MenuItem> getMenuItems(FloatingPanel dropdown) {
+ Collection<ProjectImporter<? extends Serializable, ? extends Serializable>> importers = new ArrayList<>();
+ for (ProjectImporterContribution contribution: OneDev.getExtensions(ProjectImporterContribution.class))
+ importers.addAll(contribution.getImporters());
+
+ List<MenuItem> menuItems = new ArrayList<>();
+ for (ProjectImporter<? extends Serializable, ? extends Serializable> importer: importers) {
+ menuItems.add(new MenuItem() {
+
+ @Override
+ public String getLabel() {
+ return "From " + importer.getName();
+ }
+
+ @Override
+ public WebMarkupContainer newLink(String id) {
+ return new BookmarkablePageLink<Void>(id, ProjectImportPage.class,
+ ProjectImportPage.paramsOf(importer.getName()));
+ }
+
+ });
+ }
+ return menuItems;
+ }
+
+ });
+
queryInput = new TextField<String>("input", queryStringModel);
queryInput.setOutputMarkupId(true);
queryInput.add(new ProjectQueryBehavior() {
@@ -276,52 +306,7 @@ public class ProjectListPanel extends Panel {
});
add(queryForm);
- Collection<ProjectImporter<? extends Serializable, ? extends Serializable>> importers = new ArrayList<>();
- for (ProjectImporterContribution contribution: OneDev.getExtensions(ProjectImporterContribution.class))
- importers.addAll(contribution.getImporters());
-
- if (importers.isEmpty()) {
- add(new BookmarkablePageLink<Void>("addProject", NewProjectPage.class));
- } else {
- add(new MenuLink("addProject") {
-
- @Override
- protected List<MenuItem> getMenuItems(FloatingPanel dropdown) {
- List<MenuItem> menuItems = new ArrayList<>();
- menuItems.add(new MenuItem() {
-
- @Override
- public String getLabel() {
- return "Create New";
- }
-
- @Override
- public WebMarkupContainer newLink(String id) {
- return new BookmarkablePageLink<Void>(id, NewProjectPage.class);
- }
-
- });
- for (ProjectImporter<? extends Serializable, ? extends Serializable> importer: importers) {
- menuItems.add(new MenuItem() {
-
- @Override
- public String getLabel() {
- return "Import from " + importer.getName();
- }
-
- @Override
- public WebMarkupContainer newLink(String id) {
- return new BookmarkablePageLink<Void>(id, ProjectImportPage.class,
- ProjectImportPage.paramsOf(importer.getName()));
- }
-
- });
- }
- return menuItems;
- }
-
- });
- }
+ add(new BookmarkablePageLink<Void>("addProject", NewProjectPage.class));
SortableDataProvider<Project, Void> dataProvider = new LoadableDetachableDataProvider<Project, Void>() {
diff --git a/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskButton.java b/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskButton.java
index c09f791978..d1c8c779af 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskButton.java
+++ b/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskButton.java
@@ -21,6 +21,7 @@ import org.apache.wicket.Component;
import org.apache.wicket.ajax.AjaxRequestTarget;
import org.apache.wicket.ajax.markup.html.form.AjaxButton;
import org.apache.wicket.markup.html.form.Form;
+import org.apache.wicket.request.cycle.RequestCycle;
import org.joda.time.DateTime;
import org.quartz.ScheduleBuilder;
import org.quartz.SimpleScheduleBuilder;
@@ -67,6 +68,12 @@ public abstract class TaskButton extends AjaxButton {
return WordUtils.uncamel(getId());
}
+ protected void onCompleted(AjaxRequestTarget target) {
+ }
+
+ protected void onCancelled(AjaxRequestTarget target) {
+ }
+
protected void submitTask(AjaxRequestTarget target) {
String path = getPath();
String title = getTitle().toLowerCase();
@@ -120,8 +127,14 @@ public abstract class TaskButton extends AjaxButton {
protected void onClosed() {
super.onClosed();
Future<?> future = getTaskFutures().remove(path);
- if (future != null && !future.isDone())
+
+ AjaxRequestTarget target = RequestCycle.get().find(AjaxRequestTarget.class);
+ if (future != null && !future.isDone()) {
future.cancel(true);
+ onCancelled(target);
+ } else {
+ onCompleted(target);
+ }
}
@Override
diff --git a/server-core/src/main/java/io/onedev/server/web/editable/choice/MultiChoiceEditor.java b/server-core/src/main/java/io/onedev/server/web/editable/choice/MultiChoiceEditor.java
index 434a18e912..9289455a12 100644
--- a/server-core/src/main/java/io/onedev/server/web/editable/choice/MultiChoiceEditor.java
+++ b/server-core/src/main/java/io/onedev/server/web/editable/choice/MultiChoiceEditor.java
@@ -68,10 +68,20 @@ public class MultiChoiceEditor extends PropertyEditor<List<String>> {
};
List<String> selections = getModelObject();
+
+ /*
+ * Avoid loading choicesModel which might be time-consuming. Otherwise, when this component
+ * is re-created due to a dependency property updating, there is a high chance we are
+ * operating on the old select2 component and may cause UI clutter
+ */
+ /*
if (selections != null)
selections.retainAll(choicesModel.getObject().keySet());
else
selections = new ArrayList<>();
+ */
+ if (selections == null)
+ selections = new ArrayList<>();
input = new StringMultiChoice("input", Model.of(selections), choicesModel) {
@Override
diff --git a/server-core/src/main/java/io/onedev/server/web/editable/choice/SingleChoiceEditor.java b/server-core/src/main/java/io/onedev/server/web/editable/choice/SingleChoiceEditor.java
index be03b8b3d4..0942f3f83f 100644
--- a/server-core/src/main/java/io/onedev/server/web/editable/choice/SingleChoiceEditor.java
+++ b/server-core/src/main/java/io/onedev/server/web/editable/choice/SingleChoiceEditor.java
@@ -65,8 +65,16 @@ public class SingleChoiceEditor extends PropertyEditor<String> {
};
String selection = getModelObject();
+
+ /*
+ * Avoid loading choicesModel which might be time-consuming. Otherwise, when this component
+ * is re-created due to a dependency property updating, there is a high chance we are
+ * operating on the old select2 component and may cause UI clutter
+ */
+ /*
if (!choicesModel.getObject().containsKey(selection))
selection = null;
+ */
input = new StringSingleChoice("input", Model.of(selection), choicesModel) {
@Override
diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java
index 6b355a457c..6e1a6692a5 100644
--- a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java
+++ b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java
@@ -4,6 +4,7 @@ import java.io.Serializable;
import java.util.concurrent.Callable;
import org.apache.wicket.Component;
+import org.apache.wicket.RestartResponseException;
import org.apache.wicket.ajax.AjaxRequestTarget;
import org.apache.wicket.ajax.attributes.AjaxRequestAttributes;
import org.apache.wicket.ajax.markup.html.form.AjaxButton;
@@ -14,13 +15,20 @@ import org.apache.wicket.request.mapper.parameter.PageParameters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
import io.onedev.server.OneDev;
+import io.onedev.server.model.Project;
import io.onedev.server.persistence.TransactionManager;
+import io.onedev.server.search.entity.EntitySort;
+import io.onedev.server.search.entity.EntitySort.Direction;
+import io.onedev.server.search.entity.project.ProjectQuery;
import io.onedev.server.util.SimpleLogger;
import io.onedev.server.web.ajaxlistener.ShowGlobalAjaxIndicatorListener;
import io.onedev.server.web.component.taskbutton.TaskButton;
import io.onedev.server.web.editable.BeanContext;
import io.onedev.server.web.page.layout.LayoutPage;
+import io.onedev.server.web.page.project.ProjectListPage;
@SuppressWarnings("serial")
public class ProjectImportPage<T extends Serializable, S extends Serializable> extends LayoutPage {
@@ -107,6 +115,18 @@ public class ProjectImportPage<T extends Serializable, S extends Serializable> e
form.add(new TaskButton("import") {
+ @Override
+ protected void onCompleted(AjaxRequestTarget target) {
+ super.onCompleted(target);
+
+ EntitySort sort = new EntitySort();
+ sort.setField(Project.NAME_UPDATE_DATE);
+ sort.setDirection(Direction.DESCENDING);
+ ProjectQuery query = new ProjectQuery(null, Lists.newArrayList(sort));
+ PageParameters params = ProjectListPage.paramsOf(query.toString(), 0, 0);
+ throw new RestartResponseException(ProjectListPage.class, params);
+ }
+
@Override
protected String runTask(SimpleLogger logger) {
return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {
diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.html b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.html
new file mode 100644
index 0000000000..4121f0f061
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.html
@@ -0,0 +1,13 @@
+<wicket:extend>
+ <div class="issue-import card m-2 m-sm-5">
+ <div class="card-body">
+ <form wicket:id="form">
+ <div wicket:id="feedback"></div>
+ <div wicket:id="editor"></div>
+ <input wicket:id="next" type="submit" class="btn btn-primary" value="Next">
+ <input wicket:id="import" type="submit" class="btn btn-primary mr-1" value="Import">
+ <input wicket:id="dryRun" type="submit" class="btn btn-light-primary" value="Dry Run">
+ </form>
+ </div>
+ </div>
+</wicket:extend>
diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java
new file mode 100644
index 0000000000..013a60f769
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java
@@ -0,0 +1,212 @@
+package io.onedev.server.web.page.project.issues.imports;
+
+import java.io.Serializable;
+import java.util.concurrent.Callable;
+
+import org.apache.wicket.Component;
+import org.apache.wicket.RestartResponseException;
+import org.apache.wicket.ajax.AjaxRequestTarget;
+import org.apache.wicket.ajax.attributes.AjaxRequestAttributes;
+import org.apache.wicket.ajax.markup.html.form.AjaxButton;
+import org.apache.wicket.feedback.FencedFeedbackPanel;
+import org.apache.wicket.markup.html.basic.Label;
+import org.apache.wicket.markup.html.form.Form;
+import org.apache.wicket.request.mapper.parameter.PageParameters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.model.Issue;
+import io.onedev.server.model.Project;
+import io.onedev.server.persistence.TransactionManager;
+import io.onedev.server.search.entity.EntitySort;
+import io.onedev.server.search.entity.EntitySort.Direction;
+import io.onedev.server.search.entity.issue.IssueQuery;
+import io.onedev.server.util.SimpleLogger;
+import io.onedev.server.web.ajaxlistener.ShowGlobalAjaxIndicatorListener;
+import io.onedev.server.web.component.taskbutton.TaskButton;
+import io.onedev.server.web.editable.BeanContext;
+import io.onedev.server.web.page.project.ProjectPage;
+import io.onedev.server.web.page.project.issues.list.ProjectIssueListPage;
+
+@SuppressWarnings("serial")
+public class IssueImportPage<T extends Serializable, S extends Serializable> extends ProjectPage {
+
+ private static final String PARAM_IMPORTER = "importer";
+
+ private static final Logger logger = LoggerFactory.getLogger(IssueImportPage.class);
+
+ private IssueImporter<T, S> importer;
+
+ private T importSource;
+
+ private S importOption;
+
+ @SuppressWarnings("unchecked")
+ public IssueImportPage(PageParameters params) {
+ super(params);
+
+ String importerName = params.get(PARAM_IMPORTER).toString();
+ for (IssueImporterContribution contribution: OneDev.getExtensions(IssueImporterContribution.class)) {
+ for (IssueImporter<? extends Serializable, ? extends Serializable> importer: contribution.getImporters()) {
+ if (importer.getName().equals(importerName)) {
+ this.importer = (IssueImporter<T, S>) importer;
+ break;
+ }
+ }
+ }
+
+ if (importer == null)
+ throw new RuntimeException("Undefined importer: " + importerName);
+ }
+
+ @Override
+ protected void onInitialize() {
+ super.onInitialize();
+
+ try {
+ importSource = importer.getSourceClass().newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+
+ Form<?> form = new Form<Void>("form");
+ form.add(new FencedFeedbackPanel("feedback", form));
+ form.add(BeanContext.edit("editor", importSource));
+
+ form.add(new AjaxButton("next") {
+
+ @Override
+ protected void updateAjaxAttributes(AjaxRequestAttributes attributes) {
+ super.updateAjaxAttributes(attributes);
+ attributes.getAjaxCallListeners().add(new ShowGlobalAjaxIndicatorListener());
+ }
+
+ @Override
+ protected void onSubmit(AjaxRequestTarget target, Form<?> form) {
+ super.onSubmit(target, form);
+
+ importOption = importer.getImportOption(importSource, new SimpleLogger() {
+
+ @Override
+ public void log(String message) {
+ logger.info(message);
+ }
+
+ });
+
+ form.replace(BeanContext.edit("editor", importOption));
+ setVisible(false);
+
+ form.get("import").setVisible(true);
+ form.get("dryRun").setVisible(true);
+
+ target.add(form);
+ }
+
+ @Override
+ protected void onError(AjaxRequestTarget target, Form<?> form) {
+ super.onError(target, form);
+ target.add(form);
+ }
+
+ });
+
+ Long projectId = getProject().getId();
+
+ form.add(new TaskButton("import") {
+
+ @Override
+ protected void onCompleted(AjaxRequestTarget target) {
+ super.onCompleted(target);
+
+ EntitySort sort = new EntitySort();
+ sort.setField(Issue.NAME_NUMBER);
+ sort.setDirection(Direction.DESCENDING);
+ IssueQuery query = new IssueQuery(null, Lists.newArrayList(sort));
+
+ PageParameters params = ProjectIssueListPage.paramsOf(getProject(), query.toString(), 0);
+ throw new RestartResponseException(ProjectIssueListPage.class, params);
+ }
+
+ @Override
+ protected String runTask(SimpleLogger logger) {
+ return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {
+
+ @Override
+ public String call() throws Exception {
+ Project project = OneDev.getInstance(ProjectManager.class).load(projectId);
+ return importer.doImport(project, importSource, importOption, false, logger);
+ }
+
+ });
+ }
+
+ @Override
+ protected String getTitle() {
+ return "Importing from " + importer.getName();
+ }
+
+ @Override
+ protected void onError(AjaxRequestTarget target, Form<?> form) {
+ super.onError(target, form);
+ target.add(form);
+ }
+
+ }.setVisible(false));
+
+ form.add(new TaskButton("dryRun") {
+
+ @Override
+ protected String runTask(SimpleLogger logger) {
+ return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {
+
+ @Override
+ public String call() throws Exception {
+ Project project = OneDev.getInstance(ProjectManager.class).load(projectId);
+ return importer.doImport(project, importSource, importOption, true, logger);
+ }
+
+ });
+ }
+
+ @Override
+ protected String getTitle() {
+ return "Test importing from " + importer.getName();
+ }
+
+ @Override
+ protected void onError(AjaxRequestTarget target, Form<?> form) {
+ super.onError(target, form);
+ target.add(form);
+ }
+
+ }.setVisible(false));
+
+ form.setOutputMarkupId(true);
+ add(form);
+ }
+
+ public static PageParameters paramsOf(Project project, String importer, String prevUrl) {
+ PageParameters params = ProjectPage.paramsOf(project);
+ params.add(PARAM_IMPORTER, importer);
+ return params;
+ }
+
+ public T getImportSource() {
+ return importSource;
+ }
+
+ public S getImportOption() {
+ return importOption;
+ }
+
+ @Override
+ protected Component newProjectTitle(String componentId) {
+ return new Label(componentId, "Import Issues frmo " + importer.getName());
+ }
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporter.java b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporter.java
new file mode 100644
index 0000000000..98de01607b
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporter.java
@@ -0,0 +1,42 @@
+package io.onedev.server.web.page.project.issues.imports;
+
+import java.io.Serializable;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+import io.onedev.server.model.Project;
+import io.onedev.server.util.ReflectionUtils;
+import io.onedev.server.util.SimpleLogger;
+
+public abstract class IssueImporter<T extends Serializable, S extends Serializable> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Class<T> sourceClass;
+
+ private final Class<S> optionClass;
+
+ @SuppressWarnings("unchecked")
+ public IssueImporter() {
+ List<Class<?>> typeArguments = ReflectionUtils.getTypeArguments(IssueImporter.class, getClass());
+ sourceClass = (Class<T>) typeArguments.get(0);
+ optionClass = (Class<S>) typeArguments.get(1);
+ }
+
+ public Class<T> getSourceClass() {
+ return sourceClass;
+ }
+
+ public Class<S> getOptionClass() {
+ return optionClass;
+ }
+
+ public abstract String getName();
+
+ public abstract S getImportOption(T importSource, SimpleLogger logger);
+
+ @Nullable
+ public abstract String doImport(Project project, T importSource, S importOption, boolean dryRun, SimpleLogger logger);
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporterContribution.java b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporterContribution.java
new file mode 100644
index 0000000000..45aa00bad3
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporterContribution.java
@@ -0,0 +1,13 @@
+package io.onedev.server.web.page.project.issues.imports;
+
+import java.io.Serializable;
+import java.util.Collection;
+
+import io.onedev.commons.launcher.loader.ExtensionPoint;
+
+@ExtensionPoint
+public interface IssueImporterContribution {
+
+ Collection<IssueImporter<? extends Serializable, ? extends Serializable>> getImporters();
+
+}
diff --git a/server-core/src/test/java/io/onedev/server/util/ReferenceMigratorTest.java b/server-core/src/test/java/io/onedev/server/util/ReferenceMigratorTest.java
new file mode 100644
index 0000000000..179199872b
--- /dev/null
+++ b/server-core/src/test/java/io/onedev/server/util/ReferenceMigratorTest.java
@@ -0,0 +1,26 @@
+package io.onedev.server.util;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+import io.onedev.server.model.Issue;
+
+public class ReferenceMigratorTest {
+
+ @Test
+ public void test() {
+ ReferenceMigrator migrator = new ReferenceMigrator(
+ Issue.class, CollectionUtils.newHashMap(1L, 11L, 22L, 2222L));
+ assertEquals("issue #11", migrator.migratePrefixed("QB-1", "QB-"));
+ assertEquals("issue #2222", migrator.migratePrefixed("QB-22", "QB-"));
+ assertEquals("QB-1QB-22", migrator.migratePrefixed("QB-1QB-22", "QB-"));
+ assertEquals("issue #11 issue #2222", migrator.migratePrefixed("QB-1 QB-22", "QB-"));
+ assertEquals("issue #11 issue #2222", migrator.migratePrefixed("QB-1 QB-22", "QB-"));
+ assertEquals("QB-2,issue #2222", migrator.migratePrefixed("QB-2,QB-22", "QB-"));
+ assertEquals("QB-1andQB-2", migrator.migratePrefixed("QB-1andQB-2", "QB-"));
+ assertEquals("QB-1以及QB-2", migrator.migratePrefixed("QB-1以及QB-2", "QB-"));
+ assertEquals("2QB-1,issue #2222", migrator.migratePrefixed("2QB-1,QB-22", "QB-"));
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportResult.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportResult.java
new file mode 100644
index 0000000000..bd5f191bfe
--- /dev/null
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportResult.java
@@ -0,0 +1,53 @@
+package io.onedev.server.plugin.imports.github;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.unbescape.html.HtmlEscape;
+
+public class GitHubImportResult {
+
+ Set<String> nonExistentLogins = new HashSet<>();
+
+ Set<String> unmappedIssueLabels = new HashSet<>();
+
+ Set<String> nonExistentMilestones = new HashSet<>();
+
+ boolean issuesImported = false;
+
+ public String toHtml(String leadingText) {
+ StringBuilder feedback = new StringBuilder(leadingText);
+
+ boolean hasNotice = false;
+
+ if (!nonExistentMilestones.isEmpty() || !unmappedIssueLabels.isEmpty()
+ || !nonExistentLogins.isEmpty() || issuesImported) {
+ hasNotice = true;
+ }
+
+ if (hasNotice)
+ feedback.append("<br><br><b>NOTE:</b><ul>");
+
+ if (!nonExistentMilestones.isEmpty()) {
+ feedback.append("<li> Non existent milestones: "
+ + HtmlEscape.escapeHtml5(nonExistentMilestones.toString()));
+ }
+ if (!unmappedIssueLabels.isEmpty()) {
+ feedback.append("<li> GitHub issue labels not mapped to OneDev custom field: "
+ + HtmlEscape.escapeHtml5(unmappedIssueLabels.toString()));
+ }
+ if (!nonExistentLogins.isEmpty()) {
+ feedback.append("<li> GitHub logins without public email or public email can not be mapped to OneDev account: "
+ + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));
+ }
+
+ if (issuesImported)
+ feedback.append("<li> Attachments in issue description and comments are not imported due to GitHub limitation");
+
+ if (hasNotice)
+ feedback.append("</ul>");
+
+ return feedback.toString();
+
+ }
+}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportUtils.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportUtils.java
new file mode 100644
index 0000000000..8d4aea7fa6
--- /dev/null
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportUtils.java
@@ -0,0 +1,416 @@
+package io.onedev.server.plugin.imports.github;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.annotation.Nullable;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Invocation;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Response;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.client.utils.URIBuilder;
+import org.glassfish.jersey.client.ClientProperties;
+import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
+import org.joda.time.format.ISODateTimeFormat;
+import org.unbescape.html.HtmlEscape;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import io.onedev.commons.utils.ExplicitException;
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.IssueManager;
+import io.onedev.server.entitymanager.SettingManager;
+import io.onedev.server.entitymanager.UserManager;
+import io.onedev.server.model.Issue;
+import io.onedev.server.model.IssueComment;
+import io.onedev.server.model.IssueField;
+import io.onedev.server.model.Milestone;
+import io.onedev.server.model.Project;
+import io.onedev.server.model.User;
+import io.onedev.server.model.support.LastUpdate;
+import io.onedev.server.model.support.administration.GlobalIssueSetting;
+import io.onedev.server.model.support.inputspec.InputSpec;
+import io.onedev.server.model.support.issue.field.spec.FieldSpec;
+import io.onedev.server.persistence.dao.Dao;
+import io.onedev.server.util.Pair;
+import io.onedev.server.util.ReferenceMigrator;
+import io.onedev.server.util.SimpleLogger;
+
+public class GitHubImportUtils {
+
+ static final String NAME = "GitHub";
+
+ static final int PER_PAGE = 100;
+
+ static GitHubIssueImportOption buildImportOption(GitHubProjectImportSource importSource,
+ @Nullable String gitHubRepo, SimpleLogger logger) {
+ GitHubIssueImportOption importOption = new GitHubIssueImportOption();
+ Client client = newClient(importSource);
+ try {
+ List<String> gitHubRepos = new ArrayList<>();
+ if (gitHubRepo == null) {
+ String apiEndpoint = importSource.getApiEndpoint("/user/repos");
+ for (JsonNode repoNode: list(client, apiEndpoint, logger)) {
+ String repoName = repoNode.get("name").asText();
+ String ownerName = repoNode.get("owner").get("login").asText();
+ gitHubRepos.add(ownerName + "/" + repoName);
+ }
+ } else {
+ gitHubRepos.add(gitHubRepo);
+ }
+
+ Set<String> labels = new LinkedHashSet<>();
+ for (String each: gitHubRepos) {
+ String apiEndpoint = importSource.getApiEndpoint("/repos/" + each + "/labels");
+ for (JsonNode labelNode: list(client, apiEndpoint, logger))
+ labels.add(labelNode.get("name").asText());
+ }
+
+ for (String label: labels) {
+ IssueLabelMapping mapping = new IssueLabelMapping();
+ mapping.setGitHubIssueLabel(label);
+ importOption.getIssueLabelMappings().add(mapping);
+ }
+ } finally {
+ client.close();
+ }
+ return importOption;
+ }
+
+ @Nullable
+ static User getUser(Client client, GitHubProjectImportSource importSource,
+ Map<String, Optional<User>> users, String login, SimpleLogger logger) {
+ Optional<User> userOpt = users.get(login);
+ if (userOpt == null) {
+ String apiEndpoint = importSource.getApiEndpoint("/users/" + login);
+ String email = get(client, apiEndpoint, logger).get("email").asText(null);
+ if (email != null)
+ userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByEmail(email));
+ else
+ userOpt = Optional.empty();
+ users.put(login, userOpt);
+ }
+ return userOpt.orElse(null);
+ }
+
+ static GitHubImportResult importIssues(GitHubProjectImportSource importSource, String gitHubRepo, Project oneDevProject,
+ boolean useExistingIssueNumbers, GitHubIssueImportOption importOption, Map<String, Optional<User>> users,
+ boolean dryRun, SimpleLogger logger) {
+ Client client = newClient(importSource);
+ try {
+ Set<String> nonExistentMilestones = new HashSet<>();
+ Set<String> nonExistentLogins = new HashSet<>();
+ Set<String> unmappedIssueLabels = new HashSet<>();
+
+ Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();
+ Map<String, Milestone> milestoneMappings = new HashMap<>();
+
+ for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {
+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
+ if (fieldSpec == null)
+ throw new ExplicitException("No field spec found: " + oneDevFieldName);
+ labelMappings.put(mapping.getGitHubIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));
+ }
+
+ for (Milestone milestone: oneDevProject.getMilestones())
+ milestoneMappings.put(milestone.getName(), milestone);
+
+ String initialIssueState = getIssueSetting().getInitialStateSpec().getName();
+
+ List<Issue> issues = new ArrayList<>();
+
+ Map<Long, Long> issueNumberMappings = new HashMap<>();
+
+ AtomicInteger numOfImportedIssues = new AtomicInteger(0);
+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {
+
+ private String joinAsMultilineHtml(List<String> values) {
+ List<String> escapedValues = new ArrayList<>();
+ for (String value: values)
+ escapedValues.add(HtmlEscape.escapeHtml5(value));
+ return StringUtils.join(escapedValues, "<br>");
+ }
+
+ @Override
+ public void consume(List<JsonNode> pageData) throws InterruptedException {
+ for (JsonNode issueNode: pageData) {
+ if (Thread.interrupted())
+ throw new InterruptedException();
+
+ Map<String, String> extraIssueInfo = new LinkedHashMap<>();
+
+ Issue issue = new Issue();
+ issue.setProject(oneDevProject);
+ issue.setTitle(issueNode.get("title").asText());
+ issue.setDescription(issueNode.get("body").asText(null));
+ issue.setNumberScope(oneDevProject);
+
+ Long oldNumber = issueNode.get("number").asLong();
+ Long newNumber;
+ if (dryRun || useExistingIssueNumbers)
+ newNumber = oldNumber;
+ else
+ newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);
+ issue.setNumber(newNumber);
+ issueNumberMappings.put(oldNumber, newNumber);
+
+ if (issueNode.get("state").asText().equals("closed"))
+ issue.setState(importOption.getClosedIssueState());
+ else
+ issue.setState(initialIssueState);
+
+ if (issueNode.hasNonNull("milestone")) {
+ String milestoneName = issueNode.get("milestone").get("title").asText();
+ Milestone milestone = milestoneMappings.get(milestoneName);
+ if (milestone != null) {
+ issue.setMilestone(milestone);
+ } else {
+ extraIssueInfo.put("Milestone", milestoneName);
+ nonExistentMilestones.add(milestoneName);
+ }
+ }
+
+ String login = issueNode.get("user").get("login").asText(null);
+ User user = getUser(client, importSource, users, login, logger);
+ if (user != null) {
+ issue.setSubmitter(user);
+ } else {
+ issue.setSubmitterName(login);
+ nonExistentLogins.add(login);
+ }
+
+ issue.setSubmitDate(ISODateTimeFormat.dateTimeNoMillis()
+ .parseDateTime(issueNode.get("created_at").asText())
+ .toDate());
+
+ LastUpdate lastUpdate = new LastUpdate();
+ lastUpdate.setActivity("Opened");
+ lastUpdate.setDate(issue.getSubmitDate());
+ lastUpdate.setUser(issue.getSubmitter());
+ lastUpdate.setUserName(issue.getSubmitterName());
+ issue.setLastUpdate(lastUpdate);
+
+ for (JsonNode assigneeNode: issueNode.get("assignees")) {
+ IssueField assigneeField = new IssueField();
+ assigneeField.setIssue(issue);
+ assigneeField.setName(importOption.getAssigneesIssueField());
+ assigneeField.setType(InputSpec.USER);
+
+ login = assigneeNode.get("login").asText();
+ user = getUser(client, importSource, users, login, logger);
+ if (user != null) {
+ assigneeField.setValue(user.getName());
+ } else {
+ assigneeField.setValue(login);
+ nonExistentLogins.add(login);
+ }
+ issue.getFields().add(assigneeField);
+ }
+
+ String apiEndpoint = importSource.getApiEndpoint("/repos/" + gitHubRepo
+ + "/issues/" + oldNumber + "/comments");
+ for (JsonNode commentNode: list(client, apiEndpoint, logger)) {
+ IssueComment comment = new IssueComment();
+ comment.setIssue(issue);
+ comment.setContent(commentNode.get("body").asText(null));
+ comment.setDate(ISODateTimeFormat.dateTimeNoMillis()
+ .parseDateTime(commentNode.get("created_at").asText())
+ .toDate());
+
+ login = commentNode.get("user").get("login").asText();
+ user = getUser(client, importSource, users, login, logger);
+ if (user != null) {
+ comment.setUser(user);
+ } else {
+ comment.setUserName(login);
+ nonExistentLogins.add(login);
+ }
+
+ issue.getComments().add(comment);
+ }
+
+ issue.setCommentCount(issue.getComments().size());
+
+ apiEndpoint = importSource.getApiEndpoint("/repos/" + gitHubRepo
+ + "/issues/" + oldNumber + "/labels");
+ List<String> currentUnmappedLabels = new ArrayList<>();
+ for (JsonNode labelNode: list(client, apiEndpoint, logger)) {
+ String labelName = labelNode.get("name").asText();
+ Pair<FieldSpec, String> mapped = labelMappings.get(labelName);
+ if (mapped != null) {
+ IssueField tagField = new IssueField();
+ tagField.setIssue(issue);
+ tagField.setName(mapped.getFirst().getName());
+ tagField.setType(InputSpec.ENUMERATION);
+ tagField.setValue(mapped.getSecond());
+ tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
+ issue.getFields().add(tagField);
+ } else {
+ currentUnmappedLabels.add(HtmlEscape.escapeHtml5(labelName));
+ unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));
+ }
+ }
+
+ if (!currentUnmappedLabels.isEmpty())
+ extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));
+
+ if (!extraIssueInfo.isEmpty()) {
+ StringBuilder builder = new StringBuilder("|");
+ for (String key: extraIssueInfo.keySet())
+ builder.append(key).append("|");
+ builder.append("\n|");
+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
+ builder.append("\n|");
+ for (String value: extraIssueInfo.values())
+ builder.append(value).append("|");
+
+ if (issue.getDescription() != null)
+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
+ else
+ issue.setDescription(builder.toString());
+ }
+ issues.add(issue);
+ }
+ logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");
+ }
+
+ };
+
+ String apiEndpoint = importSource.getApiEndpoint("/repos/" + gitHubRepo + "/issues?state=all");
+ list(client, apiEndpoint, pageDataConsumer, logger);
+
+ if (!dryRun) {
+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);
+ for (Issue issue: issues) {
+ if (issue.getDescription() != null)
+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));
+
+ OneDev.getInstance(IssueManager.class).save(issue);
+ for (IssueField field: issue.getFields())
+ OneDev.getInstance(Dao.class).persist(field);
+ for (IssueComment comment: issue.getComments()) {
+ comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));
+ OneDev.getInstance(Dao.class).persist(comment);
+ }
+ }
+ }
+
+ GitHubImportResult result = new GitHubImportResult();
+ result.nonExistentLogins.addAll(nonExistentLogins);
+ result.nonExistentMilestones.addAll(nonExistentMilestones);
+ result.unmappedIssueLabels.addAll(unmappedIssueLabels);
+
+ if (numOfImportedIssues.get() != 0)
+ result.issuesImported = true;
+
+ return result;
+ } finally {
+ client.close();
+ }
+ }
+
+ static GlobalIssueSetting getIssueSetting() {
+ return OneDev.getInstance(SettingManager.class).getIssueSetting();
+ }
+
+ static List<JsonNode> list(Client client, String apiEndpoint, SimpleLogger logger) {
+ List<JsonNode> result = new ArrayList<>();
+ list(client, apiEndpoint, new PageDataConsumer() {
+
+ @Override
+ public void consume(List<JsonNode> pageData) {
+ result.addAll(pageData);
+ }
+
+ }, logger);
+ return result;
+ }
+
+ static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,
+ SimpleLogger logger) {
+ URI uri;
+ try {
+ uri = new URIBuilder(apiEndpoint)
+ .addParameter("per_page", String.valueOf(PER_PAGE)).build();
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+
+ int page = 1;
+ while (true) {
+ try {
+ URIBuilder builder = new URIBuilder(uri);
+ builder.addParameter("page", String.valueOf(page));
+ List<JsonNode> pageData = new ArrayList<>();
+ for (JsonNode each: get(client, builder.build().toString(), logger))
+ pageData.add(each);
+ pageDataConsumer.consume(pageData);
+ if (pageData.size() < PER_PAGE)
+ break;
+ page++;
+ } catch (URISyntaxException|InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static JsonNode get(Client client, String apiEndpoint, SimpleLogger logger) {
+ WebTarget target = client.target(apiEndpoint);
+ Invocation.Builder builder = target.request();
+ while (true) {
+ try (Response response = builder.get()) {
+ int status = response.getStatus();
+ if (status != 200) {
+ String errorMessage = response.readEntity(String.class);
+ if (StringUtils.isNotBlank(errorMessage)) {
+ if (errorMessage.contains("rate limit exceeded")) {
+ long resetTime = Long.parseLong(response.getHeaderString("x-ratelimit-reset"))*1000L;
+ logger.log("Rate limit exceeded, wait until reset...");
+ try {
+ Thread.sleep(resetTime + 60*1000L - System.currentTimeMillis());
+ continue;
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ throw new ExplicitException(String.format("Http request failed (url: %s, status code: %d, error message: %s)",
+ apiEndpoint, status, errorMessage));
+ }
+ } else {
+ throw new ExplicitException(String.format("Http request failed (status: %s)", status));
+ }
+ }
+ return response.readEntity(JsonNode.class);
+ }
+ }
+ }
+
+ static Client newClient(GitHubProjectImportSource importSource) {
+ Client client = ClientBuilder.newClient();
+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);
+ client.register(HttpAuthenticationFeature.basic("git", importSource.getAccessToken()));
+ return client;
+ }
+
+ static interface PageDataConsumer {
+
+ void consume(List<JsonNode> pageData) throws InterruptedException;
+
+ }
+
+}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.java
deleted file mode 100644
index d265f323c9..0000000000
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.java
+++ /dev/null
@@ -1,517 +0,0 @@
-package io.onedev.server.plugin.imports.github;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import javax.annotation.Nullable;
-import javax.ws.rs.client.Client;
-import javax.ws.rs.client.ClientBuilder;
-import javax.ws.rs.client.Invocation;
-import javax.ws.rs.client.WebTarget;
-import javax.ws.rs.core.Response;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
-import org.glassfish.jersey.client.ClientProperties;
-import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
-import org.joda.time.format.ISODateTimeFormat;
-import org.unbescape.html.HtmlEscape;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-import io.onedev.commons.utils.ExplicitException;
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.IssueManager;
-import io.onedev.server.entitymanager.MilestoneManager;
-import io.onedev.server.entitymanager.ProjectManager;
-import io.onedev.server.entitymanager.SettingManager;
-import io.onedev.server.entitymanager.UserManager;
-import io.onedev.server.model.Issue;
-import io.onedev.server.model.IssueComment;
-import io.onedev.server.model.IssueField;
-import io.onedev.server.model.Milestone;
-import io.onedev.server.model.Project;
-import io.onedev.server.model.User;
-import io.onedev.server.model.support.LastUpdate;
-import io.onedev.server.model.support.administration.GlobalIssueSetting;
-import io.onedev.server.model.support.inputspec.InputSpec;
-import io.onedev.server.model.support.issue.field.spec.FieldSpec;
-import io.onedev.server.persistence.dao.Dao;
-import io.onedev.server.storage.StorageManager;
-import io.onedev.server.util.Pair;
-import io.onedev.server.util.SimpleLogger;
-import io.onedev.server.web.page.project.imports.ProjectImporter;
-
-public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImportOption> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String NAME = "GitHub";
-
- private static final int PER_PAGE = 100;
-
- @Override
- public String getName() {
- return NAME;
- }
-
- @Override
- public GitHubImportOption getImportOption(GitHubImportSource importSource, SimpleLogger logger) {
- GitHubImportOption importOption = new GitHubImportOption();
- if (importSource.isPrepopulateImportOptions()) {
- Client client = newClient(importSource);
- try {
- Set<String> labels = new LinkedHashSet<>();
- for (JsonNode repoNode: list(client, importSource, "/user/repos", logger)) {
- String repoName = repoNode.get("name").asText();
- String ownerName = repoNode.get("owner").get("login").asText();
- GitHubImport aImport = new GitHubImport();
- aImport.setGitHubRepo(ownerName + "/" + repoName);
- aImport.setOneDevProject(ownerName + "-" + repoName);
- importOption.getImports().add(aImport);
-
- for (JsonNode labelNode: list(client, importSource,
- "/repos/" + aImport.getGitHubRepo() + "/labels", logger)) {
- labels.add(labelNode.get("name").asText());
- }
- }
- for (String label: labels) {
- IssueLabelMapping mapping = new IssueLabelMapping();
- mapping.setGitHubIssueLabel(label);
- importOption.getIssueLabelMappings().add(mapping);
- }
- } finally {
- client.close();
- }
- }
- return importOption;
- }
-
- @Nullable
- private User getUser(Client client, GitHubImportSource importSource,
- Map<String, Optional<User>> users, String login, SimpleLogger logger) {
- Optional<User> userOpt = users.get(login);
- if (userOpt == null) {
- String email = get(client, importSource, "/users/" + login, logger).get("email").asText(null);
- if (email != null)
- userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByEmail(email));
- else
- userOpt = Optional.empty();
- users.put(login, userOpt);
- }
- return userOpt.orElse(null);
- }
-
- @Override
- public String doImport(GitHubImportSource importSource, GitHubImportOption importOption,
- boolean dryRun, SimpleLogger logger) {
- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
- StorageManager storageManager = OneDev.getInstance(StorageManager.class);
- Client client = newClient(importSource);
-
- Set<String> nonExistentLogins = new HashSet<>();
- Set<String> unmappedIssueLabels = new HashSet<>();
-
- Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();
-
- for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {
- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
- if (fieldSpec == null)
- throw new ExplicitException("No field spec found: " + oneDevFieldName);
- labelMappings.put(mapping.getGitHubIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));
- }
-
- Collection<Long> projectIds = new ArrayList<>();
- try {
- Map<String, Optional<User>> users = new HashMap<>();
- boolean importIssues = false;
- for (GitHubImport aImport: importOption.getImports()) {
- logger.log("Cloning code of repository " + aImport.getGitHubRepo() + "...");
- JsonNode repoNode = get(client, importSource, "/repos/" + aImport.getGitHubRepo(), logger);
- Project project = new Project();
- project.setName(aImport.getOneDevProject());
- project.setDescription(repoNode.get("description").asText(null));
- project.setIssueManagementEnabled(repoNode.get("has_issues").asBoolean());
- boolean isPrivate = repoNode.get("private").asBoolean();
- if (!isPrivate && importOption.getPublicRole() != null)
- project.setDefaultRole(importOption.getPublicRole());
-
- URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());
- if (isPrivate)
- builder.setUserInfo("git", importSource.getAccessToken());
-
- if (!dryRun) {
- projectManager.clone(project, builder.build().toString());
- projectIds.add(project.getId());
- }
-
- if (aImport.isImportIssues()) {
- importIssues = true;
- Map<Long, Milestone> milestones = new HashMap<>();
- logger.log("Importing milestones of repository " + aImport.getGitHubRepo() + "...");
- String apiEndpoint = "/repos/" + aImport.getGitHubRepo() + "/milestones?state=all";
- for (JsonNode milestoneNode: list(client, importSource, apiEndpoint, logger)) {
- Milestone milestone = new Milestone();
- milestone.setName(milestoneNode.get("title").asText());
- milestone.setDescription(milestoneNode.get("description").asText(null));
- milestone.setProject(project);
- String dueDateString = milestoneNode.get("due_on").asText(null);
- if (dueDateString != null)
- milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());
- if (milestoneNode.get("state").asText().equals("closed"))
- milestone.setClosed(true);
-
- if (!dryRun)
- OneDev.getInstance(MilestoneManager.class).save(milestone);
- milestones.put(milestoneNode.get("number").asLong(), milestone);
- }
-
- logger.log("Importing issues of repository " + aImport.getGitHubRepo() + "...");
- String initialIssueState = getIssueSetting().getInitialStateSpec().getName();
-
- AtomicInteger numOfImportedIssues = new AtomicInteger(0);
- PageDataConsumer pageDataConsumer = new PageDataConsumer() {
-
- /*
- @Nullable
- private String processAttachments(String issueUUID, Long issueNumber, @Nullable String markdown,
- Set<String> tooLargeAttachments) {
- if (markdown == null)
- return markdown;
-
- Map<String, String> attachments = new HashMap<>();
-
- Node node = OneDev.getInstance(MarkdownManager.class).parse(markdown);
- new NodeVisitor(new VisitHandler<?>[] {}) {
-
- @Override
- public void visit(Node node) {
- super.visit(node);
- if (node instanceof InlineLinkNode) {
- InlineLinkNode link = (InlineLinkNode)node;
- String url = link.getUrl().toString();
- if (url.startsWith(repoUrl + "/files/")) {
- attachments.put(url, link.getText().toString());
- }
- }
- }
-
- }.visit(node);
-
- for (Map.Entry<String, String> entry: attachments.entrySet()) {
- String attachmentUrl = entry.getKey();
- String attachmentName = entry.getValue();
-
- WebTarget target = client.target(attachmentUrl);
- Invocation.Builder builder = target.request();
- try (Response response = builder.get()) {
- String errorMessage = JerseyUtils.checkStatus(response);
- if (errorMessage != null) {
- throw new ExplicitException(String.format(
- "Error downloading attachment (url: %s, error message: %s)",
- attachmentUrl, errorMessage));
- }
- try (InputStream is = response.readEntity(InputStream.class)) {
- String oneDevAttachmentName = project.saveAttachment(issueUUID, attachmentName, is);
- String oneDevAttachmentUrl = project.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);
- markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");
- } catch (AttachmentTooLargeException e) {
- tooLargeAttachments.add(aImport.getGitHubRepo() + "#" + issueNumber + ":" + attachmentName);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- return markdown;
- }
- */
-
- private String joinAsMultilineHtml(List<String> values) {
- List<String> escapedValues = new ArrayList<>();
- for (String value: values)
- escapedValues.add(HtmlEscape.escapeHtml5(value));
- return StringUtils.join(escapedValues, "<br>");
- }
-
- @Override
- public void consume(List<JsonNode> pageData) throws InterruptedException {
- for (JsonNode issueNode: pageData) {
- if (Thread.interrupted())
- throw new InterruptedException();
- Issue issue = new Issue();
- issue.setProject(project);
- issue.setTitle(issueNode.get("title").asText());
- issue.setDescription(issueNode.get("body").asText(null));
- issue.setNumberScope(project);
- issue.setNumber(issueNode.get("number").asLong());
- if (issueNode.get("state").asText().equals("closed"))
- issue.setState(importOption.getClosedIssueState());
- else
- issue.setState(initialIssueState);
- if (issueNode.hasNonNull("milestone")) {
- long milestoneNumber = issueNode.get("milestone").get("number").asLong();
- Milestone milestone = milestones.get(milestoneNumber);
- if (milestone != null)
- issue.setMilestone(milestone);
- }
- String login = issueNode.get("user").get("login").asText(null);
- User user = getUser(client, importSource, users, login, logger);
- if (user != null) {
- issue.setSubmitter(user);
- } else {
- issue.setSubmitterName(login);
- nonExistentLogins.add(login);
- }
-
- issue.setSubmitDate(ISODateTimeFormat.dateTimeNoMillis()
- .parseDateTime(issueNode.get("created_at").asText())
- .toDate());
-
- LastUpdate lastUpdate = new LastUpdate();
- lastUpdate.setActivity("Opened");
- lastUpdate.setDate(issue.getSubmitDate());
- lastUpdate.setUser(issue.getSubmitter());
- lastUpdate.setUserName(issue.getSubmitterName());
- issue.setLastUpdate(lastUpdate);
-
- Map<String, String> extraIssueInfo = new LinkedHashMap<>();
-
- for (JsonNode assigneeNode: issueNode.get("assignees")) {
- IssueField assigneeField = new IssueField();
- assigneeField.setIssue(issue);
- assigneeField.setName(importOption.getAssigneesIssueField());
- assigneeField.setType(InputSpec.USER);
-
- login = assigneeNode.get("login").asText();
- user = getUser(client, importSource, users, login, logger);
- if (user != null) {
- assigneeField.setValue(user.getName());
- } else {
- assigneeField.setValue(login);
- nonExistentLogins.add(login);
- }
- issue.getFields().add(assigneeField);
- }
-
- String apiEndpoint = "/repos/" + aImport.getGitHubRepo()
- + "/issues/" + issue.getNumber() + "/comments";
- for (JsonNode commentNode: list(client, importSource, apiEndpoint, logger)) {
- IssueComment comment = new IssueComment();
- comment.setIssue(issue);
- comment.setContent(commentNode.get("body").asText(null));
- comment.setDate(ISODateTimeFormat.dateTimeNoMillis()
- .parseDateTime(commentNode.get("created_at").asText())
- .toDate());
-
- login = commentNode.get("user").get("login").asText();
- user = getUser(client, importSource, users, login, logger);
- if (user != null) {
- comment.setUser(user);
- } else {
- comment.setUserName(login);
- nonExistentLogins.add(login);
- }
-
- issue.getComments().add(comment);
- }
-
- issue.setCommentCount(issue.getComments().size());
-
- apiEndpoint = "/repos/" + aImport.getGitHubRepo()
- + "/issues/" + issue.getNumber() + "/labels";
- List<String> currentUnmappedLabels = new ArrayList<>();
- for (JsonNode labelNode: list(client, importSource, apiEndpoint, logger)) {
- String labelName = labelNode.get("name").asText();
- Pair<FieldSpec, String> mapped = labelMappings.get(labelName);
- if (mapped != null) {
- IssueField tagField = new IssueField();
- tagField.setIssue(issue);
- tagField.setName(mapped.getFirst().getName());
- tagField.setType(InputSpec.ENUMERATION);
- tagField.setValue(mapped.getSecond());
- tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
- issue.getFields().add(tagField);
- } else {
- currentUnmappedLabels.add(HtmlEscape.escapeHtml5(labelName));
- unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));
- }
- }
-
- if (!currentUnmappedLabels.isEmpty())
- extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));
-
- if (!extraIssueInfo.isEmpty()) {
- StringBuilder builder = new StringBuilder("|");
- for (String key: extraIssueInfo.keySet())
- builder.append(key).append("|");
- builder.append("\n|");
- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
- builder.append("\n|");
- for (String value: extraIssueInfo.values())
- builder.append(value).append("|");
-
- if (issue.getDescription() != null)
- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
- else
- issue.setDescription(builder.toString());
- }
-
- if (!dryRun) {
- OneDev.getInstance(IssueManager.class).save(issue);
- for (IssueField field: issue.getFields())
- OneDev.getInstance(Dao.class).persist(field);
- for (IssueComment comment: issue.getComments())
- OneDev.getInstance(Dao.class).persist(comment);
- }
-
- if (numOfImportedIssues.incrementAndGet() % 25 == 0)
- logger.log("Imported " + numOfImportedIssues.get() + " issues");
- }
- }
-
- };
- apiEndpoint = "/repos/" + aImport.getGitHubRepo() + "/issues?state=all";
- list(client, importSource, apiEndpoint, pageDataConsumer, logger);
- }
- }
-
- StringBuilder feedback = new StringBuilder("Repositories imported successfully");
-
- boolean hasNotes = !unmappedIssueLabels.isEmpty() || !nonExistentLogins.isEmpty() || importIssues;
-
- if (hasNotes)
- feedback.append("<br><br><b>NOTE:</b><ul>");
-
- if (!unmappedIssueLabels.isEmpty()) {
- feedback.append("<li> GitHub issue labels not mapped to OneDev custom field (mentioned as extra info in issue description): "
- + HtmlEscape.escapeHtml5(unmappedIssueLabels.toString()));
- }
- if (!nonExistentLogins.isEmpty()) {
- feedback.append("<li> GitHub logins without public email or public email can not be mapped to OneDev account: "
- + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));
- }
- if (importIssues)
- feedback.append("<li> Attachments in issues and comments are not imported due to GitHub limitation");
-
- if (hasNotes)
- feedback.append("</ul>");
-
- return feedback.toString();
- } catch (Exception e) {
- for (Long projectId: projectIds)
- storageManager.deleteProjectDir(projectId);
- throw new RuntimeException(e);
- } finally {
- client.close();
- }
- }
-
- private GlobalIssueSetting getIssueSetting() {
- return OneDev.getInstance(SettingManager.class).getIssueSetting();
- }
-
- private List<JsonNode> list(Client client, GitHubImportSource importSource,
- String apiPath, SimpleLogger logger) {
- List<JsonNode> result = new ArrayList<>();
- list(client, importSource, apiPath, new PageDataConsumer() {
-
- @Override
- public void consume(List<JsonNode> pageData) {
- result.addAll(pageData);
- }
-
- }, logger);
- return result;
- }
-
- private void list(Client client, GitHubImportSource importSource, String apiPath,
- PageDataConsumer pageDataConsumer, SimpleLogger logger) {
- URI uri;
- try {
- uri = new URIBuilder(importSource.getApiEndpoint(apiPath))
- .addParameter("per_page", String.valueOf(PER_PAGE)).build();
- } catch (URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- int page = 1;
- while (true) {
- try {
- URIBuilder builder = new URIBuilder(uri);
- builder.addParameter("page", String.valueOf(page));
- List<JsonNode> pageData = new ArrayList<>();
- for (JsonNode each: get(client, builder.build().toString(), logger))
- pageData.add(each);
- pageDataConsumer.consume(pageData);
- if (pageData.size() < PER_PAGE)
- break;
- page++;
- } catch (URISyntaxException|InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- private JsonNode get(Client client, GitHubImportSource importSource, String apiPath, SimpleLogger logger) {
- return get(client, importSource.getApiEndpoint(apiPath), logger);
- }
-
- private JsonNode get(Client client, String apiEndpoint, SimpleLogger logger) {
- WebTarget target = client.target(apiEndpoint);
- Invocation.Builder builder = target.request();
- while (true) {
- try (Response response = builder.get()) {
- int status = response.getStatus();
- if (status != 200) {
- String errorMessage = response.readEntity(String.class);
- if (StringUtils.isNotBlank(errorMessage)) {
- if (errorMessage.contains("rate limit exceeded")) {
- long resetTime = Long.parseLong(response.getHeaderString("x-ratelimit-reset"))*1000L;
- logger.log("Rate limit exceeded, wait until reset...");
- try {
- Thread.sleep(resetTime + 60*1000L - System.currentTimeMillis());
- continue;
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- } else {
- throw new ExplicitException(String.format("Http request failed (status code: %d, error message: %s)",
- status, errorMessage));
- }
- } else {
- throw new ExplicitException(String.format("Http request failed (status: %s)", status));
- }
- }
- return response.readEntity(JsonNode.class);
- }
- }
- }
-
- private Client newClient(GitHubImportSource importSource) {
- Client client = ClientBuilder.newClient();
- client.property(ClientProperties.FOLLOW_REDIRECTS, true);
- client.register(HttpAuthenticationFeature.basic("git", importSource.getAccessToken()));
- return client;
- }
-
- private static interface PageDataConsumer {
-
- void consume(List<JsonNode> pageData) throws InterruptedException;
-
- }
-
-}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportOption.java
similarity index 51%
rename from server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.java
rename to server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportOption.java
index 134c367ad0..255039723c 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportOption.java
@@ -5,72 +5,27 @@ import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
-import javax.annotation.Nullable;
-import javax.validation.ConstraintValidatorContext;
-import javax.validation.constraints.Size;
-
import org.hibernate.validator.constraints.NotEmpty;
import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.ProjectManager;
-import io.onedev.server.entitymanager.RoleManager;
import io.onedev.server.entitymanager.SettingManager;
-import io.onedev.server.model.Role;
import io.onedev.server.model.support.administration.GlobalIssueSetting;
import io.onedev.server.model.support.inputspec.InputSpec;
import io.onedev.server.model.support.issue.field.spec.FieldSpec;
-import io.onedev.server.util.validation.Validatable;
-import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.ChoiceProvider;
import io.onedev.server.web.editable.annotation.Editable;
-import io.onedev.server.web.editable.annotation.RoleChoice;
@Editable
-@ClassValidating
-public class GitHubImportOption implements Serializable, Validatable {
+public class GitHubIssueImportOption implements Serializable {
private static final long serialVersionUID = 1L;
- private List<GitHubImport> imports = new ArrayList<>();
-
- private String publicRoleName;
-
private String closedIssueState;
private String assigneesIssueField;
private List<IssueLabelMapping> issueLabelMappings = new ArrayList<>();
- @Editable(order=100, name="Repositories to Import")
- @Size(min=1, max=10000, message="No repositories to import")
- public List<GitHubImport> getImports() {
- return imports;
- }
-
- public void setImports(List<GitHubImport> imports) {
- this.imports = imports;
- }
-
- @Editable(order=200, name="Public Role", description="If specified, all public repositories imported from GitHub "
- + "will use this as default role. Private repositories are not affected")
- @RoleChoice
- @Nullable
- public String getPublicRoleName() {
- return publicRoleName;
- }
-
- public void setPublicRoleName(String publicRoleName) {
- this.publicRoleName = publicRoleName;
- }
-
- @Nullable
- public Role getPublicRole() {
- if (publicRoleName != null)
- return OneDev.getInstance(RoleManager.class).find(publicRoleName);
- else
- return null;
- }
-
@Editable(order=300, description="Specify which issue state to use for closed GitHub issues")
@ChoiceProvider("getCloseStateChoices")
@NotEmpty
@@ -127,34 +82,4 @@ public class GitHubImportOption implements Serializable, Validatable {
this.issueLabelMappings = issueLabelMappings;
}
- @Override
- public boolean isValid(ConstraintValidatorContext context) {
- boolean isValid = true;
- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
- for (int i=0; i<imports.size(); i++) {
- if (projectManager.find(imports.get(i).getOneDevProject()) != null) {
- context.buildConstraintViolationWithTemplate("Project name already used")
- .addPropertyNode("imports")
- .addPropertyNode(GitHubImport.PROP_ONEDEV_PROJECT)
- .inIterable().atIndex(i).addConstraintViolation();
- isValid = false;
- } else {
- for (int j=0; j<imports.size(); j++) {
- if (j != i && imports.get(j).getOneDevProject().equals(imports.get(i).getOneDevProject())) {
- context.buildConstraintViolationWithTemplate("Duplicate project name")
- .addPropertyNode("imports")
- .addPropertyNode(GitHubImport.PROP_ONEDEV_PROJECT)
- .inIterable().atIndex(i).addConstraintViolation();
- isValid = false;
- break;
- }
- }
- }
- }
-
- if (!isValid)
- context.disableDefaultConstraintViolation();
- return isValid;
- }
-
}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportSource.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportSource.java
new file mode 100644
index 0000000000..aec0a615d1
--- /dev/null
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportSource.java
@@ -0,0 +1,88 @@
+package io.onedev.server.plugin.imports.github;
+
+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.get;
+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.list;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+
+import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
+import org.hibernate.validator.constraints.NotEmpty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import io.onedev.commons.utils.StringUtils;
+import io.onedev.server.util.EditContext;
+import io.onedev.server.util.SimpleLogger;
+import io.onedev.server.util.validation.annotation.ClassValidating;
+import io.onedev.server.web.editable.annotation.ChoiceProvider;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+@ClassValidating
+public class GitHubIssueImportSource extends GitHubProjectImportSource {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger logger = LoggerFactory.getLogger(GitHubIssueImportSource.class);
+
+ private String repository;
+
+ @Editable(order=400, name="GitHub Repository", description="Choose GitHub repository to "
+ + "import issues from. Available repositories will be populated when above options "
+ + "are specified")
+ @ChoiceProvider("getRepositoryChoices")
+ @NotEmpty
+ public String getRepository() {
+ return repository;
+ }
+
+ public void setRepository(String repository) {
+ this.repository = repository;
+ }
+
+ @SuppressWarnings("unused")
+ private static List<String> getRepositoryChoices() {
+ List<String> choices = new ArrayList<>();
+
+ String apiUrl = (String) EditContext.get().getInputValue(PROP_API_URL);
+ String accessToken = (String) EditContext.get().getInputValue(PROP_ACCESS_TOKEN);
+
+ if (StringUtils.isNotBlank(apiUrl) && StringUtils.isNotBlank(accessToken)) {
+ Client client = ClientBuilder.newClient();
+ client.register(HttpAuthenticationFeature.basic("git", accessToken));
+ try {
+ String apiEndpoint = getApiEndpoint(apiUrl, "/user");
+ SimpleLogger logger = new SimpleLogger() {
+
+ @Override
+ public void log(String message) {
+ GitHubIssueImportSource.logger.info(message);
+ }
+
+ };
+ get(client, apiEndpoint, logger);
+
+ apiEndpoint = getApiEndpoint(apiUrl, "/user/repos");
+ for (JsonNode repoNode: list(client, apiEndpoint, logger)) {
+ String repoName = repoNode.get("name").asText();
+ String ownerName = repoNode.get("owner").get("login").asText();
+ choices.add(ownerName + "/" + repoName);
+ }
+ } catch (Exception e) {
+ } finally {
+ client.close();
+ }
+ }
+
+ Collections.sort(choices);
+ return choices;
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java
new file mode 100644
index 0000000000..58de5a7bc9
--- /dev/null
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java
@@ -0,0 +1,44 @@
+package io.onedev.server.plugin.imports.github;
+
+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.NAME;
+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.buildImportOption;
+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.importIssues;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import io.onedev.server.model.Project;
+import io.onedev.server.model.User;
+import io.onedev.server.util.SimpleLogger;
+import io.onedev.server.web.page.project.issues.imports.IssueImporter;
+
+public class GitHubIssueImporter extends IssueImporter<GitHubIssueImportSource, GitHubIssueImportOption> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public GitHubIssueImportOption getImportOption(GitHubIssueImportSource importSource, SimpleLogger logger) {
+ GitHubIssueImportOption importOption;
+ if (importSource.isPrepopulateImportOptions())
+ importOption = buildImportOption(importSource, importSource.getRepository(), logger);
+ else
+ importOption = new GitHubIssueImportOption();
+ return importOption;
+ }
+
+ @Override
+ public String doImport(Project project, GitHubIssueImportSource importSource, GitHubIssueImportOption importOption,
+ boolean dryRun, SimpleLogger logger) {
+ logger.log("Importing issues from repository " + importSource.getRepository() + "...");
+ Map<String, Optional<User>> users = new HashMap<>();
+ return importIssues(importSource, importSource.getRepository(), project, false, importOption, users, dryRun, logger)
+ .toHtml("Issues imported successfully");
+ }
+
+}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java
index 5330335a2d..21be1145de 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java
@@ -8,6 +8,8 @@ import com.google.common.collect.Lists;
import io.onedev.commons.launcher.loader.AbstractPluginModule;
import io.onedev.server.web.page.project.imports.ProjectImporter;
import io.onedev.server.web.page.project.imports.ProjectImporterContribution;
+import io.onedev.server.web.page.project.issues.imports.IssueImporter;
+import io.onedev.server.web.page.project.issues.imports.IssueImporterContribution;
/**
* NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.
@@ -23,11 +25,19 @@ public class GitHubPluginModule extends AbstractPluginModule {
@Override
public Collection<ProjectImporter<? extends Serializable, ? extends Serializable>> getImporters() {
- return Lists.newArrayList(new GitHubImporter());
+ return Lists.newArrayList(new GitHubProjectImporter());
}
});
+ contribute(IssueImporterContribution.class, new IssueImporterContribution() {
+
+ @Override
+ public Collection<IssueImporter<? extends Serializable, ? extends Serializable>> getImporters() {
+ return Lists.newArrayList(new GitHubIssueImporter());
+ }
+
+ });
}
}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportOption.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportOption.java
new file mode 100644
index 0000000000..d4e144a0aa
--- /dev/null
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportOption.java
@@ -0,0 +1,89 @@
+package io.onedev.server.plugin.imports.github;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.Nullable;
+import javax.validation.ConstraintValidatorContext;
+import javax.validation.constraints.Size;
+
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.entitymanager.RoleManager;
+import io.onedev.server.model.Role;
+import io.onedev.server.util.validation.Validatable;
+import io.onedev.server.util.validation.annotation.ClassValidating;
+import io.onedev.server.web.editable.annotation.Editable;
+import io.onedev.server.web.editable.annotation.RoleChoice;
+
+@Editable
+@ClassValidating
+public class GitHubProjectImportOption extends GitHubIssueImportOption implements Validatable {
+
+ private static final long serialVersionUID = 1L;
+
+ private List<ProjectMapping> projectMappings = new ArrayList<>();
+
+ private String publicRoleName;
+
+ @Editable(order=200, name="Public Role", description="If specified, all public repositories imported from GitHub "
+ + "will use this as default role. Private repositories are not affected")
+ @RoleChoice
+ @Nullable
+ public String getPublicRoleName() {
+ return publicRoleName;
+ }
+
+ public void setPublicRoleName(String publicRoleName) {
+ this.publicRoleName = publicRoleName;
+ }
+
+ @Nullable
+ public Role getPublicRole() {
+ if (publicRoleName != null)
+ return OneDev.getInstance(RoleManager.class).find(publicRoleName);
+ else
+ return null;
+ }
+
+ @Editable(order=100, name="Repositories to Import")
+ @Size(min=1, max=10000, message="No repositories to import")
+ public List<ProjectMapping> getProjectMappings() {
+ return projectMappings;
+ }
+
+ public void setProjectMappings(List<ProjectMapping> projectMappings) {
+ this.projectMappings = projectMappings;
+ }
+
+ @Override
+ public boolean isValid(ConstraintValidatorContext context) {
+ boolean isValid = true;
+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
+ for (int i=0; i<projectMappings.size(); i++) {
+ if (projectManager.find(projectMappings.get(i).getOneDevProject()) != null) {
+ context.buildConstraintViolationWithTemplate("Project name already used")
+ .addPropertyNode("imports")
+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
+ .inIterable().atIndex(i).addConstraintViolation();
+ isValid = false;
+ } else {
+ for (int j=0; j<projectMappings.size(); j++) {
+ if (j != i && projectMappings.get(j).getOneDevProject().equals(projectMappings.get(i).getOneDevProject())) {
+ context.buildConstraintViolationWithTemplate("Duplicate project name")
+ .addPropertyNode("imports")
+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
+ .inIterable().atIndex(i).addConstraintViolation();
+ isValid = false;
+ break;
+ }
+ }
+ }
+ }
+
+ if (!isValid)
+ context.disableDefaultConstraintViolation();
+ return isValid;
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportSource.java
similarity index 83%
rename from server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.java
rename to server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportSource.java
index 3827aad627..1ca2ea3a80 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportSource.java
@@ -17,16 +17,17 @@ import io.onedev.server.util.JerseyUtils;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
+import io.onedev.server.web.editable.annotation.Password;
@Editable
@ClassValidating
-public class GitHubImportSource implements Serializable, Validatable {
+public class GitHubProjectImportSource implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
- private static final String PROP_API_URL = "apiUrl";
+ static final String PROP_API_URL = "apiUrl";
- private static final String PROP_ACCESS_TOKEN = "accessToken";
+ static final String PROP_ACCESS_TOKEN = "accessToken";
private String apiUrl = "https://api.github.com";
@@ -46,6 +47,7 @@ public class GitHubImportSource implements Serializable, Validatable {
@Editable(order=100, name="GitHub Personal Access Token", description="GitHub personal access token should be generated with "
+ "scope <b>repo</b> and <b>read:org</b>")
+ @Password
@NotEmpty
public String getAccessToken() {
return accessToken;
@@ -55,7 +57,7 @@ public class GitHubImportSource implements Serializable, Validatable {
this.accessToken = accessToken;
}
- @Editable(order=200, description="If checked, import options will be pre-populated based on all accessible "
+ @Editable(order=1000, description="If checked, import options will be pre-populated based on all accessible "
+ "repositories and its settings")
public boolean isPrepopulateImportOptions() {
return prepopulateImportOptions;
@@ -65,16 +67,21 @@ public class GitHubImportSource implements Serializable, Validatable {
this.prepopulateImportOptions = prepopulateImportOptions;
}
- public String getApiEndpoint(String apiPath) {
+ public static String getApiEndpoint(String apiUrl, String apiPath) {
return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");
}
+
+ public String getApiEndpoint(String apiPath) {
+ return getApiEndpoint(apiUrl, apiPath);
+ }
@Override
public boolean isValid(ConstraintValidatorContext context) {
Client client = ClientBuilder.newClient();
client.register(HttpAuthenticationFeature.basic("git", getAccessToken()));
try {
- WebTarget target = client.target(getApiEndpoint("/user"));
+ String apiEndpoint = getApiEndpoint("/user");
+ WebTarget target = client.target(apiEndpoint);
Invocation.Builder builder = target.request();
try (Response response = builder.get()) {
if (response.getStatus() == 401) {
@@ -84,7 +91,7 @@ public class GitHubImportSource implements Serializable, Validatable {
.addPropertyNode(PROP_ACCESS_TOKEN).addConstraintViolation();
return false;
} else {
- String errorMessage = JerseyUtils.checkStatus(response);
+ String errorMessage = JerseyUtils.checkStatus(apiEndpoint, response);
if (errorMessage != null) {
context.disableDefaultConstraintViolation();
context.buildConstraintViolationWithTemplate(errorMessage)
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java
new file mode 100644
index 0000000000..7b543a2c5c
--- /dev/null
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java
@@ -0,0 +1,160 @@
+package io.onedev.server.plugin.imports.github;
+
+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.NAME;
+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.buildImportOption;
+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.get;
+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.importIssues;
+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.list;
+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.newClient;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import javax.annotation.Nullable;
+import javax.ws.rs.client.Client;
+
+import org.apache.http.client.utils.URIBuilder;
+import org.joda.time.format.ISODateTimeFormat;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.MilestoneManager;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.entitymanager.UserManager;
+import io.onedev.server.model.Milestone;
+import io.onedev.server.model.Project;
+import io.onedev.server.model.User;
+import io.onedev.server.storage.StorageManager;
+import io.onedev.server.util.SimpleLogger;
+import io.onedev.server.web.page.project.imports.ProjectImporter;
+
+public class GitHubProjectImporter extends ProjectImporter<GitHubProjectImportSource, GitHubProjectImportOption> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public GitHubProjectImportOption getImportOption(GitHubProjectImportSource importSource, SimpleLogger logger) {
+ GitHubProjectImportOption importOption = new GitHubProjectImportOption();
+ if (importSource.isPrepopulateImportOptions()) {
+ Client client = newClient(importSource);
+ try {
+ String apiEndpoint = importSource.getApiEndpoint("/user/repos");
+ for (JsonNode repoNode: list(client, apiEndpoint, logger)) {
+ String repoName = repoNode.get("name").asText();
+ String ownerName = repoNode.get("owner").get("login").asText();
+ ProjectMapping projectMapping = new ProjectMapping();
+ projectMapping.setGitHubRepo(ownerName + "/" + repoName);
+ projectMapping.setOneDevProject(ownerName + "-" + repoName);
+ importOption.getProjectMappings().add(projectMapping);
+ }
+ GitHubIssueImportOption issueImportOption = buildImportOption(importSource, null, logger);
+ importOption.setAssigneesIssueField(issueImportOption.getAssigneesIssueField());
+ importOption.setClosedIssueState(issueImportOption.getClosedIssueState());
+ importOption.setIssueLabelMappings(issueImportOption.getIssueLabelMappings());
+ } finally {
+ client.close();
+ }
+ }
+ return importOption;
+ }
+
+ @Nullable
+ private User getUser(Client client, GitHubProjectImportSource importSource,
+ Map<String, Optional<User>> users, String login, SimpleLogger logger) {
+ Optional<User> userOpt = users.get(login);
+ if (userOpt == null) {
+ String apiEndpoint = importSource.getApiEndpoint("/users/" + login);
+ String email = get(client, apiEndpoint, logger).get("email").asText(null);
+ if (email != null)
+ userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByEmail(email));
+ else
+ userOpt = Optional.empty();
+ users.put(login, userOpt);
+ }
+ return userOpt.orElse(null);
+ }
+
+ @Override
+ public String doImport(GitHubProjectImportSource importSource, GitHubProjectImportOption importOption,
+ boolean dryRun, SimpleLogger logger) {
+ Collection<Long> projectIds = new ArrayList<>();
+ Client client = newClient(importSource);
+ try {
+ Map<String, Optional<User>> users = new HashMap<>();
+ GitHubImportResult result = new GitHubImportResult();
+ for (ProjectMapping projectMapping: importOption.getProjectMappings()) {
+ logger.log("Cloning code from repository " + projectMapping.getGitHubRepo() + "...");
+
+ String apiEndpoint = importSource.getApiEndpoint("/repos/" + projectMapping.getGitHubRepo());
+ JsonNode repoNode = get(client, apiEndpoint, logger);
+ Project project = new Project();
+ project.setName(projectMapping.getOneDevProject());
+ project.setDescription(repoNode.get("description").asText(null));
+ project.setIssueManagementEnabled(repoNode.get("has_issues").asBoolean());
+
+ boolean isPrivate = repoNode.get("private").asBoolean();
+ if (!isPrivate && importOption.getPublicRole() != null)
+ project.setDefaultRole(importOption.getPublicRole());
+
+ URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());
+ if (isPrivate)
+ builder.setUserInfo("git", importSource.getAccessToken());
+
+ if (!dryRun) {
+ OneDev.getInstance(ProjectManager.class).clone(project, builder.build().toString());
+ projectIds.add(project.getId());
+ }
+
+ if (projectMapping.isImportIssues()) {
+ Map<Long, Milestone> milestones = new HashMap<>();
+ logger.log("Importing milestones from repository " + projectMapping.getGitHubRepo() + "...");
+ apiEndpoint = importSource.getApiEndpoint("/repos/" + projectMapping.getGitHubRepo() + "/milestones?state=all");
+ for (JsonNode milestoneNode: list(client, apiEndpoint, logger)) {
+ Milestone milestone = new Milestone();
+ milestone.setName(milestoneNode.get("title").asText());
+ milestone.setDescription(milestoneNode.get("description").asText(null));
+ milestone.setProject(project);
+ String dueDateString = milestoneNode.get("due_on").asText(null);
+ if (dueDateString != null)
+ milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());
+ if (milestoneNode.get("state").asText().equals("closed"))
+ milestone.setClosed(true);
+
+ milestones.put(milestoneNode.get("number").asLong(), milestone);
+ project.getMilestones().add(milestone);
+
+ if (!dryRun)
+ OneDev.getInstance(MilestoneManager.class).save(milestone);
+ }
+
+ logger.log("Importing issues from repository " + projectMapping.getGitHubRepo() + "...");
+ GitHubImportResult currentResult = importIssues(importSource, projectMapping.getGitHubRepo(),
+ project, true, importOption, users, dryRun, logger);
+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
+ result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);
+ result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);
+
+ result.issuesImported |= currentResult.issuesImported;
+ }
+ }
+
+ return result.toHtml("Repositories imported successfully");
+ } catch (Exception e) {
+ for (Long projectId: projectIds)
+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ throw new RuntimeException(e);
+ } finally {
+ client.close();
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImport.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectMapping.java
similarity index 95%
rename from server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImport.java
rename to server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectMapping.java
index f480d74647..cb2b4393f0 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImport.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectMapping.java
@@ -8,7 +8,7 @@ import io.onedev.server.util.validation.annotation.ProjectName;
import io.onedev.server.web.editable.annotation.Editable;
@Editable
-public class GitHubImport implements Serializable {
+public class ProjectMapping implements Serializable {
private static final long serialVersionUID = 1L;
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImport.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.java
similarity index 95%
rename from server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImport.java
rename to server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.java
index 1f07e30919..9f302f2399 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImport.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.java
@@ -8,7 +8,7 @@ import io.onedev.server.util.validation.annotation.ProjectName;
import io.onedev.server.web.editable.annotation.Editable;
@Editable
-public class YouTrackImport implements Serializable {
+public class ProjectMapping implements Serializable {
private static final long serialVersionUID = 1L;
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportResult.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportResult.java
new file mode 100644
index 0000000000..872833c197
--- /dev/null
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportResult.java
@@ -0,0 +1,76 @@
+package io.onedev.server.plugin.imports.youtrack;
+
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.unbescape.html.HtmlEscape;
+
+public class YouTrackImportResult {
+
+ Set<String> nonExistentLogins = new LinkedHashSet<>();
+
+ Set<String> unmappedIssueTags = new LinkedHashSet<>();
+
+ Set<String> unmappedIssueFields = new LinkedHashSet<>();
+
+ Set<String> unmappedIssueStates = new LinkedHashSet<>();
+
+ Map<String, String> mismatchedIssueFields = new LinkedHashMap<>();
+
+ Set<String> tooLargeAttachments = new LinkedHashSet<>();
+
+ public String toHtml(String leadingText) {
+ StringBuilder feedback = new StringBuilder(HtmlEscape.escapeHtml5(leadingText));
+
+ boolean hasNotes =
+ !unmappedIssueStates.isEmpty()
+ || !unmappedIssueFields.isEmpty()
+ || !mismatchedIssueFields.isEmpty()
+ || !unmappedIssueTags.isEmpty()
+ || !nonExistentLogins.isEmpty()
+ || !tooLargeAttachments.isEmpty();
+
+ if (hasNotes)
+ feedback.append("<br><br><b>NOTE:</b><ul>");
+
+ if (!unmappedIssueStates.isEmpty()) {
+ feedback.append("<li> Unmapped YouTrack issue states (using OneDev initial state): "
+ + HtmlEscape.escapeHtml5(unmappedIssueStates.toString()));
+ }
+ if (!unmappedIssueFields.isEmpty()) {
+ feedback.append("<li> Unmapped YouTrack issue fields: "
+ + HtmlEscape.escapeHtml5(unmappedIssueFields.toString()));
+ }
+ if (!mismatchedIssueFields.isEmpty()) {
+ feedback.append("<li> YouTrack issue fields mapped to wrong type of OneDev issue field: ");
+ feedback.append("<ul>");
+ for (Map.Entry<String, String> entry: mismatchedIssueFields.entrySet()) {
+ feedback.append("<li>")
+ .append(HtmlEscape.escapeHtml5(entry.getKey()))
+ .append(" : ")
+ .append(HtmlEscape.escapeHtml5(entry.getValue()));
+ }
+ feedback.append("</ul>");
+ }
+ if (!unmappedIssueTags.isEmpty()) {
+ feedback.append("<li> YouTrack issue tags not mapped to OneDev custom field: "
+ + HtmlEscape.escapeHtml5(unmappedIssueTags.toString()));
+ }
+ if (!nonExistentLogins.isEmpty()) {
+ feedback.append("<li> YouTrack logins without email or email can not be mapped to OneDev account: "
+ + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));
+ }
+ if (!tooLargeAttachments.isEmpty()) {
+ feedback.append("<li> Too large attachments: "
+ + HtmlEscape.escapeHtml5(tooLargeAttachments.toString()));
+ }
+
+ if (hasNotes)
+ feedback.append("</ul>");
+
+ return feedback.toString();
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportUtils.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportUtils.java
new file mode 100644
index 0000000000..cf2c9cb68e
--- /dev/null
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportUtils.java
@@ -0,0 +1,882 @@
+package io.onedev.server.plugin.imports.youtrack;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.annotation.Nullable;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Invocation;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Response;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.client.utils.URIBuilder;
+import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
+import org.unbescape.html.HtmlEscape;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import io.onedev.commons.utils.ExplicitException;
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.IssueManager;
+import io.onedev.server.entitymanager.SettingManager;
+import io.onedev.server.entitymanager.UserManager;
+import io.onedev.server.model.Issue;
+import io.onedev.server.model.IssueComment;
+import io.onedev.server.model.IssueField;
+import io.onedev.server.model.Project;
+import io.onedev.server.model.User;
+import io.onedev.server.model.support.LastUpdate;
+import io.onedev.server.model.support.administration.GlobalIssueSetting;
+import io.onedev.server.model.support.inputspec.InputSpec;
+import io.onedev.server.model.support.issue.StateSpec;
+import io.onedev.server.model.support.issue.field.spec.ChoiceField;
+import io.onedev.server.model.support.issue.field.spec.DateField;
+import io.onedev.server.model.support.issue.field.spec.DateTimeField;
+import io.onedev.server.model.support.issue.field.spec.FieldSpec;
+import io.onedev.server.model.support.issue.field.spec.FloatField;
+import io.onedev.server.model.support.issue.field.spec.IntegerField;
+import io.onedev.server.model.support.issue.field.spec.TextField;
+import io.onedev.server.model.support.issue.field.spec.UserChoiceField;
+import io.onedev.server.model.support.issue.field.spec.WorkingPeriodField;
+import io.onedev.server.persistence.dao.Dao;
+import io.onedev.server.security.SecurityUtils;
+import io.onedev.server.util.DateUtils;
+import io.onedev.server.util.JerseyUtils;
+import io.onedev.server.util.Pair;
+import io.onedev.server.util.ReferenceMigrator;
+import io.onedev.server.util.SimpleLogger;
+
+public class YouTrackImportUtils {
+
+ static final String NAME = "YouTrack";
+
+ static final int PER_PAGE = 100;
+
+ static YouTrackIssueImportOption buildImportOption(YouTrackProjectImportSource importSource, @Nullable String youTrackProjectId, SimpleLogger logger) {
+ YouTrackIssueImportOption importOption = new YouTrackIssueImportOption();
+ Client client = newClient(importSource);
+ try {
+ String fields = "customFields(field(name),bundle(values(name)))";
+
+ Set<String> youTrackIssueStates = new LinkedHashSet<>();
+ Set<String> youTrackIssueFields = new LinkedHashSet<>();
+
+ List<JsonNode> projectNodes = new ArrayList<>();
+
+ if (youTrackProjectId != null) {
+ String apiEndpoint = importSource.getApiEndpoint("/admin/projects/" + youTrackProjectId + "?fields=" + fields);
+ projectNodes.add(get(client, apiEndpoint, logger));
+ } else {
+ String apiEndpoint = importSource.getApiEndpoint("/admin/projects?fields=" + fields);
+ for (JsonNode projectNode: list(client, apiEndpoint, logger))
+ projectNodes.add(projectNode);
+ }
+
+ for (JsonNode projectNode: projectNodes) {
+ for (JsonNode customFieldNode: projectNode.get("customFields")) {
+ String fieldName = customFieldNode.get("field").get("name").asText();
+ JsonNode bundleNode = customFieldNode.get("bundle");
+ if (bundleNode != null) {
+ String bundleType = bundleNode.get("$type").asText();
+ if (bundleType.equals("StateBundle")) {
+ for (JsonNode valueNode: bundleNode.get("values"))
+ youTrackIssueStates.add(valueNode.get("name").asText());
+ } else if (bundleType.equals("EnumBundle") || bundleType.equals("OwnedBundle")) {
+ for (JsonNode valueNode: bundleNode.get("values"))
+ youTrackIssueFields.add(fieldName + "::" + valueNode.get("name").asText());
+ } else {
+ youTrackIssueFields.add(fieldName);
+ }
+ } else {
+ youTrackIssueFields.add(fieldName);
+ }
+ }
+ }
+
+ for (String youTrackIssueState: youTrackIssueStates) {
+ IssueStateMapping mapping = new IssueStateMapping();
+ mapping.setYouTrackIssueState(youTrackIssueState);
+ importOption.getIssueStateMappings().add(mapping);
+ }
+
+ for (String youTrackIssueField: youTrackIssueFields) {
+ IssueFieldMapping mapping = new IssueFieldMapping();
+ mapping.setYouTrackIssueField(youTrackIssueField);
+ importOption.getIssueFieldMappings().add(mapping);
+ }
+
+ String apiEndpoint = importSource.getApiEndpoint("/issueTags?fields=name");
+ for (JsonNode tagNode: list(client, apiEndpoint, logger)) {
+ IssueTagMapping mapping = new IssueTagMapping();
+ mapping.setYouTrackIssueTag(tagNode.get("name").asText());
+ importOption.getIssueTagMappings().add(mapping);
+ }
+
+ } finally {
+ client.close();
+ }
+ return importOption;
+ }
+
+ static YouTrackImportResult importIssues(YouTrackProjectImportSource importSource, String youTrackProjectId,
+ Project oneDevProject, boolean useExistingIssueNumbers, YouTrackIssueImportOption importOption,
+ boolean dryRun, SimpleLogger logger) {
+ Client client = newClient(importSource);
+ try {
+ String apiEndpoint = importSource.getApiEndpoint("/admin/projects/" + youTrackProjectId + "?fields=shortName");
+ String youTrackProjectShortName = get(client, apiEndpoint, logger).get("shortName").asText();
+
+ Map<String, String> stateMappings = new HashMap<>();
+ Map<String, Pair<FieldSpec, String>> fieldMappings = new HashMap<>();
+ Map<String, Pair<FieldSpec, String>> tagMappings = new HashMap<>();
+
+ Map<Long, Long> issueNumberMappings = new HashMap<>();
+
+ for (IssueStateMapping mapping: importOption.getIssueStateMappings())
+ stateMappings.put(mapping.getYouTrackIssueState(), mapping.getOneDevIssueState());
+
+ for (IssueFieldMapping mapping: importOption.getIssueFieldMappings()) {
+ String oneDevFieldName;
+ String oneDevFieldValue;
+ if (mapping.getOneDevIssueField().contains("::")) {
+ oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
+ oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
+ } else {
+ oneDevFieldName = mapping.getOneDevIssueField();
+ oneDevFieldValue = null;
+ }
+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
+ if (fieldSpec == null)
+ throw new ExplicitException("No field spec found: " + oneDevFieldName);
+ fieldMappings.put(mapping.getYouTrackIssueField(), new Pair<>(fieldSpec, oneDevFieldValue));
+ }
+ for (IssueTagMapping mapping: importOption.getIssueTagMappings()) {
+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
+ if (fieldSpec == null)
+ throw new ExplicitException("No field spec found: " + oneDevFieldName);
+ tagMappings.put(mapping.getYouTrackIssueTag(), new Pair<>(fieldSpec, oneDevFieldValue));
+ }
+
+ Set<String> nonExistentLogins = new LinkedHashSet<>();
+ Set<String> unmappedIssueTags = new LinkedHashSet<>();
+ Set<String> unmappedIssueFields = new LinkedHashSet<>();
+ Set<String> unmappedIssueStates = new LinkedHashSet<>();
+ Map<String, String> mismatchedIssueFields = new LinkedHashMap<>();
+ Set<String> tooLargeAttachments = new LinkedHashSet<>();
+
+ AtomicInteger numOfImportedIssues = new AtomicInteger(0);
+
+ List<Issue> issues = new ArrayList<>();
+
+ String fields = ""
+ + "idReadable,"
+ + "numberInProject,"
+ + "summary,"
+ + "description,"
+ + "created,"
+ + "comments(created,text,author(login,name,email),attachments(name,url,size)),"
+ + "attachments(name,url,comment,size),"
+ + "reporter(login,name,email),"
+ + "tags(name),"
+ + "customFields(name,value(name,login,email,presentation,text),projectCustomField(field(fieldType(id)))),"
+ + "links(direction,linkType(sourceToTarget,targetToSource),issues(numberInProject))";
+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {
+
+ @Nullable
+ private String processAttachments(String issueUUID, String readableIssueId, @Nullable String markdown,
+ List<JsonNode> attachmentNodes, Set<String> tooLargeAttachments) {
+ if (markdown == null)
+ markdown = "";
+
+ Map<String, String> unreferencedAttachments = new LinkedHashMap<>();
+
+ for (JsonNode attachmentNode: attachmentNodes) {
+ String attachmentName = attachmentNode.get("name").asText(null);
+ String attachmentUrl = attachmentNode.get("url").asText(null);
+ long attachmentSize = attachmentNode.get("size").asLong(0);
+ if (attachmentSize != 0 && attachmentName != null && attachmentUrl != null) {
+ if (attachmentSize > Project.MAX_ATTACHMENT_SIZE) {
+ tooLargeAttachments.add(readableIssueId + ":" + attachmentName);
+ } else {
+ if (!attachmentUrl.startsWith("/api"))
+ throw new ExplicitException("Unexpected attachment url: " + attachmentUrl);
+ attachmentUrl = attachmentUrl.substring("/api".length());
+
+ String endpoint = importSource.getApiEndpoint(attachmentUrl);
+ WebTarget target = client.target(endpoint);
+ Invocation.Builder builder = target.request();
+ try (Response response = builder.get()) {
+ String errorMessage = JerseyUtils.checkStatus(endpoint, response);
+ if (errorMessage != null) {
+ throw new ExplicitException(String.format(
+ "Error downloading attachment (url: %s, error message: %s)",
+ endpoint, errorMessage));
+ }
+ try (InputStream is = response.readEntity(InputStream.class)) {
+ String oneDevAttachmentName = oneDevProject.saveAttachment(issueUUID, attachmentName, is);
+ String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);
+ if (markdown.contains("(" + attachmentName + ")")) {
+ markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");
+ } else {
+ unreferencedAttachments.put(attachmentName, oneDevAttachmentUrl);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+ }
+
+ if (!unreferencedAttachments.isEmpty()) {
+ markdown += "\n\n";
+ for (Map.Entry<String, String> entry: unreferencedAttachments.entrySet())
+ markdown += "[" + entry.getKey() + "](" + entry.getValue() + ") ";
+ }
+
+ if (markdown.length() == 0)
+ markdown = null;
+
+ return markdown;
+ }
+
+ private String joinAsMultilineHtml(List<String> values) {
+ List<String> escapedValues = new ArrayList<>();
+ for (String value: values)
+ escapedValues.add(HtmlEscape.escapeHtml5(value));
+ return StringUtils.join(escapedValues, "<br>");
+ }
+
+ @Nullable
+ private String getEmail(JsonNode userNode) {
+ JsonNode emailNode = userNode.get("email");
+ if (emailNode != null)
+ return emailNode.asText(null);
+ else
+ return null;
+ }
+
+ @Override
+ public void consume(List<JsonNode> pageData) throws InterruptedException {
+ for (JsonNode issueNode: pageData) {
+ if (Thread.interrupted())
+ throw new InterruptedException();
+
+ Issue issue = new Issue();
+ String readableId = issueNode.get("idReadable").asText();
+ Long oldNumber = issueNode.get("numberInProject").asLong();
+ Long newNumber;
+ if (dryRun || useExistingIssueNumbers)
+ newNumber = oldNumber;
+ else
+ newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);
+ issue.setNumber(newNumber);
+ issueNumberMappings.put(oldNumber, newNumber);
+ issue.setTitle(issueNode.get("summary").asText());
+ issue.setDescription(issueNode.get("description").asText(null));
+ issue.setSubmitDate(new Date(issueNode.get("created").asLong(System.currentTimeMillis())));
+ issue.setProject(oneDevProject);
+ issue.setNumberScope(oneDevProject);
+
+ if (!dryRun) {
+ List<JsonNode> attachmentNodes = new ArrayList<>();
+ for (JsonNode attachmentNode: issueNode.get("attachments")) {
+ if (!attachmentNode.hasNonNull("comment"))
+ attachmentNodes.add(attachmentNode);
+ }
+ issue.setDescription(processAttachments(issue.getUUID(), readableId,
+ issue.getDescription(), attachmentNodes, tooLargeAttachments));
+ }
+
+ Map<String, String> extraIssueInfo = new LinkedHashMap<>();
+
+ if (issueNode.hasNonNull("reporter")) {
+ JsonNode reporterNode = issueNode.get("reporter");
+ String email = getEmail(reporterNode);
+ String fullName = reporterNode.get("name").asText();
+ String login = reporterNode.get("login").asText();
+ if (email != null) {
+ User user = OneDev.getInstance(UserManager.class).findByEmail(email);
+ if (user != null) {
+ issue.setSubmitter(user);
+ } else {
+ issue.setSubmitterName(fullName);
+ nonExistentLogins.add(login);
+ }
+ } else {
+ issue.setSubmitterName(fullName);
+ nonExistentLogins.add(login);
+ }
+ } else {
+ issue.setSubmitter(SecurityUtils.getUser());
+ }
+
+ LastUpdate lastUpdate = new LastUpdate();
+ lastUpdate.setActivity("Opened");
+ lastUpdate.setDate(issue.getSubmitDate());
+ lastUpdate.setUser(issue.getSubmitter());
+ lastUpdate.setUserName(issue.getSubmitterName());
+ issue.setLastUpdate(lastUpdate);
+
+ StateSpec initialState = getIssueSetting().getInitialStateSpec();
+ for (JsonNode customFieldNode: issueNode.get("customFields")) {
+ String fieldName = customFieldNode.get("name").asText();
+ String fieldType = customFieldNode.get("$type").asText();
+ switch (fieldType) {
+ case "StateIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ String state = customFieldNode.get("value").get("name").asText();
+ String mappedState = stateMappings.get(state);
+ if (mappedState != null)
+ issue.setState(mappedState);
+ else
+ unmappedIssueStates.add(state);
+ }
+ break;
+ case "SingleBuildIssueCustomField":
+ case "SingleGroupIssueCustomField":
+ case "SingleVersionIssueCustomField":
+ String fieldValue = customFieldNode.get("value").asText(null);
+ if (fieldValue != null) {
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof TextField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.TEXT);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ }
+ break;
+ case "MultiBuildIssueCustomField":
+ case "MultiGroupIssueCustomField":
+ case "MultiVersionIssueCustomField":
+ List<String> fieldValues = new ArrayList<>();
+ for (JsonNode valueNode: customFieldNode.get("value"))
+ fieldValues.add(valueNode.get("name").asText());
+ if (!fieldValues.isEmpty()) {
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));
+ } else if (!(mapped.getFirst() instanceof TextField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");
+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.TEXT);
+ issueField.setValue(StringUtils.join(fieldValues, ", "));
+ issue.getFields().add(issueField);
+ }
+ }
+ break;
+ case "SingleEnumIssueCustomField":
+ case "SingleOwnedIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ fieldValue = customFieldNode.get("value").get("name").asText();
+ String fieldNameWithValue = fieldName + "::" + fieldValue;
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldNameWithValue);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof ChoiceField) || mapped.getFirst().isAllowMultiple()) {
+ mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a single-valued enum field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.ENUMERATION);
+ issueField.setValue(mapped.getSecond());
+ issue.getFields().add(issueField);
+ }
+ }
+ break;
+ case "MultiEnumIssueCustomField":
+ case "MultiOwnedIssueCustomField":
+ fieldValues = new ArrayList<>();
+ for (JsonNode valueNode: customFieldNode.get("value"))
+ fieldValues.add(valueNode.get("name").asText());
+
+ boolean hasErrors = false;
+ List<IssueField> issueFields = new ArrayList<>();
+ for (String each: fieldValues) {
+ String fieldNameWithValue = fieldName + "::" + each;
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldNameWithValue);
+ hasErrors = true;
+ } else if (!(mapped.getFirst() instanceof ChoiceField) || !mapped.getFirst().isAllowMultiple()) {
+ mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a multi-valued enum field");
+ hasErrors = true;
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.ENUMERATION);
+ issueField.setValue(mapped.getSecond());
+ issueFields.add(issueField);
+ }
+ }
+ if (hasErrors)
+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));
+ else
+ issue.getFields().addAll(issueFields);
+ break;
+ case "SingleUserIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ JsonNode valueNode = customFieldNode.get("value");
+
+ String login = valueNode.get("login").asText();
+ String fullName = valueNode.get("name").asText();
+ String email = getEmail(valueNode);
+
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fullName);
+ } else if (!(mapped.getFirst() instanceof UserChoiceField) || mapped.getFirst().isAllowMultiple()) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a single-valued user field");
+ extraIssueInfo.put(fieldName, fullName);
+ } else {
+ if (email != null) {
+ User user = OneDev.getInstance(UserManager.class).findByEmail(email);
+ if (user != null) {
+ fieldValue = user.getName();
+ } else {
+ fieldValue = fullName;
+ nonExistentLogins.add(login);
+ }
+ } else {
+ fieldValue = fullName;
+ nonExistentLogins.add(login);
+ }
+
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.USER);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ }
+ break;
+ case "MultiUserIssueCustomField":
+ List<String> fullNames = new ArrayList<>();
+ for (JsonNode valueNode: customFieldNode.get("value"))
+ fullNames.add(valueNode.get("name").asText());
+ if (!fullNames.isEmpty()) {
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));
+ } else if (!(mapped.getFirst() instanceof UserChoiceField) || !mapped.getFirst().isAllowMultiple()) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-valued user field");
+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));
+ } else {
+ for (JsonNode valueNode: customFieldNode.get("value")) {
+ String login = valueNode.get("login").asText();
+ String fullName = valueNode.get("name").asText();
+ String email = getEmail(valueNode);
+
+ if (email != null) {
+ User user = OneDev.getInstance(UserManager.class).findByEmail(email);
+ if (user != null) {
+ fieldValue = user.getName();
+ } else {
+ fieldValue = fullName;
+ nonExistentLogins.add(login);
+ }
+ } else {
+ fieldValue = fullName;
+ nonExistentLogins.add(login);
+ }
+
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.USER);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ }
+ }
+ break;
+ case "DateIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ Date date = new Date(customFieldNode.get("value").asLong());
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, DateUtils.formatDate(date));
+ } else if (!(mapped.getFirst() instanceof DateField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a date field");
+ extraIssueInfo.put(fieldName, DateUtils.formatDate(date));
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.DATE);
+ issueField.setValue(DateUtils.formatDate(date));
+ issue.getFields().add(issueField);
+ }
+ }
+ break;
+ case "SimpleIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ JsonNode projectCustomField = customFieldNode.get("projectCustomField");
+ JsonNode fieldNode = projectCustomField.get("field");
+ if (fieldNode != null) {
+ switch (fieldNode.get("fieldType").get("id").asText()) {
+ case "date":
+ Date date = new Date(customFieldNode.get("value").asLong());
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));
+ } else if (!(mapped.getFirst() instanceof DateTimeField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a datetime field");
+ extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.DATE_TIME);
+ issueField.setValue(DateUtils.formatDateTime(date));
+ issue.getFields().add(issueField);
+ }
+ break;
+ case "float":
+ fieldValue = customFieldNode.get("value").asText();
+ mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof FloatField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a float field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.FLOAT);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ break;
+ case "integer":
+ fieldValue = customFieldNode.get("value").asText();
+ mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof IntegerField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to an integer field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.INTEGER);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ break;
+ case "string":
+ fieldValue = customFieldNode.get("value").asText();
+ mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof TextField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.TEXT);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ break;
+ }
+ }
+ }
+ break;
+ case "TextIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ fieldValue = customFieldNode.get("value").get("text").asText();
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof TextField)
+ || !((TextField)mapped.getFirst()).isMultiline()) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-line text field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.TEXT);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ }
+ break;
+ case "PeriodIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ fieldValue = customFieldNode.get("value").get("presentation").asText();
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof WorkingPeriodField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a working period field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.WORKING_PERIOD);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ }
+ break;
+ }
+ }
+ if (issue.getState() == null)
+ issue.setState(initialState.getName());
+
+ for (JsonNode linkNode: issueNode.get("links")) {
+ List<String> linkedIssueLinks = new ArrayList<>();
+ for (JsonNode linkedIssueNode: linkNode.get("issues"))
+ linkedIssueLinks.add(youTrackProjectShortName + "-" + linkedIssueNode.get("numberInProject").asLong());
+
+ if (!linkedIssueLinks.isEmpty() && linkNode.hasNonNull("linkType")) {
+ JsonNode linkTypeNode = linkNode.get("linkType");
+ switch (linkNode.get("direction").asText()) {
+ case "BOTH":
+ case "OUTWARD":
+ String linkName = linkTypeNode.get("sourceToTarget").asText(null);
+ if (linkName != null)
+ extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));
+ break;
+ case "INWARD":
+ linkName = linkTypeNode.get("targetToSource").asText(null);
+ if (linkName != null)
+ extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));
+ break;
+ }
+ }
+ }
+
+ List<String> currentUnmappedTags = new ArrayList<>();
+ for (JsonNode tagNode: issueNode.get("tags")) {
+ String tagName = tagNode.get("name").asText();
+ Pair<FieldSpec, String> mapped = tagMappings.get(tagName);
+ if (mapped != null) {
+ IssueField tagField = new IssueField();
+ tagField.setIssue(issue);
+ tagField.setName(mapped.getFirst().getName());
+ tagField.setType(InputSpec.ENUMERATION);
+ tagField.setValue(mapped.getSecond());
+ tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
+ issue.getFields().add(tagField);
+ } else {
+ currentUnmappedTags.add(HtmlEscape.escapeHtml5(tagName));
+ unmappedIssueTags.add(HtmlEscape.escapeHtml5(tagName));
+ }
+ }
+ if (!currentUnmappedTags.isEmpty())
+ extraIssueInfo.put("Tags", joinAsMultilineHtml(currentUnmappedTags));
+
+ for (JsonNode commentNode: issueNode.get("comments")) {
+ String commentContent = commentNode.get("text").asText(null);
+ if (commentContent != null || !commentNode.get("attachments").isEmpty()) {
+ IssueComment comment = new IssueComment();
+ comment.setIssue(issue);
+ if (!dryRun) {
+ List<JsonNode> attachmentNodes = new ArrayList<>();
+ for (JsonNode attachmentNode: commentNode.get("attachments"))
+ attachmentNodes.add(attachmentNode);
+ comment.setContent(processAttachments(issue.getUUID(), readableId,
+ commentContent, attachmentNodes, tooLargeAttachments));
+ }
+ comment.setDate(new Date(commentNode.get("created").asLong(System.currentTimeMillis())));
+ if (commentNode.hasNonNull("author")) {
+ JsonNode authorNode = commentNode.get("author");
+ String email = getEmail(authorNode);
+ String fullName = authorNode.get("name").asText();
+ String login = authorNode.get("login").asText();
+ if (email != null) {
+ User user = OneDev.getInstance(UserManager.class).findByEmail(email);
+ if (user != null) {
+ comment.setUser(user);
+ } else {
+ comment.setUserName(fullName);
+ nonExistentLogins.add(login);
+ }
+ } else {
+ comment.setUserName(fullName);
+ nonExistentLogins.add(login);
+ }
+ } else {
+ issue.setSubmitter(SecurityUtils.getUser());
+ }
+ issue.getComments().add(comment);
+ }
+ }
+
+ issue.setCommentCount(issue.getComments().size());
+
+ if (!extraIssueInfo.isEmpty()) {
+ StringBuilder builder = new StringBuilder("|");
+ for (String key: extraIssueInfo.keySet())
+ builder.append(key).append("|");
+ builder.append("\n|");
+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
+ builder.append("\n|");
+ for (String value: extraIssueInfo.values())
+ builder.append(value).append("|");
+
+ if (issue.getDescription() != null)
+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
+ else
+ issue.setDescription(builder.toString());
+ }
+ issues.add(issue);
+ }
+ logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");
+ }
+
+ };
+
+ apiEndpoint = importSource.getApiEndpoint("/admin/projects/" + youTrackProjectId + "/issues?fields=" + fields);
+ list(client, apiEndpoint, pageDataConsumer, logger);
+
+ if (!dryRun) {
+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);
+ for (Issue issue: issues) {
+ if (issue.getDescription() != null)
+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), youTrackProjectShortName + "-"));
+
+ OneDev.getInstance(IssueManager.class).save(issue);
+ for (IssueField field: issue.getFields())
+ OneDev.getInstance(Dao.class).persist(field);
+ for (IssueComment comment: issue.getComments()) {
+ comment.setContent(migrator.migratePrefixed(comment.getContent(), youTrackProjectShortName + "-"));
+ OneDev.getInstance(Dao.class).persist(comment);
+ }
+ }
+ }
+
+ YouTrackImportResult result = new YouTrackImportResult();
+ result.mismatchedIssueFields.putAll(mismatchedIssueFields);
+ result.nonExistentLogins.addAll(nonExistentLogins);
+ result.tooLargeAttachments.addAll(tooLargeAttachments);
+ result.unmappedIssueFields.addAll(unmappedIssueFields);
+ result.unmappedIssueStates.addAll(unmappedIssueStates);
+ result.unmappedIssueTags.addAll(unmappedIssueTags);
+
+ return result;
+ } finally {
+ client.close();
+ }
+ }
+
+ static GlobalIssueSetting getIssueSetting() {
+ return OneDev.getInstance(SettingManager.class).getIssueSetting();
+ }
+
+ static List<JsonNode> list(Client client, String apiEndpoint, SimpleLogger logger) {
+ List<JsonNode> result = new ArrayList<>();
+ list(client, apiEndpoint, new PageDataConsumer() {
+
+ @Override
+ public void consume(List<JsonNode> pageData) {
+ result.addAll(pageData);
+ }
+
+ }, logger);
+ return result;
+ }
+
+ static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer, SimpleLogger logger) {
+ URI uri;
+ try {
+ uri = new URIBuilder(apiEndpoint)
+ .addParameter("$top", String.valueOf(PER_PAGE)).build();
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+
+ int page = 0;
+ while (true) {
+ try {
+ URIBuilder builder = new URIBuilder(uri);
+ builder.addParameter("$skip", String.valueOf(page*PER_PAGE));
+ List<JsonNode> pageData = new ArrayList<>();
+ for (JsonNode each: get(client, builder.build().toString(), logger))
+ pageData.add(each);
+ pageDataConsumer.consume(pageData);
+ if (pageData.size() < PER_PAGE)
+ break;
+ page++;
+ } catch (URISyntaxException|InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static JsonNode get(Client client, String apiEndpoint, SimpleLogger logger) {
+ WebTarget target = client.target(apiEndpoint);
+ Invocation.Builder builder = target.request();
+ try (Response response = builder.get()) {
+ String errorMessage = JerseyUtils.checkStatus(apiEndpoint, response);
+ if (errorMessage != null)
+ throw new ExplicitException(errorMessage);
+ else
+ return response.readEntity(JsonNode.class);
+ }
+ }
+
+ static Client newClient(YouTrackProjectImportSource importSource) {
+ Client client = ClientBuilder.newClient();
+ client.register(HttpAuthenticationFeature.basic(importSource.getUserName(), importSource.getPassword()));
+ return client;
+ }
+
+ private static interface PageDataConsumer {
+
+ void consume(List<JsonNode> pageData) throws InterruptedException;
+
+ }
+
+}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.java
deleted file mode 100644
index 9f3237b82d..0000000000
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.java
+++ /dev/null
@@ -1,938 +0,0 @@
-package io.onedev.server.plugin.imports.youtrack;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import javax.annotation.Nullable;
-import javax.ws.rs.client.Client;
-import javax.ws.rs.client.ClientBuilder;
-import javax.ws.rs.client.Invocation;
-import javax.ws.rs.client.WebTarget;
-import javax.ws.rs.core.Response;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
-import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
-import org.unbescape.html.HtmlEscape;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-import io.onedev.commons.utils.ExceptionUtils;
-import io.onedev.commons.utils.ExplicitException;
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.IssueManager;
-import io.onedev.server.entitymanager.ProjectManager;
-import io.onedev.server.entitymanager.SettingManager;
-import io.onedev.server.entitymanager.UserManager;
-import io.onedev.server.model.Issue;
-import io.onedev.server.model.IssueComment;
-import io.onedev.server.model.IssueField;
-import io.onedev.server.model.Project;
-import io.onedev.server.model.User;
-import io.onedev.server.model.support.LastUpdate;
-import io.onedev.server.model.support.administration.GlobalIssueSetting;
-import io.onedev.server.model.support.inputspec.InputSpec;
-import io.onedev.server.model.support.issue.StateSpec;
-import io.onedev.server.model.support.issue.field.spec.ChoiceField;
-import io.onedev.server.model.support.issue.field.spec.DateField;
-import io.onedev.server.model.support.issue.field.spec.DateTimeField;
-import io.onedev.server.model.support.issue.field.spec.FieldSpec;
-import io.onedev.server.model.support.issue.field.spec.FloatField;
-import io.onedev.server.model.support.issue.field.spec.IntegerField;
-import io.onedev.server.model.support.issue.field.spec.TextField;
-import io.onedev.server.model.support.issue.field.spec.UserChoiceField;
-import io.onedev.server.model.support.issue.field.spec.WorkingPeriodField;
-import io.onedev.server.persistence.dao.Dao;
-import io.onedev.server.security.SecurityUtils;
-import io.onedev.server.storage.StorageManager;
-import io.onedev.server.util.DateUtils;
-import io.onedev.server.util.JerseyUtils;
-import io.onedev.server.util.Pair;
-import io.onedev.server.util.SimpleLogger;
-import io.onedev.server.web.page.project.imports.ProjectImporter;
-
-public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouTrackImportOption> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String NAME = "YouTrack";
-
- private static final int PER_PAGE = 100;
-
- @Override
- public String getName() {
- return NAME;
- }
-
- @Override
- public YouTrackImportOption getImportOption(YouTrackImportSource importSource, SimpleLogger logger) {
- YouTrackImportOption importOption = new YouTrackImportOption();
- if (importSource.isPrepopulateImportOptions()) {
- Client client = newClient(importSource);
- try {
- String fields = "name,customFields(field(name),bundle(values(name)))";
-
- Set<String> youTrackIssueStates = new LinkedHashSet<>();
- Set<String> youTrackIssueFields = new LinkedHashSet<>();
-
- for (JsonNode projectNode: list(client, importSource, "/admin/projects?fields=" + fields, logger)) {
- YouTrackImport aImport = new YouTrackImport();
- aImport.setYouTrackProject(projectNode.get("name").asText());
- aImport.setOneDevProject(aImport.getYouTrackProject().replace(' ', '-'));
- importOption.getImports().add(aImport);
-
- for (JsonNode customFieldNode: projectNode.get("customFields")) {
- String fieldName = customFieldNode.get("field").get("name").asText();
- JsonNode bundleNode = customFieldNode.get("bundle");
- if (bundleNode != null) {
- String bundleType = bundleNode.get("$type").asText();
- if (bundleType.equals("StateBundle")) {
- for (JsonNode valueNode: bundleNode.get("values"))
- youTrackIssueStates.add(valueNode.get("name").asText());
- } else if (bundleType.equals("EnumBundle") || bundleType.equals("OwnedBundle")) {
- for (JsonNode valueNode: bundleNode.get("values"))
- youTrackIssueFields.add(fieldName + "::" + valueNode.get("name").asText());
- } else {
- youTrackIssueFields.add(fieldName);
- }
- } else {
- youTrackIssueFields.add(fieldName);
- }
- }
- }
-
- for (String youTrackIssueState: youTrackIssueStates) {
- IssueStateMapping mapping = new IssueStateMapping();
- mapping.setYouTrackIssueState(youTrackIssueState);
- importOption.getIssueStateMappings().add(mapping);
- }
-
- for (String youTrackIssueField: youTrackIssueFields) {
- IssueFieldMapping mapping = new IssueFieldMapping();
- mapping.setYouTrackIssueField(youTrackIssueField);
- importOption.getIssueFieldMappings().add(mapping);
- }
-
- for (JsonNode tagNode: list(client, importSource, "/issueTags?fields=name", logger)) {
- IssueTagMapping mapping = new IssueTagMapping();
- mapping.setYouTrackIssueTag(tagNode.get("name").asText());
- importOption.getIssueTagMappings().add(mapping);
- }
-
- } finally {
- client.close();
- }
- }
- return importOption;
- }
-
- @Override
- public String doImport(YouTrackImportSource importSource, YouTrackImportOption importOption, boolean dryRun,
- SimpleLogger logger) {
- Collection<Long> projectIds = new ArrayList<>();
- Client client = newClient(importSource);
- try {
- Map<String, String> stateMappings = new HashMap<>();
- Map<String, Pair<FieldSpec, String>> fieldMappings = new HashMap<>();
- Map<String, Pair<FieldSpec, String>> tagMappings = new HashMap<>();
-
- for (IssueStateMapping mapping: importOption.getIssueStateMappings())
- stateMappings.put(mapping.getYouTrackIssueState(), mapping.getOneDevIssueState());
-
- for (IssueFieldMapping mapping: importOption.getIssueFieldMappings()) {
- String oneDevFieldName;
- String oneDevFieldValue;
- if (mapping.getOneDevIssueField().contains("::")) {
- oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
- oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
- } else {
- oneDevFieldName = mapping.getOneDevIssueField();
- oneDevFieldValue = null;
- }
- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
- if (fieldSpec == null)
- throw new ExplicitException("No field spec found: " + oneDevFieldName);
- fieldMappings.put(mapping.getYouTrackIssueField(), new Pair<>(fieldSpec, oneDevFieldValue));
- }
- for (IssueTagMapping mapping: importOption.getIssueTagMappings()) {
- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
- if (fieldSpec == null)
- throw new ExplicitException("No field spec found: " + oneDevFieldName);
- tagMappings.put(mapping.getYouTrackIssueTag(), new Pair<>(fieldSpec, oneDevFieldValue));
- }
-
- Map<String, String> youTrackProjectIds = new HashMap<>();
- Map<String, String> projectDescriptions = new HashMap<>();
- for (JsonNode projectNode: list(client, importSource, "/admin/projects?fields=id,name,description", logger)) {
- String projectName = projectNode.get("name").asText();
- youTrackProjectIds.put(projectName, projectNode.get("id").asText());
- projectDescriptions.put(projectName, projectNode.get("description").asText(null));
- }
-
- Set<String> nonExistentLogins = new LinkedHashSet<>();
- Set<String> unmappedIssueTags = new LinkedHashSet<>();
- Set<String> unmappedIssueFields = new LinkedHashSet<>();
- Set<String> unmappedIssueStates = new LinkedHashSet<>();
- Map<String, String> mismatchedIssueFields = new LinkedHashMap<>();
- Set<String> tooLargeAttachments = new LinkedHashSet<>();
-
- for (YouTrackImport aImport: importOption.getImports()) {
- logger.log("Importing project '" + aImport.getYouTrackProject() + "'...");
-
- String youTrackProjectId = youTrackProjectIds.get(aImport.getYouTrackProject());
- if (youTrackProjectId == null)
- throw new ExplicitException("Unable to find YouTrack project: " + aImport.getYouTrackProject());
-
- Project project = new Project();
- project.setName(aImport.getOneDevProject());
- project.setDescription(projectDescriptions.get(aImport.getYouTrackProject()));
- project.setIssueManagementEnabled(true);
-
- if (!dryRun) {
- OneDev.getInstance(ProjectManager.class).create(project);
- projectIds.add(project.getId());
- }
-
- AtomicInteger numOfImportedIssues = new AtomicInteger(0);
-
- String fields = ""
- + "idReadable,"
- + "numberInProject,"
- + "summary,"
- + "description,"
- + "created,"
- + "comments(created,text,author(login,name,email),attachments(name,url,size)),"
- + "attachments(name,url,comment,size),"
- + "reporter(login,name,email),"
- + "tags(name),"
- + "customFields(name,value(name,login,email,presentation,text),projectCustomField(field(fieldType(id)))),"
- + "links(direction,linkType(sourceToTarget,targetToSource),issues(numberInProject))";
- PageDataConsumer pageDataConsumer = new PageDataConsumer() {
-
- @Nullable
- private String processAttachments(String issueUUID, String readableIssueId, @Nullable String markdown,
- List<JsonNode> attachmentNodes, Set<String> tooLargeAttachments) {
- if (markdown == null)
- markdown = "";
-
- Map<String, String> unreferencedAttachments = new LinkedHashMap<>();
-
- for (JsonNode attachmentNode: attachmentNodes) {
- String attachmentName = attachmentNode.get("name").asText(null);
- String attachmentUrl = attachmentNode.get("url").asText(null);
- long attachmentSize = attachmentNode.get("size").asLong(0);
- if (attachmentSize != 0 && attachmentName != null && attachmentUrl != null) {
- if (attachmentSize > Project.MAX_ATTACHMENT_SIZE) {
- tooLargeAttachments.add(readableIssueId + ":" + attachmentName);
- } else {
- if (!attachmentUrl.startsWith("/api"))
- throw new ExplicitException("Unexpected attachment url: " + attachmentUrl);
- attachmentUrl = attachmentUrl.substring("/api".length());
-
- String endpoint = importSource.getApiEndpoint(attachmentUrl);
- WebTarget target = client.target(endpoint);
- Invocation.Builder builder = target.request();
- try (Response response = builder.get()) {
- String errorMessage = JerseyUtils.checkStatus(response);
- if (errorMessage != null) {
- throw new ExplicitException(String.format(
- "Error downloading attachment (url: %s, error message: %s)",
- endpoint, errorMessage));
- }
- try (InputStream is = response.readEntity(InputStream.class)) {
- String oneDevAttachmentName = project.saveAttachment(issueUUID, attachmentName, is);
- String oneDevAttachmentUrl = project.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);
- if (markdown.contains("(" + attachmentName + ")")) {
- markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");
- } else {
- unreferencedAttachments.put(attachmentName, oneDevAttachmentUrl);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
- }
-
- if (!unreferencedAttachments.isEmpty()) {
- markdown += "\n\n";
- for (Map.Entry<String, String> entry: unreferencedAttachments.entrySet())
- markdown += "[" + entry.getKey() + "](" + entry.getValue() + ") ";
- }
-
- if (markdown.length() == 0)
- markdown = null;
-
- return markdown;
- }
-
- private String joinAsMultilineHtml(List<String> values) {
- List<String> escapedValues = new ArrayList<>();
- for (String value: values)
- escapedValues.add(HtmlEscape.escapeHtml5(value));
- return StringUtils.join(escapedValues, "<br>");
- }
-
- @Nullable
- private String getEmail(JsonNode userNode) {
- JsonNode emailNode = userNode.get("email");
- if (emailNode != null)
- return emailNode.asText(null);
- else
- return null;
- }
-
- @Override
- public void consume(List<JsonNode> pageData) throws InterruptedException {
- for (JsonNode issueNode: pageData) {
- if (Thread.interrupted())
- throw new InterruptedException();
-
- Issue issue = new Issue();
- String readableId = issueNode.get("idReadable").asText();
- issue.setNumber(issueNode.get("numberInProject").asLong());
- issue.setTitle(issueNode.get("summary").asText());
- issue.setDescription(issueNode.get("description").asText(null));
- issue.setSubmitDate(new Date(issueNode.get("created").asLong(System.currentTimeMillis())));
- issue.setProject(project);
- issue.setNumberScope(project);
-
- if (!dryRun) {
- List<JsonNode> attachmentNodes = new ArrayList<>();
- for (JsonNode attachmentNode: issueNode.get("attachments")) {
- if (!attachmentNode.hasNonNull("comment"))
- attachmentNodes.add(attachmentNode);
- }
- issue.setDescription(processAttachments(issue.getUUID(), readableId,
- issue.getDescription(), attachmentNodes, tooLargeAttachments));
- }
-
- Map<String, String> extraIssueInfo = new LinkedHashMap<>();
-
- if (issueNode.hasNonNull("reporter")) {
- JsonNode reporterNode = issueNode.get("reporter");
- String email = getEmail(reporterNode);
- String fullName = reporterNode.get("name").asText();
- String login = reporterNode.get("login").asText();
- if (email != null) {
- User user = OneDev.getInstance(UserManager.class).findByEmail(email);
- if (user != null) {
- issue.setSubmitter(user);
- } else {
- issue.setSubmitterName(fullName);
- nonExistentLogins.add(login);
- }
- } else {
- issue.setSubmitterName(fullName);
- nonExistentLogins.add(login);
- }
- } else {
- issue.setSubmitter(SecurityUtils.getUser());
- }
-
- LastUpdate lastUpdate = new LastUpdate();
- lastUpdate.setActivity("Opened");
- lastUpdate.setDate(issue.getSubmitDate());
- lastUpdate.setUser(issue.getSubmitter());
- lastUpdate.setUserName(issue.getSubmitterName());
- issue.setLastUpdate(lastUpdate);
-
- StateSpec initialState = getIssueSetting().getInitialStateSpec();
- for (JsonNode customFieldNode: issueNode.get("customFields")) {
- String fieldName = customFieldNode.get("name").asText();
- String fieldType = customFieldNode.get("$type").asText();
- switch (fieldType) {
- case "StateIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- String state = customFieldNode.get("value").get("name").asText();
- String mappedState = stateMappings.get(state);
- if (mappedState != null)
- issue.setState(mappedState);
- else
- unmappedIssueStates.add(state);
- }
- break;
- case "SingleBuildIssueCustomField":
- case "SingleGroupIssueCustomField":
- case "SingleVersionIssueCustomField":
- String fieldValue = customFieldNode.get("value").asText(null);
- if (fieldValue != null) {
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof TextField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.TEXT);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- }
- break;
- case "MultiBuildIssueCustomField":
- case "MultiGroupIssueCustomField":
- case "MultiVersionIssueCustomField":
- List<String> fieldValues = new ArrayList<>();
- for (JsonNode valueNode: customFieldNode.get("value"))
- fieldValues.add(valueNode.get("name").asText());
- if (!fieldValues.isEmpty()) {
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));
- } else if (!(mapped.getFirst() instanceof TextField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");
- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.TEXT);
- issueField.setValue(StringUtils.join(fieldValues, ", "));
- issue.getFields().add(issueField);
- }
- }
- break;
- case "SingleEnumIssueCustomField":
- case "SingleOwnedIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- fieldValue = customFieldNode.get("value").get("name").asText();
- String fieldNameWithValue = fieldName + "::" + fieldValue;
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);
- if (mapped == null) {
- unmappedIssueFields.add(fieldNameWithValue);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof ChoiceField) || mapped.getFirst().isAllowMultiple()) {
- mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a single-valued enum field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.ENUMERATION);
- issueField.setValue(mapped.getSecond());
- issue.getFields().add(issueField);
- }
- }
- break;
- case "MultiEnumIssueCustomField":
- case "MultiOwnedIssueCustomField":
- fieldValues = new ArrayList<>();
- for (JsonNode valueNode: customFieldNode.get("value"))
- fieldValues.add(valueNode.get("name").asText());
-
- boolean hasErrors = false;
- List<IssueField> issueFields = new ArrayList<>();
- for (String each: fieldValues) {
- String fieldNameWithValue = fieldName + "::" + each;
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);
- if (mapped == null) {
- unmappedIssueFields.add(fieldNameWithValue);
- hasErrors = true;
- } else if (!(mapped.getFirst() instanceof ChoiceField) || !mapped.getFirst().isAllowMultiple()) {
- mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a multi-valued enum field");
- hasErrors = true;
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.ENUMERATION);
- issueField.setValue(mapped.getSecond());
- issueFields.add(issueField);
- }
- }
- if (hasErrors)
- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));
- else
- issue.getFields().addAll(issueFields);
- break;
- case "SingleUserIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- JsonNode valueNode = customFieldNode.get("value");
-
- String login = valueNode.get("login").asText();
- String fullName = valueNode.get("name").asText();
- String email = getEmail(valueNode);
-
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fullName);
- } else if (!(mapped.getFirst() instanceof UserChoiceField) || mapped.getFirst().isAllowMultiple()) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a single-valued user field");
- extraIssueInfo.put(fieldName, fullName);
- } else {
- if (email != null) {
- User user = OneDev.getInstance(UserManager.class).findByEmail(email);
- if (user != null) {
- fieldValue = user.getName();
- } else {
- fieldValue = fullName;
- nonExistentLogins.add(login);
- }
- } else {
- fieldValue = fullName;
- nonExistentLogins.add(login);
- }
-
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.USER);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- }
- break;
- case "MultiUserIssueCustomField":
- List<String> fullNames = new ArrayList<>();
- for (JsonNode valueNode: customFieldNode.get("value"))
- fullNames.add(valueNode.get("name").asText());
- if (!fullNames.isEmpty()) {
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));
- } else if (!(mapped.getFirst() instanceof UserChoiceField) || !mapped.getFirst().isAllowMultiple()) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-valued user field");
- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));
- } else {
- for (JsonNode valueNode: customFieldNode.get("value")) {
- String login = valueNode.get("login").asText();
- String fullName = valueNode.get("name").asText();
- String email = getEmail(valueNode);
-
- if (email != null) {
- User user = OneDev.getInstance(UserManager.class).findByEmail(email);
- if (user != null) {
- fieldValue = user.getName();
- } else {
- fieldValue = fullName;
- nonExistentLogins.add(login);
- }
- } else {
- fieldValue = fullName;
- nonExistentLogins.add(login);
- }
-
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.USER);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- }
- }
- break;
- case "DateIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- Date date = new Date(customFieldNode.get("value").asLong());
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, DateUtils.formatDate(date));
- } else if (!(mapped.getFirst() instanceof DateField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a date field");
- extraIssueInfo.put(fieldName, DateUtils.formatDate(date));
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.DATE);
- issueField.setValue(DateUtils.formatDate(date));
- issue.getFields().add(issueField);
- }
- }
- break;
- case "SimpleIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- JsonNode projectCustomField = customFieldNode.get("projectCustomField");
- JsonNode fieldNode = projectCustomField.get("field");
- if (fieldNode != null) {
- switch (fieldNode.get("fieldType").get("id").asText()) {
- case "date":
- Date date = new Date(customFieldNode.get("value").asLong());
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));
- } else if (!(mapped.getFirst() instanceof DateTimeField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a datetime field");
- extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.DATE_TIME);
- issueField.setValue(DateUtils.formatDateTime(date));
- issue.getFields().add(issueField);
- }
- break;
- case "float":
- fieldValue = customFieldNode.get("value").asText();
- mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof FloatField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a float field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.FLOAT);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- break;
- case "integer":
- fieldValue = customFieldNode.get("value").asText();
- mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof IntegerField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to an integer field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.INTEGER);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- break;
- case "string":
- fieldValue = customFieldNode.get("value").asText();
- mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof TextField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.TEXT);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- break;
- }
- }
- }
- break;
- case "TextIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- fieldValue = customFieldNode.get("value").get("text").asText();
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof TextField)
- || !((TextField)mapped.getFirst()).isMultiline()) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-line text field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.TEXT);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- }
- break;
- case "PeriodIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- fieldValue = customFieldNode.get("value").get("presentation").asText();
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof WorkingPeriodField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a working period field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.WORKING_PERIOD);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- }
- break;
- }
- }
- if (issue.getState() == null)
- issue.setState(initialState.getName());
-
- for (JsonNode linkNode: issueNode.get("links")) {
- List<String> linkedIssueLinks = new ArrayList<>();
- for (JsonNode linkedIssueNode: linkNode.get("issues"))
- linkedIssueLinks.add("issue #" + linkedIssueNode.get("numberInProject").asLong());
-
- if (!linkedIssueLinks.isEmpty() && linkNode.hasNonNull("linkType")) {
- JsonNode linkTypeNode = linkNode.get("linkType");
- switch (linkNode.get("direction").asText()) {
- case "BOTH":
- case "OUTWARD":
- String linkName = linkTypeNode.get("sourceToTarget").asText(null);
- if (linkName != null)
- extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));
- break;
- case "INWARD":
- linkName = linkTypeNode.get("targetToSource").asText(null);
- if (linkName != null)
- extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));
- break;
- }
- }
- }
-
- List<String> currentUnmappedTags = new ArrayList<>();
- for (JsonNode tagNode: issueNode.get("tags")) {
- String tagName = tagNode.get("name").asText();
- Pair<FieldSpec, String> mapped = tagMappings.get(tagName);
- if (mapped != null) {
- IssueField tagField = new IssueField();
- tagField.setIssue(issue);
- tagField.setName(mapped.getFirst().getName());
- tagField.setType(InputSpec.ENUMERATION);
- tagField.setValue(mapped.getSecond());
- tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
- issue.getFields().add(tagField);
- } else {
- currentUnmappedTags.add(HtmlEscape.escapeHtml5(tagName));
- unmappedIssueTags.add(HtmlEscape.escapeHtml5(tagName));
- }
- }
- if (!currentUnmappedTags.isEmpty())
- extraIssueInfo.put("Tags", joinAsMultilineHtml(currentUnmappedTags));
-
- for (JsonNode commentNode: issueNode.get("comments")) {
- String commentContent = commentNode.get("text").asText(null);
- if (commentContent != null || !commentNode.get("attachments").isEmpty()) {
- IssueComment comment = new IssueComment();
- comment.setIssue(issue);
- if (!dryRun) {
- List<JsonNode> attachmentNodes = new ArrayList<>();
- for (JsonNode attachmentNode: commentNode.get("attachments"))
- attachmentNodes.add(attachmentNode);
- comment.setContent(processAttachments(issue.getUUID(), readableId,
- commentContent, attachmentNodes, tooLargeAttachments));
- }
- comment.setDate(new Date(commentNode.get("created").asLong(System.currentTimeMillis())));
- if (commentNode.hasNonNull("author")) {
- JsonNode authorNode = commentNode.get("author");
- String email = getEmail(authorNode);
- String fullName = authorNode.get("name").asText();
- String login = authorNode.get("login").asText();
- if (email != null) {
- User user = OneDev.getInstance(UserManager.class).findByEmail(email);
- if (user != null) {
- comment.setUser(user);
- } else {
- comment.setUserName(fullName);
- nonExistentLogins.add(login);
- }
- } else {
- comment.setUserName(fullName);
- nonExistentLogins.add(login);
- }
- } else {
- issue.setSubmitter(SecurityUtils.getUser());
- }
- issue.getComments().add(comment);
- }
- }
-
- issue.setCommentCount(issue.getComments().size());
-
- if (!extraIssueInfo.isEmpty()) {
- StringBuilder builder = new StringBuilder("|");
- for (String key: extraIssueInfo.keySet())
- builder.append(key).append("|");
- builder.append("\n|");
- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
- builder.append("\n|");
- for (String value: extraIssueInfo.values())
- builder.append(value).append("|");
-
- if (issue.getDescription() != null)
- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
- else
- issue.setDescription(builder.toString());
- }
-
- if (!dryRun) {
- OneDev.getInstance(IssueManager.class).save(issue);
- for (IssueField field: issue.getFields())
- OneDev.getInstance(Dao.class).persist(field);
- for (IssueComment comment: issue.getComments())
- OneDev.getInstance(Dao.class).persist(comment);
- }
-
- }
- logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");
- }
-
- };
- list(client, importSource, "/admin/projects/" + youTrackProjectId + "/issues?fields=" + fields,
- pageDataConsumer, logger);
- }
- StringBuilder feedback = new StringBuilder("Projects imported successfully");
-
- boolean hasNotes =
- !unmappedIssueStates.isEmpty()
- || !unmappedIssueFields.isEmpty()
- || !mismatchedIssueFields.isEmpty()
- || !unmappedIssueTags.isEmpty()
- || !nonExistentLogins.isEmpty()
- || !tooLargeAttachments.isEmpty();
-
- if (hasNotes)
- feedback.append("<br><br><b>NOTE:</b><ul>");
-
- if (!unmappedIssueStates.isEmpty()) {
- feedback.append("<li> Unmapped YouTrack issue states (using OneDev initial state): "
- + HtmlEscape.escapeHtml5(unmappedIssueStates.toString()));
- }
- if (!unmappedIssueFields.isEmpty()) {
- feedback.append("<li> Unmapped YouTrack issue fields (mentioned as extra info in issue description): "
- + HtmlEscape.escapeHtml5(unmappedIssueFields.toString()));
- }
- if (!mismatchedIssueFields.isEmpty()) {
- feedback.append("<li> YouTrack issue fields mapped to wrong type of OneDev issue field (mentioned as extra info in issue description):");
- feedback.append("<ul>");
- for (Map.Entry<String, String> entry: mismatchedIssueFields.entrySet()) {
- feedback.append("<li>")
- .append(HtmlEscape.escapeHtml5(entry.getKey()))
- .append(" : ")
- .append(HtmlEscape.escapeHtml5(entry.getValue()));
- }
- feedback.append("</ul>");
- }
- if (!unmappedIssueTags.isEmpty()) {
- feedback.append("<li> YouTrack issue tags not mapped to OneDev custom field (mentioned as extra info in issue description): "
- + HtmlEscape.escapeHtml5(unmappedIssueTags.toString()));
- }
- if (!nonExistentLogins.isEmpty()) {
- feedback.append("<li> YouTrack logins without email or email can not be mapped to OneDev account: "
- + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));
- }
- if (!tooLargeAttachments.isEmpty()) {
- feedback.append("<li> Too large attachments: "
- + HtmlEscape.escapeHtml5(tooLargeAttachments.toString()));
- }
-
- if (hasNotes)
- feedback.append("</ul>");
-
- return feedback.toString();
- } catch (Exception e) {
- for (Long projectId: projectIds)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
- throw ExceptionUtils.unchecked(e);
- } finally {
- client.close();
- }
- }
-
- private GlobalIssueSetting getIssueSetting() {
- return OneDev.getInstance(SettingManager.class).getIssueSetting();
- }
-
- private List<JsonNode> list(Client client, YouTrackImportSource importSource,
- String apiPath, SimpleLogger logger) {
- List<JsonNode> result = new ArrayList<>();
- list(client, importSource, apiPath, new PageDataConsumer() {
-
- @Override
- public void consume(List<JsonNode> pageData) {
- result.addAll(pageData);
- }
-
- }, logger);
- return result;
- }
-
- private void list(Client client, YouTrackImportSource importSource, String apiPath,
- PageDataConsumer pageDataConsumer, SimpleLogger logger) {
- URI uri;
- try {
- uri = new URIBuilder(importSource.getApiEndpoint(apiPath))
- .addParameter("$top", String.valueOf(PER_PAGE)).build();
- } catch (URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- int page = 0;
- while (true) {
- try {
- URIBuilder builder = new URIBuilder(uri);
- builder.addParameter("$skip", String.valueOf(page*PER_PAGE));
- List<JsonNode> pageData = new ArrayList<>();
- for (JsonNode each: get(client, builder.build().toString(), logger))
- pageData.add(each);
- pageDataConsumer.consume(pageData);
- if (pageData.size() < PER_PAGE)
- break;
- page++;
- } catch (URISyntaxException|InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- private JsonNode get(Client client, String apiEndpoint, SimpleLogger logger) {
- WebTarget target = client.target(apiEndpoint);
- Invocation.Builder builder = target.request();
- try (Response response = builder.get()) {
- String errorMessage = JerseyUtils.checkStatus(response);
- if (errorMessage != null)
- throw new ExplicitException(errorMessage);
- else
- return response.readEntity(JsonNode.class);
- }
- }
-
- private Client newClient(YouTrackImportSource importSource) {
- Client client = ClientBuilder.newClient();
- client.register(HttpAuthenticationFeature.basic(importSource.getUserName(), importSource.getPassword()));
- return client;
- }
-
- private static interface PageDataConsumer {
-
- void consume(List<JsonNode> pageData) throws InterruptedException;
-
- }
-
-}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportOption.java
similarity index 51%
rename from server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.java
rename to server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportOption.java
index b03e6e16a9..327e3a5c76 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportOption.java
@@ -4,39 +4,19 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import javax.validation.ConstraintValidatorContext;
-import javax.validation.constraints.Size;
-
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.ProjectManager;
-import io.onedev.server.util.validation.Validatable;
-import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@Editable
-@ClassValidating
-public class YouTrackImportOption implements Serializable, Validatable {
+public class YouTrackIssueImportOption implements Serializable {
private static final long serialVersionUID = 1L;
- private List<YouTrackImport> imports = new ArrayList<>();
-
private List<IssueStateMapping> issueStateMappings = new ArrayList<>();
private List<IssueFieldMapping> issueFieldMappings = new ArrayList<>();
private List<IssueTagMapping> issueTagMappings = new ArrayList<>();
- @Editable(order=100, name="Projects to Import")
- @Size(min=1, max=10000, message="No projects to import")
- public List<YouTrackImport> getImports() {
- return imports;
- }
-
- public void setImports(List<YouTrackImport> imports) {
- this.imports = imports;
- }
-
@Editable(order=200, description="Specify how to map YouTrack issue state to OneDev issue state. "
+ "Unmapped states will use the initial state in OneDev")
public List<IssueStateMapping> getIssueStateMappings() {
@@ -70,34 +50,4 @@ public class YouTrackImportOption implements Serializable, Validatable {
this.issueTagMappings = issueTagMappings;
}
- @Override
- public boolean isValid(ConstraintValidatorContext context) {
- boolean isValid = true;
- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
- for (int i=0; i<imports.size(); i++) {
- if (projectManager.find(imports.get(i).getOneDevProject()) != null) {
- context.buildConstraintViolationWithTemplate("Project name already used")
- .addPropertyNode("imports")
- .addPropertyNode(YouTrackImport.PROP_ONEDEV_PROJECT)
- .inIterable().atIndex(i).addConstraintViolation();
- isValid = false;
- } else {
- for (int j=0; j<imports.size(); j++) {
- if (j != i && imports.get(j).getOneDevProject().equals(imports.get(i).getOneDevProject())) {
- context.buildConstraintViolationWithTemplate("Duplicate project name")
- .addPropertyNode("imports")
- .addPropertyNode(YouTrackImport.PROP_ONEDEV_PROJECT)
- .inIterable().atIndex(i).addConstraintViolation();
- isValid = false;
- break;
- }
- }
- }
- }
-
- if (!isValid)
- context.disableDefaultConstraintViolation();
- return isValid;
- }
-
}
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportSource.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportSource.java
new file mode 100644
index 0000000000..caf0896744
--- /dev/null
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportSource.java
@@ -0,0 +1,85 @@
+package io.onedev.server.plugin.imports.youtrack;
+
+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.get;
+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.list;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+
+import org.apache.commons.lang3.StringUtils;
+import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
+import org.hibernate.validator.constraints.NotEmpty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import io.onedev.server.util.EditContext;
+import io.onedev.server.util.SimpleLogger;
+import io.onedev.server.web.editable.annotation.ChoiceProvider;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+public class YouTrackIssueImportSource extends YouTrackProjectImportSource {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger logger = LoggerFactory.getLogger(YouTrackIssueImportSource.class);
+
+ private String project;
+
+ @Editable(order=400, name="YouTrack Project", description="Choose YouTrack project to "
+ + "import issues from. Available projects will be populated when above options "
+ + "are specified")
+ @ChoiceProvider("getProjectChoices")
+ @NotEmpty
+ public String getProject() {
+ return project;
+ }
+
+ public void setProject(String project) {
+ this.project = project;
+ }
+
+ @SuppressWarnings("unused")
+ private static List<String> getProjectChoices() {
+ List<String> choices = new ArrayList<>();
+
+ String apiUrl = (String) EditContext.get().getInputValue(PROP_API_URL);
+ String userName = (String) EditContext.get().getInputValue(PROP_USER_NAME);
+ String password = (String) EditContext.get().getInputValue(PROP_PASSWORD);
+
+ if (StringUtils.isNotBlank(apiUrl) && StringUtils.isNotBlank(userName)
+ && StringUtils.isNotBlank(password)) {
+ Client client = ClientBuilder.newClient();
+ try {
+ client.register(HttpAuthenticationFeature.basic(userName, password));
+ SimpleLogger logger = new SimpleLogger() {
+
+ @Override
+ public void log(String message) {
+ YouTrackIssueImportSource.logger.info(message);
+ }
+
+ };
+ String apiEndpoint = getApiEndpoint(apiUrl, "/users/me?fields=guest");
+ if (!get(client, apiEndpoint, logger).get("guest").asBoolean()) {
+ apiEndpoint = getApiEndpoint(apiUrl, "/admin/projects?fields=name");
+ for (JsonNode projectNode: list(client, apiEndpoint, logger))
+ choices.add(projectNode.get("name").asText());
+ }
+ } catch (Exception e) {
+ } finally {
+ client.close();
+ }
+ }
+
+ Collections.sort(choices);
+ return choices;
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java
new file mode 100644
index 0000000000..cc7bbdc6df
--- /dev/null
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java
@@ -0,0 +1,57 @@
+package io.onedev.server.plugin.imports.youtrack;
+
+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.*;
+
+import javax.ws.rs.client.Client;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import io.onedev.commons.utils.ExplicitException;
+import io.onedev.server.model.Project;
+import io.onedev.server.util.SimpleLogger;
+import io.onedev.server.web.page.project.issues.imports.IssueImporter;
+
+public class YouTrackIssueImporter extends IssueImporter<YouTrackIssueImportSource, YouTrackIssueImportOption> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ private String getYouTrackProjectId(YouTrackProjectImportSource importSource,
+ String youTrackProjectName, SimpleLogger logger) {
+ Client client = newClient(importSource);
+ try {
+ String apiEndpoint = importSource.getApiEndpoint("/admin/projects?fields=id,name");
+ for (JsonNode projectNode: list(client, apiEndpoint, logger)) {
+ if (youTrackProjectName.equals(projectNode.get("name").asText()))
+ return projectNode.get("id").asText();
+ }
+ throw new ExplicitException("Unable to find YouTrack project: " + youTrackProjectName);
+ } finally {
+ client.close();
+ }
+ }
+
+ @Override
+ public YouTrackIssueImportOption getImportOption(YouTrackIssueImportSource importSource, SimpleLogger logger) {
+ if (importSource.isPrepopulateImportOptions()) {
+ String youTrackProjectId = getYouTrackProjectId(importSource, importSource.getProject(), logger);
+ return buildImportOption(importSource, youTrackProjectId, logger);
+ } else {
+ return new YouTrackIssueImportOption();
+ }
+ }
+
+ @Override
+ public String doImport(Project project, YouTrackIssueImportSource importSource,
+ YouTrackIssueImportOption importOption, boolean dryRun, SimpleLogger logger) {
+ logger.log("Importing issues from '" + importSource.getProject() + "'...");
+ String youTrackProjectId = getYouTrackProjectId(importSource, importSource.getProject(), logger);
+ return importIssues(importSource, youTrackProjectId, project, false, importOption, dryRun, logger)
+ .toHtml("Issues imported successfully");
+ }
+
+}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java
index ce5c2ef6fd..4f7170b203 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java
@@ -8,6 +8,8 @@ import com.google.common.collect.Lists;
import io.onedev.commons.launcher.loader.AbstractPluginModule;
import io.onedev.server.web.page.project.imports.ProjectImporter;
import io.onedev.server.web.page.project.imports.ProjectImporterContribution;
+import io.onedev.server.web.page.project.issues.imports.IssueImporter;
+import io.onedev.server.web.page.project.issues.imports.IssueImporterContribution;
/**
* NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.
@@ -24,7 +26,16 @@ public class YouTrackPluginModule extends AbstractPluginModule {
@Override
public Collection<ProjectImporter<? extends Serializable, ? extends Serializable>> getImporters() {
- return Lists.newArrayList(new YouTrackImporter());
+ return Lists.newArrayList(new YouTrackProjectImporter());
+ }
+
+ });
+
+ contribute(IssueImporterContribution.class, new IssueImporterContribution() {
+
+ @Override
+ public Collection<IssueImporter<? extends Serializable, ? extends Serializable>> getImporters() {
+ return Lists.newArrayList(new YouTrackIssueImporter());
}
});
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportOption.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportOption.java
new file mode 100644
index 0000000000..14e4397376
--- /dev/null
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportOption.java
@@ -0,0 +1,63 @@
+package io.onedev.server.plugin.imports.youtrack;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.validation.ConstraintValidatorContext;
+import javax.validation.constraints.Size;
+
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.util.validation.Validatable;
+import io.onedev.server.util.validation.annotation.ClassValidating;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+@ClassValidating
+public class YouTrackProjectImportOption extends YouTrackIssueImportOption implements Validatable {
+
+ private static final long serialVersionUID = 1L;
+
+ private List<ProjectMapping> projecMappings = new ArrayList<>();
+
+ @Editable(order=100, name="Projects to Import")
+ @Size(min=1, max=10000, message="No projects to import")
+ public List<ProjectMapping> getProjectMappings() {
+ return projecMappings;
+ }
+
+ public void setProjectMappings(List<ProjectMapping> projectMappings) {
+ this.projecMappings = projectMappings;
+ }
+
+ @Override
+ public boolean isValid(ConstraintValidatorContext context) {
+ boolean isValid = true;
+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
+ for (int i=0; i<projecMappings.size(); i++) {
+ if (projectManager.find(projecMappings.get(i).getOneDevProject()) != null) {
+ context.buildConstraintViolationWithTemplate("Project name already used")
+ .addPropertyNode("imports")
+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
+ .inIterable().atIndex(i).addConstraintViolation();
+ isValid = false;
+ } else {
+ for (int j=0; j<projecMappings.size(); j++) {
+ if (j != i && projecMappings.get(j).getOneDevProject().equals(projecMappings.get(i).getOneDevProject())) {
+ context.buildConstraintViolationWithTemplate("Duplicate project name")
+ .addPropertyNode("imports")
+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)
+ .inIterable().atIndex(i).addConstraintViolation();
+ isValid = false;
+ break;
+ }
+ }
+ }
+ }
+
+ if (!isValid)
+ context.disableDefaultConstraintViolation();
+ return isValid;
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportSource.java
similarity index 83%
rename from server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.java
rename to server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportSource.java
index e7d052383e..8722357070 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportSource.java
@@ -23,15 +23,15 @@ import io.onedev.server.web.editable.annotation.Password;
@Editable
@ClassValidating
-public class YouTrackImportSource implements Serializable, Validatable {
+public class YouTrackProjectImportSource implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
- private static final String PROP_API_URL = "apiUrl";
+ protected static final String PROP_API_URL = "apiUrl";
- private static final String PROP_USER_NAME = "userName";
+ protected static final String PROP_USER_NAME = "userName";
- private static final String PROP_PASSWORD = "password";
+ protected static final String PROP_PASSWORD = "password";
private String apiUrl;
@@ -77,7 +77,7 @@ public class YouTrackImportSource implements Serializable, Validatable {
this.password = password;
}
- @Editable(order=400, description="If checked, import options will be pre-populated based on all accessible "
+ @Editable(order=1000, description="If checked, import options will be pre-populated based on all accessible "
+ "projects and its settings")
public boolean isPrepopulateImportOptions() {
return prepopulateImportOptions;
@@ -87,19 +87,24 @@ public class YouTrackImportSource implements Serializable, Validatable {
this.prepopulateImportOptions = prepopulateImportOptions;
}
- public String getApiEndpoint(String apiPath) {
+ static String getApiEndpoint(String apiUrl, String apiPath) {
return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");
}
+ public String getApiEndpoint(String apiPath) {
+ return getApiEndpoint(apiUrl, apiPath);
+ }
+
@Override
public boolean isValid(ConstraintValidatorContext context) {
Client client = ClientBuilder.newClient();
- client.register(HttpAuthenticationFeature.basic(getUserName(), getPassword()));
try {
- WebTarget target = client.target(getApiEndpoint("/users/me?fields=guest"));
+ client.register(HttpAuthenticationFeature.basic(getUserName(), getPassword()));
+ String apiEndpoint = getApiEndpoint("/users/me?fields=guest");
+ WebTarget target = client.target(apiEndpoint);
Invocation.Builder builder = target.request();
try (Response response = builder.get()) {
- String errorMessage = JerseyUtils.checkStatus(response);
+ String errorMessage = JerseyUtils.checkStatus(apiEndpoint, response);
if (errorMessage != null) {
context.disableDefaultConstraintViolation();
context.buildConstraintViolationWithTemplate(errorMessage)
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java
new file mode 100644
index 0000000000..963c0f5e14
--- /dev/null
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java
@@ -0,0 +1,116 @@
+package io.onedev.server.plugin.imports.youtrack;
+
+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.NAME;
+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.buildImportOption;
+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.importIssues;
+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.list;
+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.newClient;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.ws.rs.client.Client;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import io.onedev.commons.utils.ExceptionUtils;
+import io.onedev.commons.utils.ExplicitException;
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.model.Project;
+import io.onedev.server.storage.StorageManager;
+import io.onedev.server.util.SimpleLogger;
+import io.onedev.server.web.page.project.imports.ProjectImporter;
+
+public class YouTrackProjectImporter extends ProjectImporter<YouTrackProjectImportSource, YouTrackProjectImportOption> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public YouTrackProjectImportOption getImportOption(YouTrackProjectImportSource importSource, SimpleLogger logger) {
+ YouTrackProjectImportOption importOption = new YouTrackProjectImportOption();
+ if (importSource.isPrepopulateImportOptions()) {
+ Client client = newClient(importSource);
+ try {
+ String apiEndpoint = importSource.getApiEndpoint("/admin/projects?fields=name");
+ for (JsonNode projectNode: list(client, apiEndpoint, logger)) {
+ ProjectMapping projectMapping = new ProjectMapping();
+ projectMapping.setYouTrackProject(projectNode.get("name").asText());
+ projectMapping.setOneDevProject(projectMapping.getYouTrackProject().replace(' ', '-'));
+ importOption.getProjectMappings().add(projectMapping);
+ }
+ YouTrackIssueImportOption issueImportOption = buildImportOption(importSource, null, logger);
+ importOption.setIssueFieldMappings(issueImportOption.getIssueFieldMappings());
+ importOption.setIssueStateMappings(issueImportOption.getIssueStateMappings());
+ importOption.setIssueTagMappings(issueImportOption.getIssueTagMappings());
+ } finally {
+ client.close();
+ }
+ }
+ return importOption;
+ }
+
+ @Override
+ public String doImport(YouTrackProjectImportSource importSource, YouTrackProjectImportOption importOption,
+ boolean dryRun, SimpleLogger logger) {
+
+ Map<String, String> youTrackProjectIds = new HashMap<>();
+ Map<String, String> youTrackProjectDescriptions = new HashMap<>();
+ Collection<Long> projectIds = new ArrayList<>();
+
+ Client client = newClient(importSource);
+ try {
+ String apiEndpoint = importSource.getApiEndpoint("/admin/projects?fields=id,name,description");
+ for (JsonNode projectNode: list(client, apiEndpoint, logger)) {
+ String projectName = projectNode.get("name").asText();
+ youTrackProjectIds.put(projectName, projectNode.get("id").asText());
+ youTrackProjectDescriptions.put(projectName, projectNode.get("description").asText(null));
+ }
+
+ YouTrackImportResult result = new YouTrackImportResult();
+
+ for (ProjectMapping projectMapping: importOption.getProjectMappings()) {
+ logger.log("Importing project '" + projectMapping.getYouTrackProject() + "'...");
+
+ String youTrackProjectId = youTrackProjectIds.get(projectMapping.getYouTrackProject());
+ if (youTrackProjectId == null)
+ throw new ExplicitException("Unable to find YouTrack project: " + projectMapping.getYouTrackProject());
+
+ Project project = new Project();
+ project.setName(projectMapping.getOneDevProject());
+ project.setDescription(youTrackProjectDescriptions.get(projectMapping.getYouTrackProject()));
+ project.setIssueManagementEnabled(true);
+
+ if (!dryRun) {
+ OneDev.getInstance(ProjectManager.class).create(project);
+ projectIds.add(project.getId());
+ }
+
+ YouTrackImportResult currentResult = importIssues(importSource, youTrackProjectId, project,
+ true, importOption, dryRun, logger);
+ result.mismatchedIssueFields.putAll(currentResult.mismatchedIssueFields);
+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
+ result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);
+ result.unmappedIssueFields.addAll(currentResult.unmappedIssueFields);
+ result.unmappedIssueStates.addAll(currentResult.unmappedIssueStates);
+ result.unmappedIssueTags.addAll(currentResult.unmappedIssueTags);
+ }
+
+ return result.toHtml("Projects imported successfully");
+ } catch (Exception e) {
+ for (Long projectId: projectIds)
+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ throw ExceptionUtils.unchecked(e);
+ } finally {
+ client.close();
+ }
+ }
+
+}
\ No newline at end of file