The Unified and Autonomous Development Platform
Fix issue #342 - Able to import YouTrack/GitHub issues into existing project
Robin Shen469dee7765a3
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/IssueManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/IssueManager.javaindex 4a051ca331..51bcb34809 100644--- a/server-core/src/main/java/io/onedev/server/entitymanager/IssueManager.java+++ b/server-core/src/main/java/io/onedev/server/entitymanager/IssueManager.java@@ -32,6 +32,8 @@ public interface IssueManager extends EntityManager<Issue> {void open(Issue issue);+ Long getNextNumber(Project numberScope);+List<Issue> query(@Nullable Project project, EntityQuery<Issue> issueQuery,int firstResult, int maxResults, boolean loadFields);diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultIssueManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultIssueManager.javaindex 5237f0692f..2695592784 100644--- a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultIssueManager.java+++ b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultIssueManager.java@@ -164,10 +164,7 @@ public class DefaultIssueManager extends BaseEntityManager<Issue> implements Isspublic void open(Issue issue) {Preconditions.checkArgument(issue.isNew());issue.setNumberScope(issue.getProject().getForkRoot());- Query<?> query = getSession().createQuery(String.format("select max(%s) from Issue where %s=:numberScope",- Issue.PROP_NUMBER, Issue.PROP_NUMBER_SCOPE));- query.setParameter("numberScope", issue.getNumberScope());- issue.setNumber(getNextNumber(issue.getNumberScope(), query));+ issue.setNumber(getNextNumber(issue.getNumberScope()));IssueOpened event = new IssueOpened(issue);issue.setLastUpdate(event.getLastUpdate());@@ -178,6 +175,14 @@ public class DefaultIssueManager extends BaseEntityManager<Issue> implements IsslistenerRegistry.post(event);}++ @Sessional+ public Long getNextNumber(Project numberScope) {+ Query<?> query = getSession().createQuery(String.format("select max(%s) from Issue where %s=:numberScope",+ Issue.PROP_NUMBER, Issue.PROP_NUMBER_SCOPE));+ query.setParameter(Issue.PROP_NUMBER_SCOPE, numberScope);+ return getNextNumber(numberScope, query);+ }@Transactional@Overridediff --git a/server-core/src/main/java/io/onedev/server/model/Project.java b/server-core/src/main/java/io/onedev/server/model/Project.javaindex 5e792d5f11..ee78dea58a 100644--- a/server-core/src/main/java/io/onedev/server/model/Project.java+++ b/server-core/src/main/java/io/onedev/server/model/Project.java@@ -1040,7 +1040,7 @@ public class Project extends AbstractEntity implements NameAware {this.codeComments = codeComments;}- @Editable(order=300, name="Issue management", description="Whether or not to provide issue management for the project")+ @Editable(order=300, name="Issue management", description="Whether or not to enable issue management for the project")public boolean isIssueManagementEnabled() {return issueManagementEnabled;}diff --git a/server-core/src/main/java/io/onedev/server/util/JerseyUtils.java b/server-core/src/main/java/io/onedev/server/util/JerseyUtils.javaindex 2fdcd7d008..db017c9084 100644--- a/server-core/src/main/java/io/onedev/server/util/JerseyUtils.java+++ b/server-core/src/main/java/io/onedev/server/util/JerseyUtils.java@@ -8,15 +8,15 @@ import io.onedev.commons.utils.StringUtils;public class JerseyUtils {@Nullable- public static String checkStatus(Response response) {+ public static String checkStatus(String url, Response response) {int status = response.getStatus();if (status != 200) {String errorMessage = response.readEntity(String.class);if (StringUtils.isNotBlank(errorMessage)) {- return String.format("Http request failed (status code: %d, error message: %s)",- status, errorMessage);+ return String.format("Http request failed (url: %s, status code: %d, error message: %s)",+ url, status, errorMessage);} else {- return String.format("Http request failed (status code: %d)", status);+ return String.format("Http request failed (url: %s, status code: %d)", url, status);}} else {return null;diff --git a/server-core/src/main/java/io/onedev/server/util/ReferenceMigrator.java b/server-core/src/main/java/io/onedev/server/util/ReferenceMigrator.javanew file mode 100644index 0000000000..92609c67be--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/util/ReferenceMigrator.java@@ -0,0 +1,59 @@+package io.onedev.server.util;++import java.util.Map;++import io.onedev.commons.utils.WordUtils;++public class ReferenceMigrator {++ private final String referenceType;++ private final Map<Long, Long> numberMappings;++ public ReferenceMigrator(Class<?> referenceClass, Map<Long, Long> numberMappings) {+ referenceType = WordUtils.uncamel(referenceClass.getSimpleName()).toLowerCase();+ this.numberMappings = numberMappings;+ }++ public String migratePrefixed(String content, String prefix) {+ StringBuilder builder = new StringBuilder();+ int start = 0;+ while (true) {+ int index = content.indexOf(prefix, start);+ if (index == -1) {+ builder.append(content.substring(start));+ break;+ } else {+ builder.append(content.substring(start, index));+ start = index + prefix.length();+ if (index > 0 && Character.isLetterOrDigit(content.charAt(index-1))) {+ builder.append(prefix);+ } else {+ StringBuilder numberBuilder = new StringBuilder();+ while (start < content.length()) {+ char ch = content.charAt(start);+ if (Character.isDigit(ch)) {+ numberBuilder.append(ch);+ start++;+ } else {+ if (Character.isAlphabetic(ch))+ numberBuilder.setLength(0);+ break;+ }+ }+ if (numberBuilder.length() != 0) {+ Long newNumber = numberMappings.get(Long.valueOf(numberBuilder.toString()));+ if (newNumber != null)+ builder.append(referenceType).append(" #").append(newNumber);+ else+ builder.append(content.substring(index, start));+ } else {+ builder.append(content.substring(index, start));+ }+ }+ }+ }+ return builder.toString();+ }++}diff --git a/server-core/src/main/java/io/onedev/server/web/OneUrlMapper.java b/server-core/src/main/java/io/onedev/server/web/OneUrlMapper.javaindex c8253172a0..071d30c9d9 100644--- a/server-core/src/main/java/io/onedev/server/web/OneUrlMapper.java+++ b/server-core/src/main/java/io/onedev/server/web/OneUrlMapper.java@@ -80,6 +80,7 @@ import io.onedev.server.web.page.project.issues.detail.IssueActivitiesPage;import io.onedev.server.web.page.project.issues.detail.IssueBuildsPage;import io.onedev.server.web.page.project.issues.detail.IssueCommitsPage;import io.onedev.server.web.page.project.issues.detail.IssuePullRequestsPage;+import io.onedev.server.web.page.project.issues.imports.IssueImportPage;import io.onedev.server.web.page.project.issues.list.ProjectIssueListPage;import io.onedev.server.web.page.project.issues.milestones.MilestoneDetailPage;import io.onedev.server.web.page.project.issues.milestones.MilestoneEditPage;@@ -258,6 +259,7 @@ public class OneUrlMapper extends CompoundRequestMapper {add(new DynamicPathPageMapper("projects/${project}/issues/${issue}/pull-requests", IssuePullRequestsPage.class));add(new DynamicPathPageMapper("projects/${project}/issues/${issue}/builds", IssueBuildsPage.class));add(new DynamicPathPageMapper("projects/${project}/issues/new", NewIssuePage.class));+ add(new DynamicPathPageMapper("projects/${project}/issues/import/${importer}", IssueImportPage.class));add(new DynamicPathPageMapper("projects/${project}/milestones", MilestoneListPage.class));add(new DynamicPathPageMapper("projects/${project}/milestones/${milestone}", MilestoneDetailPage.class));add(new DynamicPathPageMapper("projects/${project}/milestones/${milestone}/edit", MilestoneEditPage.class));diff --git a/server-core/src/main/java/io/onedev/server/web/asset/icon/import.svg b/server-core/src/main/java/io/onedev/server/web/asset/icon/import.svgnew file mode 100644index 0000000000..a7ea11fd46--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/web/asset/icon/import.svg@@ -0,0 +1 @@+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1624065252808" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="1212" xmlns:xlink="http://www.w3.org/1999/xlink" width="200" height="200"><defs><style type="text/css"></style></defs><path d="M481.28 715.946667a32 32 0 0 0 22.613333 9.386666h16.213334a32.853333 32.853333 0 0 0 22.613333-9.386666l218.88-219.306667a21.333333 21.333333 0 0 0 0-30.293333l-30.293333-30.293334a20.906667 20.906667 0 0 0-29.866667 0L554.666667 583.253333V149.333333a21.333333 21.333333 0 0 0-21.333334-21.333333h-42.666666a21.333333 21.333333 0 0 0-21.333334 21.333333v433.92l-146.773333-147.2a21.333333 21.333333 0 0 0-30.293333 0l-29.866667 30.293334a21.333333 21.333333 0 0 0 0 30.293333z m30.72-90.026667zM896 810.666667v-149.333334a21.333333 21.333333 0 0 0-21.333333-21.333333h-42.666667a21.333333 21.333333 0 0 0-21.333333 21.333333V810.666667H213.333333v-149.333334a21.333333 21.333333 0 0 0-21.333333-21.333333h-42.666667a21.333333 21.333333 0 0 0-21.333333 21.333333V810.666667a85.333333 85.333333 0 0 0 85.333333 85.333333h597.333334a85.333333 85.333333 0 0 0 85.333333-85.333333z" p-id="1213"></path></svg>\ No newline at end of filediff --git a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.html b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.htmlindex 055d0556b0..39da962fc4 100644--- a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.html+++ b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.html@@ -28,9 +28,12 @@<a wicket:id="listFields" class="list-fields text-gray mr-4"><wicket:svg href="select" class="icon"></wicket:svg> Display Fields</a>- <a wicket:id="orderBy" class="order-by text-gray">+ <a wicket:id="orderBy" class="order-by text-gray mr-4"><wicket:svg href="sort" class="icon"></wicket:svg> Order By</a>+ <a wicket:id="importIssues" class="import-issues text-gray">+ <wicket:svg href="import" class="icon"></wicket:svg> Import Issues+ </a></div><div wicket:id="body" class="body"><div wicket:id="feedback"></div>diff --git a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.javaindex 3014d6e769..1cb156520e 100644--- a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java+++ b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java@@ -1,5 +1,6 @@package io.onedev.server.web.component.issue.list;+import java.io.Serializable;import java.util.ArrayList;import java.util.Collection;import java.util.Comparator;@@ -85,6 +86,8 @@ import io.onedev.server.web.component.issue.fieldvalues.FieldValuesPanel;import io.onedev.server.web.component.link.ActionablePageLink;import io.onedev.server.web.component.link.DropdownLink;import io.onedev.server.web.component.link.copytoclipboard.CopyToClipboardLink;+import io.onedev.server.web.component.menu.MenuItem;+import io.onedev.server.web.component.menu.MenuLink;import io.onedev.server.web.component.modal.ModalLink;import io.onedev.server.web.component.modal.ModalPanel;import io.onedev.server.web.component.orderedit.OrderEditPanel;@@ -97,6 +100,10 @@ import io.onedev.server.web.component.user.ident.Mode;import io.onedev.server.web.component.user.ident.UserIdentPanel;import io.onedev.server.web.page.project.issues.create.NewIssuePage;import io.onedev.server.web.page.project.issues.detail.IssueActivitiesPage;+import io.onedev.server.web.page.project.issues.imports.IssueImportPage;+import io.onedev.server.web.page.project.issues.imports.IssueImporter;+import io.onedev.server.web.page.project.issues.imports.IssueImporterContribution;+import io.onedev.server.web.page.project.issues.list.ProjectIssueListPage;import io.onedev.server.web.util.Cursor;import io.onedev.server.web.util.LoadableDetachableDataProvider;import io.onedev.server.web.util.PagingHistorySupport;@@ -298,6 +305,43 @@ public abstract class IssueListPanel extends Panel {});+ add(new MenuLink("importIssues") {++ @Override+ protected List<MenuItem> getMenuItems(FloatingPanel dropdown) {+ Collection<IssueImporter<? extends Serializable, ? extends Serializable>> importers = new ArrayList<>();+ for (IssueImporterContribution contribution: OneDev.getExtensions(IssueImporterContribution.class))+ importers.addAll(contribution.getImporters());++ List<MenuItem> menuItems = new ArrayList<>();+ for (IssueImporter<? extends Serializable, ? extends Serializable> importer: importers) {+ menuItems.add(new MenuItem() {++ @Override+ public String getLabel() {+ return "From " + importer.getName();+ }++ @Override+ public WebMarkupContainer newLink(String id) {+ String url = urlFor(getPage().getClass(), getPage().getPageParameters()).toString();+ return new BookmarkablePageLink<Void>(id, IssueImportPage.class,+ IssueImportPage.paramsOf(getProject(), importer.getName(), url));+ }++ });+ }+ return menuItems;+ }++ @Override+ protected void onConfigure() {+ super.onConfigure();+ setVisible(getPage() instanceof ProjectIssueListPage);+ }++ });+queryInput = new TextField<String>("input", queryStringModel);queryInput.add(new IssueQueryBehavior(new AbstractReadOnlyModel<Project>() {diff --git a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.html b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.htmlindex 22798e2398..490cc3c459 100644--- a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.html+++ b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.html@@ -19,9 +19,12 @@<a wicket:id="saveQuery" class="save-query text-gray mr-4"><wicket:svg href="save" class="icon"></wicket:svg> Save Query</a>- <a wicket:id="orderBy" class="order-by text-gray">+ <a wicket:id="orderBy" class="order-by text-gray mr-4"><wicket:svg href="sort" class="icon"></wicket:svg> Order By</a>+ <a wicket:id="importProjects" class="import-projects text-gray">+ <wicket:svg href="import" class="icon"></wicket:svg> Import Projects+ </a></div><div wicket:id="body"><div wicket:id="feedback"></div>diff --git a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.javaindex d7e323b920..af8f223050 100644--- a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java+++ b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java@@ -240,6 +240,36 @@ public class ProjectListPanel extends Panel {});+ add(new MenuLink("importProjects") {++ @Override+ protected List<MenuItem> getMenuItems(FloatingPanel dropdown) {+ Collection<ProjectImporter<? extends Serializable, ? extends Serializable>> importers = new ArrayList<>();+ for (ProjectImporterContribution contribution: OneDev.getExtensions(ProjectImporterContribution.class))+ importers.addAll(contribution.getImporters());++ List<MenuItem> menuItems = new ArrayList<>();+ for (ProjectImporter<? extends Serializable, ? extends Serializable> importer: importers) {+ menuItems.add(new MenuItem() {++ @Override+ public String getLabel() {+ return "From " + importer.getName();+ }++ @Override+ public WebMarkupContainer newLink(String id) {+ return new BookmarkablePageLink<Void>(id, ProjectImportPage.class,+ ProjectImportPage.paramsOf(importer.getName()));+ }++ });+ }+ return menuItems;+ }++ });+queryInput = new TextField<String>("input", queryStringModel);queryInput.setOutputMarkupId(true);queryInput.add(new ProjectQueryBehavior() {@@ -276,52 +306,7 @@ public class ProjectListPanel extends Panel {});add(queryForm);- Collection<ProjectImporter<? extends Serializable, ? extends Serializable>> importers = new ArrayList<>();- for (ProjectImporterContribution contribution: OneDev.getExtensions(ProjectImporterContribution.class))- importers.addAll(contribution.getImporters());-- if (importers.isEmpty()) {- add(new BookmarkablePageLink<Void>("addProject", NewProjectPage.class));- } else {- add(new MenuLink("addProject") {-- @Override- protected List<MenuItem> getMenuItems(FloatingPanel dropdown) {- List<MenuItem> menuItems = new ArrayList<>();- menuItems.add(new MenuItem() {-- @Override- public String getLabel() {- return "Create New";- }-- @Override- public WebMarkupContainer newLink(String id) {- return new BookmarkablePageLink<Void>(id, NewProjectPage.class);- }-- });- for (ProjectImporter<? extends Serializable, ? extends Serializable> importer: importers) {- menuItems.add(new MenuItem() {-- @Override- public String getLabel() {- return "Import from " + importer.getName();- }-- @Override- public WebMarkupContainer newLink(String id) {- return new BookmarkablePageLink<Void>(id, ProjectImportPage.class,- ProjectImportPage.paramsOf(importer.getName()));- }-- });- }- return menuItems;- }-- });- }+ add(new BookmarkablePageLink<Void>("addProject", NewProjectPage.class));SortableDataProvider<Project, Void> dataProvider = new LoadableDetachableDataProvider<Project, Void>() {diff --git a/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskButton.java b/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskButton.javaindex c09f791978..d1c8c779af 100644--- a/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskButton.java+++ b/server-core/src/main/java/io/onedev/server/web/component/taskbutton/TaskButton.java@@ -21,6 +21,7 @@ import org.apache.wicket.Component;import org.apache.wicket.ajax.AjaxRequestTarget;import org.apache.wicket.ajax.markup.html.form.AjaxButton;import org.apache.wicket.markup.html.form.Form;+import org.apache.wicket.request.cycle.RequestCycle;import org.joda.time.DateTime;import org.quartz.ScheduleBuilder;import org.quartz.SimpleScheduleBuilder;@@ -67,6 +68,12 @@ public abstract class TaskButton extends AjaxButton {return WordUtils.uncamel(getId());}+ protected void onCompleted(AjaxRequestTarget target) {+ }++ protected void onCancelled(AjaxRequestTarget target) {+ }+protected void submitTask(AjaxRequestTarget target) {String path = getPath();String title = getTitle().toLowerCase();@@ -120,8 +127,14 @@ public abstract class TaskButton extends AjaxButton {protected void onClosed() {super.onClosed();Future<?> future = getTaskFutures().remove(path);- if (future != null && !future.isDone())++ AjaxRequestTarget target = RequestCycle.get().find(AjaxRequestTarget.class);+ if (future != null && !future.isDone()) {future.cancel(true);+ onCancelled(target);+ } else {+ onCompleted(target);+ }}@Overridediff --git a/server-core/src/main/java/io/onedev/server/web/editable/choice/MultiChoiceEditor.java b/server-core/src/main/java/io/onedev/server/web/editable/choice/MultiChoiceEditor.javaindex 434a18e912..9289455a12 100644--- a/server-core/src/main/java/io/onedev/server/web/editable/choice/MultiChoiceEditor.java+++ b/server-core/src/main/java/io/onedev/server/web/editable/choice/MultiChoiceEditor.java@@ -68,10 +68,20 @@ public class MultiChoiceEditor extends PropertyEditor<List<String>> {};List<String> selections = getModelObject();++ /*+ * Avoid loading choicesModel which might be time-consuming. Otherwise, when this component+ * is re-created due to a dependency property updating, there is a high chance we are+ * operating on the old select2 component and may cause UI clutter+ */+ /*if (selections != null)selections.retainAll(choicesModel.getObject().keySet());elseselections = new ArrayList<>();+ */+ if (selections == null)+ selections = new ArrayList<>();input = new StringMultiChoice("input", Model.of(selections), choicesModel) {@Overridediff --git a/server-core/src/main/java/io/onedev/server/web/editable/choice/SingleChoiceEditor.java b/server-core/src/main/java/io/onedev/server/web/editable/choice/SingleChoiceEditor.javaindex be03b8b3d4..0942f3f83f 100644--- a/server-core/src/main/java/io/onedev/server/web/editable/choice/SingleChoiceEditor.java+++ b/server-core/src/main/java/io/onedev/server/web/editable/choice/SingleChoiceEditor.java@@ -65,8 +65,16 @@ public class SingleChoiceEditor extends PropertyEditor<String> {};String selection = getModelObject();++ /*+ * Avoid loading choicesModel which might be time-consuming. Otherwise, when this component+ * is re-created due to a dependency property updating, there is a high chance we are+ * operating on the old select2 component and may cause UI clutter+ */+ /*if (!choicesModel.getObject().containsKey(selection))selection = null;+ */input = new StringSingleChoice("input", Model.of(selection), choicesModel) {@Overridediff --git a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.javaindex 6b355a457c..6e1a6692a5 100644--- a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java+++ b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java@@ -4,6 +4,7 @@ import java.io.Serializable;import java.util.concurrent.Callable;import org.apache.wicket.Component;+import org.apache.wicket.RestartResponseException;import org.apache.wicket.ajax.AjaxRequestTarget;import org.apache.wicket.ajax.attributes.AjaxRequestAttributes;import org.apache.wicket.ajax.markup.html.form.AjaxButton;@@ -14,13 +15,20 @@ import org.apache.wicket.request.mapper.parameter.PageParameters;import org.slf4j.Logger;import org.slf4j.LoggerFactory;+import com.google.common.collect.Lists;+import io.onedev.server.OneDev;+import io.onedev.server.model.Project;import io.onedev.server.persistence.TransactionManager;+import io.onedev.server.search.entity.EntitySort;+import io.onedev.server.search.entity.EntitySort.Direction;+import io.onedev.server.search.entity.project.ProjectQuery;import io.onedev.server.util.SimpleLogger;import io.onedev.server.web.ajaxlistener.ShowGlobalAjaxIndicatorListener;import io.onedev.server.web.component.taskbutton.TaskButton;import io.onedev.server.web.editable.BeanContext;import io.onedev.server.web.page.layout.LayoutPage;+import io.onedev.server.web.page.project.ProjectListPage;@SuppressWarnings("serial")public class ProjectImportPage<T extends Serializable, S extends Serializable> extends LayoutPage {@@ -107,6 +115,18 @@ public class ProjectImportPage<T extends Serializable, S extends Serializable> eform.add(new TaskButton("import") {+ @Override+ protected void onCompleted(AjaxRequestTarget target) {+ super.onCompleted(target);++ EntitySort sort = new EntitySort();+ sort.setField(Project.NAME_UPDATE_DATE);+ sort.setDirection(Direction.DESCENDING);+ ProjectQuery query = new ProjectQuery(null, Lists.newArrayList(sort));+ PageParameters params = ProjectListPage.paramsOf(query.toString(), 0, 0);+ throw new RestartResponseException(ProjectListPage.class, params);+ }+@Overrideprotected String runTask(SimpleLogger logger) {return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.html b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.htmlnew file mode 100644index 0000000000..4121f0f061--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.html@@ -0,0 +1,13 @@+<wicket:extend>+ <div class="issue-import card m-2 m-sm-5">+ <div class="card-body">+ <form wicket:id="form">+ <div wicket:id="feedback"></div>+ <div wicket:id="editor"></div>+ <input wicket:id="next" type="submit" class="btn btn-primary" value="Next">+ <input wicket:id="import" type="submit" class="btn btn-primary mr-1" value="Import">+ <input wicket:id="dryRun" type="submit" class="btn btn-light-primary" value="Dry Run">+ </form>+ </div>+ </div>+</wicket:extend>diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.javanew file mode 100644index 0000000000..013a60f769--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java@@ -0,0 +1,212 @@+package io.onedev.server.web.page.project.issues.imports;++import java.io.Serializable;+import java.util.concurrent.Callable;++import org.apache.wicket.Component;+import org.apache.wicket.RestartResponseException;+import org.apache.wicket.ajax.AjaxRequestTarget;+import org.apache.wicket.ajax.attributes.AjaxRequestAttributes;+import org.apache.wicket.ajax.markup.html.form.AjaxButton;+import org.apache.wicket.feedback.FencedFeedbackPanel;+import org.apache.wicket.markup.html.basic.Label;+import org.apache.wicket.markup.html.form.Form;+import org.apache.wicket.request.mapper.parameter.PageParameters;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import com.google.common.collect.Lists;++import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.model.Issue;+import io.onedev.server.model.Project;+import io.onedev.server.persistence.TransactionManager;+import io.onedev.server.search.entity.EntitySort;+import io.onedev.server.search.entity.EntitySort.Direction;+import io.onedev.server.search.entity.issue.IssueQuery;+import io.onedev.server.util.SimpleLogger;+import io.onedev.server.web.ajaxlistener.ShowGlobalAjaxIndicatorListener;+import io.onedev.server.web.component.taskbutton.TaskButton;+import io.onedev.server.web.editable.BeanContext;+import io.onedev.server.web.page.project.ProjectPage;+import io.onedev.server.web.page.project.issues.list.ProjectIssueListPage;++@SuppressWarnings("serial")+public class IssueImportPage<T extends Serializable, S extends Serializable> extends ProjectPage {++ private static final String PARAM_IMPORTER = "importer";++ private static final Logger logger = LoggerFactory.getLogger(IssueImportPage.class);++ private IssueImporter<T, S> importer;++ private T importSource;++ private S importOption;++ @SuppressWarnings("unchecked")+ public IssueImportPage(PageParameters params) {+ super(params);++ String importerName = params.get(PARAM_IMPORTER).toString();+ for (IssueImporterContribution contribution: OneDev.getExtensions(IssueImporterContribution.class)) {+ for (IssueImporter<? extends Serializable, ? extends Serializable> importer: contribution.getImporters()) {+ if (importer.getName().equals(importerName)) {+ this.importer = (IssueImporter<T, S>) importer;+ break;+ }+ }+ }++ if (importer == null)+ throw new RuntimeException("Undefined importer: " + importerName);+ }++ @Override+ protected void onInitialize() {+ super.onInitialize();++ try {+ importSource = importer.getSourceClass().newInstance();+ } catch (InstantiationException | IllegalAccessException e) {+ throw new RuntimeException(e);+ }++ Form<?> form = new Form<Void>("form");+ form.add(new FencedFeedbackPanel("feedback", form));+ form.add(BeanContext.edit("editor", importSource));++ form.add(new AjaxButton("next") {++ @Override+ protected void updateAjaxAttributes(AjaxRequestAttributes attributes) {+ super.updateAjaxAttributes(attributes);+ attributes.getAjaxCallListeners().add(new ShowGlobalAjaxIndicatorListener());+ }++ @Override+ protected void onSubmit(AjaxRequestTarget target, Form<?> form) {+ super.onSubmit(target, form);++ importOption = importer.getImportOption(importSource, new SimpleLogger() {++ @Override+ public void log(String message) {+ logger.info(message);+ }++ });++ form.replace(BeanContext.edit("editor", importOption));+ setVisible(false);++ form.get("import").setVisible(true);+ form.get("dryRun").setVisible(true);++ target.add(form);+ }++ @Override+ protected void onError(AjaxRequestTarget target, Form<?> form) {+ super.onError(target, form);+ target.add(form);+ }++ });++ Long projectId = getProject().getId();++ form.add(new TaskButton("import") {++ @Override+ protected void onCompleted(AjaxRequestTarget target) {+ super.onCompleted(target);++ EntitySort sort = new EntitySort();+ sort.setField(Issue.NAME_NUMBER);+ sort.setDirection(Direction.DESCENDING);+ IssueQuery query = new IssueQuery(null, Lists.newArrayList(sort));++ PageParameters params = ProjectIssueListPage.paramsOf(getProject(), query.toString(), 0);+ throw new RestartResponseException(ProjectIssueListPage.class, params);+ }++ @Override+ protected String runTask(SimpleLogger logger) {+ return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {++ @Override+ public String call() throws Exception {+ Project project = OneDev.getInstance(ProjectManager.class).load(projectId);+ return importer.doImport(project, importSource, importOption, false, logger);+ }++ });+ }++ @Override+ protected String getTitle() {+ return "Importing from " + importer.getName();+ }++ @Override+ protected void onError(AjaxRequestTarget target, Form<?> form) {+ super.onError(target, form);+ target.add(form);+ }++ }.setVisible(false));++ form.add(new TaskButton("dryRun") {++ @Override+ protected String runTask(SimpleLogger logger) {+ return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {++ @Override+ public String call() throws Exception {+ Project project = OneDev.getInstance(ProjectManager.class).load(projectId);+ return importer.doImport(project, importSource, importOption, true, logger);+ }++ });+ }++ @Override+ protected String getTitle() {+ return "Test importing from " + importer.getName();+ }++ @Override+ protected void onError(AjaxRequestTarget target, Form<?> form) {+ super.onError(target, form);+ target.add(form);+ }++ }.setVisible(false));++ form.setOutputMarkupId(true);+ add(form);+ }++ public static PageParameters paramsOf(Project project, String importer, String prevUrl) {+ PageParameters params = ProjectPage.paramsOf(project);+ params.add(PARAM_IMPORTER, importer);+ return params;+ }++ public T getImportSource() {+ return importSource;+ }++ public S getImportOption() {+ return importOption;+ }++ @Override+ protected Component newProjectTitle(String componentId) {+ return new Label(componentId, "Import Issues frmo " + importer.getName());+ }++}diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporter.java b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporter.javanew file mode 100644index 0000000000..98de01607b--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporter.java@@ -0,0 +1,42 @@+package io.onedev.server.web.page.project.issues.imports;++import java.io.Serializable;+import java.util.List;++import javax.annotation.Nullable;++import io.onedev.server.model.Project;+import io.onedev.server.util.ReflectionUtils;+import io.onedev.server.util.SimpleLogger;++public abstract class IssueImporter<T extends Serializable, S extends Serializable> implements Serializable {++ private static final long serialVersionUID = 1L;++ private final Class<T> sourceClass;++ private final Class<S> optionClass;++ @SuppressWarnings("unchecked")+ public IssueImporter() {+ List<Class<?>> typeArguments = ReflectionUtils.getTypeArguments(IssueImporter.class, getClass());+ sourceClass = (Class<T>) typeArguments.get(0);+ optionClass = (Class<S>) typeArguments.get(1);+ }++ public Class<T> getSourceClass() {+ return sourceClass;+ }++ public Class<S> getOptionClass() {+ return optionClass;+ }++ public abstract String getName();++ public abstract S getImportOption(T importSource, SimpleLogger logger);++ @Nullable+ public abstract String doImport(Project project, T importSource, S importOption, boolean dryRun, SimpleLogger logger);++}diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporterContribution.java b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporterContribution.javanew file mode 100644index 0000000000..45aa00bad3--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImporterContribution.java@@ -0,0 +1,13 @@+package io.onedev.server.web.page.project.issues.imports;++import java.io.Serializable;+import java.util.Collection;++import io.onedev.commons.launcher.loader.ExtensionPoint;++@ExtensionPoint+public interface IssueImporterContribution {++ Collection<IssueImporter<? extends Serializable, ? extends Serializable>> getImporters();++}diff --git a/server-core/src/test/java/io/onedev/server/util/ReferenceMigratorTest.java b/server-core/src/test/java/io/onedev/server/util/ReferenceMigratorTest.javanew file mode 100644index 0000000000..179199872b--- /dev/null+++ b/server-core/src/test/java/io/onedev/server/util/ReferenceMigratorTest.java@@ -0,0 +1,26 @@+package io.onedev.server.util;++import static org.junit.Assert.*;++import org.junit.Test;++import io.onedev.server.model.Issue;++public class ReferenceMigratorTest {++ @Test+ public void test() {+ ReferenceMigrator migrator = new ReferenceMigrator(+ Issue.class, CollectionUtils.newHashMap(1L, 11L, 22L, 2222L));+ assertEquals("issue #11", migrator.migratePrefixed("QB-1", "QB-"));+ assertEquals("issue #2222", migrator.migratePrefixed("QB-22", "QB-"));+ assertEquals("QB-1QB-22", migrator.migratePrefixed("QB-1QB-22", "QB-"));+ assertEquals("issue #11 issue #2222", migrator.migratePrefixed("QB-1 QB-22", "QB-"));+ assertEquals("issue #11 issue #2222", migrator.migratePrefixed("QB-1 QB-22", "QB-"));+ assertEquals("QB-2,issue #2222", migrator.migratePrefixed("QB-2,QB-22", "QB-"));+ assertEquals("QB-1andQB-2", migrator.migratePrefixed("QB-1andQB-2", "QB-"));+ assertEquals("QB-1以及QB-2", migrator.migratePrefixed("QB-1以及QB-2", "QB-"));+ assertEquals("2QB-1,issue #2222", migrator.migratePrefixed("2QB-1,QB-22", "QB-"));+ }++}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportResult.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportResult.javanew file mode 100644index 0000000000..bd5f191bfe--- /dev/null+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportResult.java@@ -0,0 +1,53 @@+package io.onedev.server.plugin.imports.github;++import java.util.HashSet;+import java.util.Set;++import org.unbescape.html.HtmlEscape;++public class GitHubImportResult {++ Set<String> nonExistentLogins = new HashSet<>();++ Set<String> unmappedIssueLabels = new HashSet<>();++ Set<String> nonExistentMilestones = new HashSet<>();++ boolean issuesImported = false;++ public String toHtml(String leadingText) {+ StringBuilder feedback = new StringBuilder(leadingText);++ boolean hasNotice = false;++ if (!nonExistentMilestones.isEmpty() || !unmappedIssueLabels.isEmpty()+ || !nonExistentLogins.isEmpty() || issuesImported) {+ hasNotice = true;+ }++ if (hasNotice)+ feedback.append("<br><br><b>NOTE:</b><ul>");++ if (!nonExistentMilestones.isEmpty()) {+ feedback.append("<li> Non existent milestones: "+ + HtmlEscape.escapeHtml5(nonExistentMilestones.toString()));+ }+ if (!unmappedIssueLabels.isEmpty()) {+ feedback.append("<li> GitHub issue labels not mapped to OneDev custom field: "+ + HtmlEscape.escapeHtml5(unmappedIssueLabels.toString()));+ }+ if (!nonExistentLogins.isEmpty()) {+ feedback.append("<li> GitHub logins without public email or public email can not be mapped to OneDev account: "+ + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));+ }++ if (issuesImported)+ feedback.append("<li> Attachments in issue description and comments are not imported due to GitHub limitation");++ if (hasNotice)+ feedback.append("</ul>");++ return feedback.toString();++ }+}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportUtils.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportUtils.javanew file mode 100644index 0000000000..8d4aea7fa6--- /dev/null+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportUtils.java@@ -0,0 +1,416 @@+package io.onedev.server.plugin.imports.github;++import java.net.URI;+import java.net.URISyntaxException;+import java.util.ArrayList;+import java.util.HashMap;+import java.util.HashSet;+import java.util.LinkedHashMap;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.atomic.AtomicInteger;++import javax.annotation.Nullable;+import javax.ws.rs.client.Client;+import javax.ws.rs.client.ClientBuilder;+import javax.ws.rs.client.Invocation;+import javax.ws.rs.client.WebTarget;+import javax.ws.rs.core.Response;++import org.apache.commons.lang3.StringUtils;+import org.apache.http.client.utils.URIBuilder;+import org.glassfish.jersey.client.ClientProperties;+import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;+import org.joda.time.format.ISODateTimeFormat;+import org.unbescape.html.HtmlEscape;++import com.fasterxml.jackson.databind.JsonNode;++import io.onedev.commons.utils.ExplicitException;+import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.IssueManager;+import io.onedev.server.entitymanager.SettingManager;+import io.onedev.server.entitymanager.UserManager;+import io.onedev.server.model.Issue;+import io.onedev.server.model.IssueComment;+import io.onedev.server.model.IssueField;+import io.onedev.server.model.Milestone;+import io.onedev.server.model.Project;+import io.onedev.server.model.User;+import io.onedev.server.model.support.LastUpdate;+import io.onedev.server.model.support.administration.GlobalIssueSetting;+import io.onedev.server.model.support.inputspec.InputSpec;+import io.onedev.server.model.support.issue.field.spec.FieldSpec;+import io.onedev.server.persistence.dao.Dao;+import io.onedev.server.util.Pair;+import io.onedev.server.util.ReferenceMigrator;+import io.onedev.server.util.SimpleLogger;++public class GitHubImportUtils {++ static final String NAME = "GitHub";++ static final int PER_PAGE = 100;++ static GitHubIssueImportOption buildImportOption(GitHubProjectImportSource importSource,+ @Nullable String gitHubRepo, SimpleLogger logger) {+ GitHubIssueImportOption importOption = new GitHubIssueImportOption();+ Client client = newClient(importSource);+ try {+ List<String> gitHubRepos = new ArrayList<>();+ if (gitHubRepo == null) {+ String apiEndpoint = importSource.getApiEndpoint("/user/repos");+ for (JsonNode repoNode: list(client, apiEndpoint, logger)) {+ String repoName = repoNode.get("name").asText();+ String ownerName = repoNode.get("owner").get("login").asText();+ gitHubRepos.add(ownerName + "/" + repoName);+ }+ } else {+ gitHubRepos.add(gitHubRepo);+ }++ Set<String> labels = new LinkedHashSet<>();+ for (String each: gitHubRepos) {+ String apiEndpoint = importSource.getApiEndpoint("/repos/" + each + "/labels");+ for (JsonNode labelNode: list(client, apiEndpoint, logger))+ labels.add(labelNode.get("name").asText());+ }++ for (String label: labels) {+ IssueLabelMapping mapping = new IssueLabelMapping();+ mapping.setGitHubIssueLabel(label);+ importOption.getIssueLabelMappings().add(mapping);+ }+ } finally {+ client.close();+ }+ return importOption;+ }++ @Nullable+ static User getUser(Client client, GitHubProjectImportSource importSource,+ Map<String, Optional<User>> users, String login, SimpleLogger logger) {+ Optional<User> userOpt = users.get(login);+ if (userOpt == null) {+ String apiEndpoint = importSource.getApiEndpoint("/users/" + login);+ String email = get(client, apiEndpoint, logger).get("email").asText(null);+ if (email != null)+ userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByEmail(email));+ else+ userOpt = Optional.empty();+ users.put(login, userOpt);+ }+ return userOpt.orElse(null);+ }++ static GitHubImportResult importIssues(GitHubProjectImportSource importSource, String gitHubRepo, Project oneDevProject,+ boolean useExistingIssueNumbers, GitHubIssueImportOption importOption, Map<String, Optional<User>> users,+ boolean dryRun, SimpleLogger logger) {+ Client client = newClient(importSource);+ try {+ Set<String> nonExistentMilestones = new HashSet<>();+ Set<String> nonExistentLogins = new HashSet<>();+ Set<String> unmappedIssueLabels = new HashSet<>();++ Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();+ Map<String, Milestone> milestoneMappings = new HashMap<>();++ for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);+ if (fieldSpec == null)+ throw new ExplicitException("No field spec found: " + oneDevFieldName);+ labelMappings.put(mapping.getGitHubIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));+ }++ for (Milestone milestone: oneDevProject.getMilestones())+ milestoneMappings.put(milestone.getName(), milestone);++ String initialIssueState = getIssueSetting().getInitialStateSpec().getName();++ List<Issue> issues = new ArrayList<>();++ Map<Long, Long> issueNumberMappings = new HashMap<>();++ AtomicInteger numOfImportedIssues = new AtomicInteger(0);+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {++ private String joinAsMultilineHtml(List<String> values) {+ List<String> escapedValues = new ArrayList<>();+ for (String value: values)+ escapedValues.add(HtmlEscape.escapeHtml5(value));+ return StringUtils.join(escapedValues, "<br>");+ }++ @Override+ public void consume(List<JsonNode> pageData) throws InterruptedException {+ for (JsonNode issueNode: pageData) {+ if (Thread.interrupted())+ throw new InterruptedException();++ Map<String, String> extraIssueInfo = new LinkedHashMap<>();++ Issue issue = new Issue();+ issue.setProject(oneDevProject);+ issue.setTitle(issueNode.get("title").asText());+ issue.setDescription(issueNode.get("body").asText(null));+ issue.setNumberScope(oneDevProject);++ Long oldNumber = issueNode.get("number").asLong();+ Long newNumber;+ if (dryRun || useExistingIssueNumbers)+ newNumber = oldNumber;+ else+ newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);+ issue.setNumber(newNumber);+ issueNumberMappings.put(oldNumber, newNumber);++ if (issueNode.get("state").asText().equals("closed"))+ issue.setState(importOption.getClosedIssueState());+ else+ issue.setState(initialIssueState);++ if (issueNode.hasNonNull("milestone")) {+ String milestoneName = issueNode.get("milestone").get("title").asText();+ Milestone milestone = milestoneMappings.get(milestoneName);+ if (milestone != null) {+ issue.setMilestone(milestone);+ } else {+ extraIssueInfo.put("Milestone", milestoneName);+ nonExistentMilestones.add(milestoneName);+ }+ }++ String login = issueNode.get("user").get("login").asText(null);+ User user = getUser(client, importSource, users, login, logger);+ if (user != null) {+ issue.setSubmitter(user);+ } else {+ issue.setSubmitterName(login);+ nonExistentLogins.add(login);+ }++ issue.setSubmitDate(ISODateTimeFormat.dateTimeNoMillis()+ .parseDateTime(issueNode.get("created_at").asText())+ .toDate());++ LastUpdate lastUpdate = new LastUpdate();+ lastUpdate.setActivity("Opened");+ lastUpdate.setDate(issue.getSubmitDate());+ lastUpdate.setUser(issue.getSubmitter());+ lastUpdate.setUserName(issue.getSubmitterName());+ issue.setLastUpdate(lastUpdate);++ for (JsonNode assigneeNode: issueNode.get("assignees")) {+ IssueField assigneeField = new IssueField();+ assigneeField.setIssue(issue);+ assigneeField.setName(importOption.getAssigneesIssueField());+ assigneeField.setType(InputSpec.USER);++ login = assigneeNode.get("login").asText();+ user = getUser(client, importSource, users, login, logger);+ if (user != null) {+ assigneeField.setValue(user.getName());+ } else {+ assigneeField.setValue(login);+ nonExistentLogins.add(login);+ }+ issue.getFields().add(assigneeField);+ }++ String apiEndpoint = importSource.getApiEndpoint("/repos/" + gitHubRepo+ + "/issues/" + oldNumber + "/comments");+ for (JsonNode commentNode: list(client, apiEndpoint, logger)) {+ IssueComment comment = new IssueComment();+ comment.setIssue(issue);+ comment.setContent(commentNode.get("body").asText(null));+ comment.setDate(ISODateTimeFormat.dateTimeNoMillis()+ .parseDateTime(commentNode.get("created_at").asText())+ .toDate());++ login = commentNode.get("user").get("login").asText();+ user = getUser(client, importSource, users, login, logger);+ if (user != null) {+ comment.setUser(user);+ } else {+ comment.setUserName(login);+ nonExistentLogins.add(login);+ }++ issue.getComments().add(comment);+ }++ issue.setCommentCount(issue.getComments().size());++ apiEndpoint = importSource.getApiEndpoint("/repos/" + gitHubRepo+ + "/issues/" + oldNumber + "/labels");+ List<String> currentUnmappedLabels = new ArrayList<>();+ for (JsonNode labelNode: list(client, apiEndpoint, logger)) {+ String labelName = labelNode.get("name").asText();+ Pair<FieldSpec, String> mapped = labelMappings.get(labelName);+ if (mapped != null) {+ IssueField tagField = new IssueField();+ tagField.setIssue(issue);+ tagField.setName(mapped.getFirst().getName());+ tagField.setType(InputSpec.ENUMERATION);+ tagField.setValue(mapped.getSecond());+ tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));+ issue.getFields().add(tagField);+ } else {+ currentUnmappedLabels.add(HtmlEscape.escapeHtml5(labelName));+ unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));+ }+ }++ if (!currentUnmappedLabels.isEmpty())+ extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));++ if (!extraIssueInfo.isEmpty()) {+ StringBuilder builder = new StringBuilder("|");+ for (String key: extraIssueInfo.keySet())+ builder.append(key).append("|");+ builder.append("\n|");+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));+ builder.append("\n|");+ for (String value: extraIssueInfo.values())+ builder.append(value).append("|");++ if (issue.getDescription() != null)+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());+ else+ issue.setDescription(builder.toString());+ }+ issues.add(issue);+ }+ logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");+ }++ };++ String apiEndpoint = importSource.getApiEndpoint("/repos/" + gitHubRepo + "/issues?state=all");+ list(client, apiEndpoint, pageDataConsumer, logger);++ if (!dryRun) {+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);+ for (Issue issue: issues) {+ if (issue.getDescription() != null)+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));++ OneDev.getInstance(IssueManager.class).save(issue);+ for (IssueField field: issue.getFields())+ OneDev.getInstance(Dao.class).persist(field);+ for (IssueComment comment: issue.getComments()) {+ comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));+ OneDev.getInstance(Dao.class).persist(comment);+ }+ }+ }++ GitHubImportResult result = new GitHubImportResult();+ result.nonExistentLogins.addAll(nonExistentLogins);+ result.nonExistentMilestones.addAll(nonExistentMilestones);+ result.unmappedIssueLabels.addAll(unmappedIssueLabels);++ if (numOfImportedIssues.get() != 0)+ result.issuesImported = true;++ return result;+ } finally {+ client.close();+ }+ }++ static GlobalIssueSetting getIssueSetting() {+ return OneDev.getInstance(SettingManager.class).getIssueSetting();+ }++ static List<JsonNode> list(Client client, String apiEndpoint, SimpleLogger logger) {+ List<JsonNode> result = new ArrayList<>();+ list(client, apiEndpoint, new PageDataConsumer() {++ @Override+ public void consume(List<JsonNode> pageData) {+ result.addAll(pageData);+ }++ }, logger);+ return result;+ }++ static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,+ SimpleLogger logger) {+ URI uri;+ try {+ uri = new URIBuilder(apiEndpoint)+ .addParameter("per_page", String.valueOf(PER_PAGE)).build();+ } catch (URISyntaxException e) {+ throw new RuntimeException(e);+ }++ int page = 1;+ while (true) {+ try {+ URIBuilder builder = new URIBuilder(uri);+ builder.addParameter("page", String.valueOf(page));+ List<JsonNode> pageData = new ArrayList<>();+ for (JsonNode each: get(client, builder.build().toString(), logger))+ pageData.add(each);+ pageDataConsumer.consume(pageData);+ if (pageData.size() < PER_PAGE)+ break;+ page++;+ } catch (URISyntaxException|InterruptedException e) {+ throw new RuntimeException(e);+ }+ }+ }++ static JsonNode get(Client client, String apiEndpoint, SimpleLogger logger) {+ WebTarget target = client.target(apiEndpoint);+ Invocation.Builder builder = target.request();+ while (true) {+ try (Response response = builder.get()) {+ int status = response.getStatus();+ if (status != 200) {+ String errorMessage = response.readEntity(String.class);+ if (StringUtils.isNotBlank(errorMessage)) {+ if (errorMessage.contains("rate limit exceeded")) {+ long resetTime = Long.parseLong(response.getHeaderString("x-ratelimit-reset"))*1000L;+ logger.log("Rate limit exceeded, wait until reset...");+ try {+ Thread.sleep(resetTime + 60*1000L - System.currentTimeMillis());+ continue;+ } catch (InterruptedException e) {+ throw new RuntimeException(e);+ }+ } else {+ throw new ExplicitException(String.format("Http request failed (url: %s, status code: %d, error message: %s)",+ apiEndpoint, status, errorMessage));+ }+ } else {+ throw new ExplicitException(String.format("Http request failed (status: %s)", status));+ }+ }+ return response.readEntity(JsonNode.class);+ }+ }+ }++ static Client newClient(GitHubProjectImportSource importSource) {+ Client client = ClientBuilder.newClient();+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);+ client.register(HttpAuthenticationFeature.basic("git", importSource.getAccessToken()));+ return client;+ }++ static interface PageDataConsumer {++ void consume(List<JsonNode> pageData) throws InterruptedException;++ }++}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.javadeleted file mode 100644index d265f323c9..0000000000--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.java+++ /dev/null@@ -1,517 +0,0 @@-package io.onedev.server.plugin.imports.github;--import java.net.URI;-import java.net.URISyntaxException;-import java.util.ArrayList;-import java.util.Collection;-import java.util.HashMap;-import java.util.HashSet;-import java.util.LinkedHashMap;-import java.util.LinkedHashSet;-import java.util.List;-import java.util.Map;-import java.util.Optional;-import java.util.Set;-import java.util.concurrent.atomic.AtomicInteger;--import javax.annotation.Nullable;-import javax.ws.rs.client.Client;-import javax.ws.rs.client.ClientBuilder;-import javax.ws.rs.client.Invocation;-import javax.ws.rs.client.WebTarget;-import javax.ws.rs.core.Response;--import org.apache.commons.lang3.StringUtils;-import org.apache.http.client.utils.URIBuilder;-import org.glassfish.jersey.client.ClientProperties;-import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;-import org.joda.time.format.ISODateTimeFormat;-import org.unbescape.html.HtmlEscape;--import com.fasterxml.jackson.databind.JsonNode;--import io.onedev.commons.utils.ExplicitException;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.IssueManager;-import io.onedev.server.entitymanager.MilestoneManager;-import io.onedev.server.entitymanager.ProjectManager;-import io.onedev.server.entitymanager.SettingManager;-import io.onedev.server.entitymanager.UserManager;-import io.onedev.server.model.Issue;-import io.onedev.server.model.IssueComment;-import io.onedev.server.model.IssueField;-import io.onedev.server.model.Milestone;-import io.onedev.server.model.Project;-import io.onedev.server.model.User;-import io.onedev.server.model.support.LastUpdate;-import io.onedev.server.model.support.administration.GlobalIssueSetting;-import io.onedev.server.model.support.inputspec.InputSpec;-import io.onedev.server.model.support.issue.field.spec.FieldSpec;-import io.onedev.server.persistence.dao.Dao;-import io.onedev.server.storage.StorageManager;-import io.onedev.server.util.Pair;-import io.onedev.server.util.SimpleLogger;-import io.onedev.server.web.page.project.imports.ProjectImporter;--public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImportOption> {-- private static final long serialVersionUID = 1L;-- public static final String NAME = "GitHub";-- private static final int PER_PAGE = 100;-- @Override- public String getName() {- return NAME;- }-- @Override- public GitHubImportOption getImportOption(GitHubImportSource importSource, SimpleLogger logger) {- GitHubImportOption importOption = new GitHubImportOption();- if (importSource.isPrepopulateImportOptions()) {- Client client = newClient(importSource);- try {- Set<String> labels = new LinkedHashSet<>();- for (JsonNode repoNode: list(client, importSource, "/user/repos", logger)) {- String repoName = repoNode.get("name").asText();- String ownerName = repoNode.get("owner").get("login").asText();- GitHubImport aImport = new GitHubImport();- aImport.setGitHubRepo(ownerName + "/" + repoName);- aImport.setOneDevProject(ownerName + "-" + repoName);- importOption.getImports().add(aImport);-- for (JsonNode labelNode: list(client, importSource,- "/repos/" + aImport.getGitHubRepo() + "/labels", logger)) {- labels.add(labelNode.get("name").asText());- }- }- for (String label: labels) {- IssueLabelMapping mapping = new IssueLabelMapping();- mapping.setGitHubIssueLabel(label);- importOption.getIssueLabelMappings().add(mapping);- }- } finally {- client.close();- }- }- return importOption;- }-- @Nullable- private User getUser(Client client, GitHubImportSource importSource,- Map<String, Optional<User>> users, String login, SimpleLogger logger) {- Optional<User> userOpt = users.get(login);- if (userOpt == null) {- String email = get(client, importSource, "/users/" + login, logger).get("email").asText(null);- if (email != null)- userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByEmail(email));- else- userOpt = Optional.empty();- users.put(login, userOpt);- }- return userOpt.orElse(null);- }-- @Override- public String doImport(GitHubImportSource importSource, GitHubImportOption importOption,- boolean dryRun, SimpleLogger logger) {- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);- StorageManager storageManager = OneDev.getInstance(StorageManager.class);- Client client = newClient(importSource);-- Set<String> nonExistentLogins = new HashSet<>();- Set<String> unmappedIssueLabels = new HashSet<>();-- Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();-- for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);- if (fieldSpec == null)- throw new ExplicitException("No field spec found: " + oneDevFieldName);- labelMappings.put(mapping.getGitHubIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));- }-- Collection<Long> projectIds = new ArrayList<>();- try {- Map<String, Optional<User>> users = new HashMap<>();- boolean importIssues = false;- for (GitHubImport aImport: importOption.getImports()) {- logger.log("Cloning code of repository " + aImport.getGitHubRepo() + "...");- JsonNode repoNode = get(client, importSource, "/repos/" + aImport.getGitHubRepo(), logger);- Project project = new Project();- project.setName(aImport.getOneDevProject());- project.setDescription(repoNode.get("description").asText(null));- project.setIssueManagementEnabled(repoNode.get("has_issues").asBoolean());- boolean isPrivate = repoNode.get("private").asBoolean();- if (!isPrivate && importOption.getPublicRole() != null)- project.setDefaultRole(importOption.getPublicRole());-- URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());- if (isPrivate)- builder.setUserInfo("git", importSource.getAccessToken());-- if (!dryRun) {- projectManager.clone(project, builder.build().toString());- projectIds.add(project.getId());- }-- if (aImport.isImportIssues()) {- importIssues = true;- Map<Long, Milestone> milestones = new HashMap<>();- logger.log("Importing milestones of repository " + aImport.getGitHubRepo() + "...");- String apiEndpoint = "/repos/" + aImport.getGitHubRepo() + "/milestones?state=all";- for (JsonNode milestoneNode: list(client, importSource, apiEndpoint, logger)) {- Milestone milestone = new Milestone();- milestone.setName(milestoneNode.get("title").asText());- milestone.setDescription(milestoneNode.get("description").asText(null));- milestone.setProject(project);- String dueDateString = milestoneNode.get("due_on").asText(null);- if (dueDateString != null)- milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());- if (milestoneNode.get("state").asText().equals("closed"))- milestone.setClosed(true);-- if (!dryRun)- OneDev.getInstance(MilestoneManager.class).save(milestone);- milestones.put(milestoneNode.get("number").asLong(), milestone);- }-- logger.log("Importing issues of repository " + aImport.getGitHubRepo() + "...");- String initialIssueState = getIssueSetting().getInitialStateSpec().getName();-- AtomicInteger numOfImportedIssues = new AtomicInteger(0);- PageDataConsumer pageDataConsumer = new PageDataConsumer() {-- /*- @Nullable- private String processAttachments(String issueUUID, Long issueNumber, @Nullable String markdown,- Set<String> tooLargeAttachments) {- if (markdown == null)- return markdown;-- Map<String, String> attachments = new HashMap<>();-- Node node = OneDev.getInstance(MarkdownManager.class).parse(markdown);- new NodeVisitor(new VisitHandler<?>[] {}) {-- @Override- public void visit(Node node) {- super.visit(node);- if (node instanceof InlineLinkNode) {- InlineLinkNode link = (InlineLinkNode)node;- String url = link.getUrl().toString();- if (url.startsWith(repoUrl + "/files/")) {- attachments.put(url, link.getText().toString());- }- }- }-- }.visit(node);-- for (Map.Entry<String, String> entry: attachments.entrySet()) {- String attachmentUrl = entry.getKey();- String attachmentName = entry.getValue();-- WebTarget target = client.target(attachmentUrl);- Invocation.Builder builder = target.request();- try (Response response = builder.get()) {- String errorMessage = JerseyUtils.checkStatus(response);- if (errorMessage != null) {- throw new ExplicitException(String.format(- "Error downloading attachment (url: %s, error message: %s)",- attachmentUrl, errorMessage));- }- try (InputStream is = response.readEntity(InputStream.class)) {- String oneDevAttachmentName = project.saveAttachment(issueUUID, attachmentName, is);- String oneDevAttachmentUrl = project.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);- markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");- } catch (AttachmentTooLargeException e) {- tooLargeAttachments.add(aImport.getGitHubRepo() + "#" + issueNumber + ":" + attachmentName);- } catch (IOException e) {- throw new RuntimeException(e);- }- }- }-- return markdown;- }- */-- private String joinAsMultilineHtml(List<String> values) {- List<String> escapedValues = new ArrayList<>();- for (String value: values)- escapedValues.add(HtmlEscape.escapeHtml5(value));- return StringUtils.join(escapedValues, "<br>");- }-- @Override- public void consume(List<JsonNode> pageData) throws InterruptedException {- for (JsonNode issueNode: pageData) {- if (Thread.interrupted())- throw new InterruptedException();- Issue issue = new Issue();- issue.setProject(project);- issue.setTitle(issueNode.get("title").asText());- issue.setDescription(issueNode.get("body").asText(null));- issue.setNumberScope(project);- issue.setNumber(issueNode.get("number").asLong());- if (issueNode.get("state").asText().equals("closed"))- issue.setState(importOption.getClosedIssueState());- else- issue.setState(initialIssueState);- if (issueNode.hasNonNull("milestone")) {- long milestoneNumber = issueNode.get("milestone").get("number").asLong();- Milestone milestone = milestones.get(milestoneNumber);- if (milestone != null)- issue.setMilestone(milestone);- }- String login = issueNode.get("user").get("login").asText(null);- User user = getUser(client, importSource, users, login, logger);- if (user != null) {- issue.setSubmitter(user);- } else {- issue.setSubmitterName(login);- nonExistentLogins.add(login);- }-- issue.setSubmitDate(ISODateTimeFormat.dateTimeNoMillis()- .parseDateTime(issueNode.get("created_at").asText())- .toDate());-- LastUpdate lastUpdate = new LastUpdate();- lastUpdate.setActivity("Opened");- lastUpdate.setDate(issue.getSubmitDate());- lastUpdate.setUser(issue.getSubmitter());- lastUpdate.setUserName(issue.getSubmitterName());- issue.setLastUpdate(lastUpdate);-- Map<String, String> extraIssueInfo = new LinkedHashMap<>();-- for (JsonNode assigneeNode: issueNode.get("assignees")) {- IssueField assigneeField = new IssueField();- assigneeField.setIssue(issue);- assigneeField.setName(importOption.getAssigneesIssueField());- assigneeField.setType(InputSpec.USER);-- login = assigneeNode.get("login").asText();- user = getUser(client, importSource, users, login, logger);- if (user != null) {- assigneeField.setValue(user.getName());- } else {- assigneeField.setValue(login);- nonExistentLogins.add(login);- }- issue.getFields().add(assigneeField);- }-- String apiEndpoint = "/repos/" + aImport.getGitHubRepo()- + "/issues/" + issue.getNumber() + "/comments";- for (JsonNode commentNode: list(client, importSource, apiEndpoint, logger)) {- IssueComment comment = new IssueComment();- comment.setIssue(issue);- comment.setContent(commentNode.get("body").asText(null));- comment.setDate(ISODateTimeFormat.dateTimeNoMillis()- .parseDateTime(commentNode.get("created_at").asText())- .toDate());-- login = commentNode.get("user").get("login").asText();- user = getUser(client, importSource, users, login, logger);- if (user != null) {- comment.setUser(user);- } else {- comment.setUserName(login);- nonExistentLogins.add(login);- }-- issue.getComments().add(comment);- }-- issue.setCommentCount(issue.getComments().size());-- apiEndpoint = "/repos/" + aImport.getGitHubRepo()- + "/issues/" + issue.getNumber() + "/labels";- List<String> currentUnmappedLabels = new ArrayList<>();- for (JsonNode labelNode: list(client, importSource, apiEndpoint, logger)) {- String labelName = labelNode.get("name").asText();- Pair<FieldSpec, String> mapped = labelMappings.get(labelName);- if (mapped != null) {- IssueField tagField = new IssueField();- tagField.setIssue(issue);- tagField.setName(mapped.getFirst().getName());- tagField.setType(InputSpec.ENUMERATION);- tagField.setValue(mapped.getSecond());- tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));- issue.getFields().add(tagField);- } else {- currentUnmappedLabels.add(HtmlEscape.escapeHtml5(labelName));- unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));- }- }-- if (!currentUnmappedLabels.isEmpty())- extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));-- if (!extraIssueInfo.isEmpty()) {- StringBuilder builder = new StringBuilder("|");- for (String key: extraIssueInfo.keySet())- builder.append(key).append("|");- builder.append("\n|");- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));- builder.append("\n|");- for (String value: extraIssueInfo.values())- builder.append(value).append("|");-- if (issue.getDescription() != null)- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());- else- issue.setDescription(builder.toString());- }-- if (!dryRun) {- OneDev.getInstance(IssueManager.class).save(issue);- for (IssueField field: issue.getFields())- OneDev.getInstance(Dao.class).persist(field);- for (IssueComment comment: issue.getComments())- OneDev.getInstance(Dao.class).persist(comment);- }-- if (numOfImportedIssues.incrementAndGet() % 25 == 0)- logger.log("Imported " + numOfImportedIssues.get() + " issues");- }- }-- };- apiEndpoint = "/repos/" + aImport.getGitHubRepo() + "/issues?state=all";- list(client, importSource, apiEndpoint, pageDataConsumer, logger);- }- }-- StringBuilder feedback = new StringBuilder("Repositories imported successfully");-- boolean hasNotes = !unmappedIssueLabels.isEmpty() || !nonExistentLogins.isEmpty() || importIssues;-- if (hasNotes)- feedback.append("<br><br><b>NOTE:</b><ul>");-- if (!unmappedIssueLabels.isEmpty()) {- feedback.append("<li> GitHub issue labels not mapped to OneDev custom field (mentioned as extra info in issue description): "- + HtmlEscape.escapeHtml5(unmappedIssueLabels.toString()));- }- if (!nonExistentLogins.isEmpty()) {- feedback.append("<li> GitHub logins without public email or public email can not be mapped to OneDev account: "- + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));- }- if (importIssues)- feedback.append("<li> Attachments in issues and comments are not imported due to GitHub limitation");-- if (hasNotes)- feedback.append("</ul>");-- return feedback.toString();- } catch (Exception e) {- for (Long projectId: projectIds)- storageManager.deleteProjectDir(projectId);- throw new RuntimeException(e);- } finally {- client.close();- }- }-- private GlobalIssueSetting getIssueSetting() {- return OneDev.getInstance(SettingManager.class).getIssueSetting();- }-- private List<JsonNode> list(Client client, GitHubImportSource importSource,- String apiPath, SimpleLogger logger) {- List<JsonNode> result = new ArrayList<>();- list(client, importSource, apiPath, new PageDataConsumer() {-- @Override- public void consume(List<JsonNode> pageData) {- result.addAll(pageData);- }-- }, logger);- return result;- }-- private void list(Client client, GitHubImportSource importSource, String apiPath,- PageDataConsumer pageDataConsumer, SimpleLogger logger) {- URI uri;- try {- uri = new URIBuilder(importSource.getApiEndpoint(apiPath))- .addParameter("per_page", String.valueOf(PER_PAGE)).build();- } catch (URISyntaxException e) {- throw new RuntimeException(e);- }-- int page = 1;- while (true) {- try {- URIBuilder builder = new URIBuilder(uri);- builder.addParameter("page", String.valueOf(page));- List<JsonNode> pageData = new ArrayList<>();- for (JsonNode each: get(client, builder.build().toString(), logger))- pageData.add(each);- pageDataConsumer.consume(pageData);- if (pageData.size() < PER_PAGE)- break;- page++;- } catch (URISyntaxException|InterruptedException e) {- throw new RuntimeException(e);- }- }- }-- private JsonNode get(Client client, GitHubImportSource importSource, String apiPath, SimpleLogger logger) {- return get(client, importSource.getApiEndpoint(apiPath), logger);- }-- private JsonNode get(Client client, String apiEndpoint, SimpleLogger logger) {- WebTarget target = client.target(apiEndpoint);- Invocation.Builder builder = target.request();- while (true) {- try (Response response = builder.get()) {- int status = response.getStatus();- if (status != 200) {- String errorMessage = response.readEntity(String.class);- if (StringUtils.isNotBlank(errorMessage)) {- if (errorMessage.contains("rate limit exceeded")) {- long resetTime = Long.parseLong(response.getHeaderString("x-ratelimit-reset"))*1000L;- logger.log("Rate limit exceeded, wait until reset...");- try {- Thread.sleep(resetTime + 60*1000L - System.currentTimeMillis());- continue;- } catch (InterruptedException e) {- throw new RuntimeException(e);- }- } else {- throw new ExplicitException(String.format("Http request failed (status code: %d, error message: %s)",- status, errorMessage));- }- } else {- throw new ExplicitException(String.format("Http request failed (status: %s)", status));- }- }- return response.readEntity(JsonNode.class);- }- }- }-- private Client newClient(GitHubImportSource importSource) {- Client client = ClientBuilder.newClient();- client.property(ClientProperties.FOLLOW_REDIRECTS, true);- client.register(HttpAuthenticationFeature.basic("git", importSource.getAccessToken()));- return client;- }-- private static interface PageDataConsumer {-- void consume(List<JsonNode> pageData) throws InterruptedException;-- }--}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportOption.javasimilarity index 51%rename from server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.javarename to server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportOption.javaindex 134c367ad0..255039723c 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportOption.java@@ -5,72 +5,27 @@ import java.util.ArrayList;import java.util.List;import java.util.stream.Collectors;-import javax.annotation.Nullable;-import javax.validation.ConstraintValidatorContext;-import javax.validation.constraints.Size;-import org.hibernate.validator.constraints.NotEmpty;import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.ProjectManager;-import io.onedev.server.entitymanager.RoleManager;import io.onedev.server.entitymanager.SettingManager;-import io.onedev.server.model.Role;import io.onedev.server.model.support.administration.GlobalIssueSetting;import io.onedev.server.model.support.inputspec.InputSpec;import io.onedev.server.model.support.issue.field.spec.FieldSpec;-import io.onedev.server.util.validation.Validatable;-import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.ChoiceProvider;import io.onedev.server.web.editable.annotation.Editable;-import io.onedev.server.web.editable.annotation.RoleChoice;@Editable-@ClassValidating-public class GitHubImportOption implements Serializable, Validatable {+public class GitHubIssueImportOption implements Serializable {private static final long serialVersionUID = 1L;- private List<GitHubImport> imports = new ArrayList<>();-- private String publicRoleName;-private String closedIssueState;private String assigneesIssueField;private List<IssueLabelMapping> issueLabelMappings = new ArrayList<>();- @Editable(order=100, name="Repositories to Import")- @Size(min=1, max=10000, message="No repositories to import")- public List<GitHubImport> getImports() {- return imports;- }-- public void setImports(List<GitHubImport> imports) {- this.imports = imports;- }-- @Editable(order=200, name="Public Role", description="If specified, all public repositories imported from GitHub "- + "will use this as default role. Private repositories are not affected")- @RoleChoice- @Nullable- public String getPublicRoleName() {- return publicRoleName;- }-- public void setPublicRoleName(String publicRoleName) {- this.publicRoleName = publicRoleName;- }-- @Nullable- public Role getPublicRole() {- if (publicRoleName != null)- return OneDev.getInstance(RoleManager.class).find(publicRoleName);- else- return null;- }-@Editable(order=300, description="Specify which issue state to use for closed GitHub issues")@ChoiceProvider("getCloseStateChoices")@NotEmpty@@ -127,34 +82,4 @@ public class GitHubImportOption implements Serializable, Validatable {this.issueLabelMappings = issueLabelMappings;}- @Override- public boolean isValid(ConstraintValidatorContext context) {- boolean isValid = true;- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);- for (int i=0; i<imports.size(); i++) {- if (projectManager.find(imports.get(i).getOneDevProject()) != null) {- context.buildConstraintViolationWithTemplate("Project name already used")- .addPropertyNode("imports")- .addPropertyNode(GitHubImport.PROP_ONEDEV_PROJECT)- .inIterable().atIndex(i).addConstraintViolation();- isValid = false;- } else {- for (int j=0; j<imports.size(); j++) {- if (j != i && imports.get(j).getOneDevProject().equals(imports.get(i).getOneDevProject())) {- context.buildConstraintViolationWithTemplate("Duplicate project name")- .addPropertyNode("imports")- .addPropertyNode(GitHubImport.PROP_ONEDEV_PROJECT)- .inIterable().atIndex(i).addConstraintViolation();- isValid = false;- break;- }- }- }- }-- if (!isValid)- context.disableDefaultConstraintViolation();- return isValid;- }-}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportSource.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportSource.javanew file mode 100644index 0000000000..aec0a615d1--- /dev/null+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImportSource.java@@ -0,0 +1,88 @@+package io.onedev.server.plugin.imports.github;++import static io.onedev.server.plugin.imports.github.GitHubImportUtils.get;+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.list;++import java.util.ArrayList;+import java.util.Collections;+import java.util.List;++import javax.ws.rs.client.Client;+import javax.ws.rs.client.ClientBuilder;++import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;+import org.hibernate.validator.constraints.NotEmpty;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import com.fasterxml.jackson.databind.JsonNode;++import io.onedev.commons.utils.StringUtils;+import io.onedev.server.util.EditContext;+import io.onedev.server.util.SimpleLogger;+import io.onedev.server.util.validation.annotation.ClassValidating;+import io.onedev.server.web.editable.annotation.ChoiceProvider;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+@ClassValidating+public class GitHubIssueImportSource extends GitHubProjectImportSource {++ private static final long serialVersionUID = 1L;++ private static final Logger logger = LoggerFactory.getLogger(GitHubIssueImportSource.class);++ private String repository;++ @Editable(order=400, name="GitHub Repository", description="Choose GitHub repository to "+ + "import issues from. Available repositories will be populated when above options "+ + "are specified")+ @ChoiceProvider("getRepositoryChoices")+ @NotEmpty+ public String getRepository() {+ return repository;+ }++ public void setRepository(String repository) {+ this.repository = repository;+ }++ @SuppressWarnings("unused")+ private static List<String> getRepositoryChoices() {+ List<String> choices = new ArrayList<>();++ String apiUrl = (String) EditContext.get().getInputValue(PROP_API_URL);+ String accessToken = (String) EditContext.get().getInputValue(PROP_ACCESS_TOKEN);++ if (StringUtils.isNotBlank(apiUrl) && StringUtils.isNotBlank(accessToken)) {+ Client client = ClientBuilder.newClient();+ client.register(HttpAuthenticationFeature.basic("git", accessToken));+ try {+ String apiEndpoint = getApiEndpoint(apiUrl, "/user");+ SimpleLogger logger = new SimpleLogger() {++ @Override+ public void log(String message) {+ GitHubIssueImportSource.logger.info(message);+ }++ };+ get(client, apiEndpoint, logger);++ apiEndpoint = getApiEndpoint(apiUrl, "/user/repos");+ for (JsonNode repoNode: list(client, apiEndpoint, logger)) {+ String repoName = repoNode.get("name").asText();+ String ownerName = repoNode.get("owner").get("login").asText();+ choices.add(ownerName + "/" + repoName);+ }+ } catch (Exception e) {+ } finally {+ client.close();+ }+ }++ Collections.sort(choices);+ return choices;+ }++}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.javanew file mode 100644index 0000000000..58de5a7bc9--- /dev/null+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java@@ -0,0 +1,44 @@+package io.onedev.server.plugin.imports.github;++import static io.onedev.server.plugin.imports.github.GitHubImportUtils.NAME;+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.buildImportOption;+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.importIssues;++import java.util.HashMap;+import java.util.Map;+import java.util.Optional;++import io.onedev.server.model.Project;+import io.onedev.server.model.User;+import io.onedev.server.util.SimpleLogger;+import io.onedev.server.web.page.project.issues.imports.IssueImporter;++public class GitHubIssueImporter extends IssueImporter<GitHubIssueImportSource, GitHubIssueImportOption> {++ private static final long serialVersionUID = 1L;++ @Override+ public String getName() {+ return NAME;+ }++ @Override+ public GitHubIssueImportOption getImportOption(GitHubIssueImportSource importSource, SimpleLogger logger) {+ GitHubIssueImportOption importOption;+ if (importSource.isPrepopulateImportOptions())+ importOption = buildImportOption(importSource, importSource.getRepository(), logger);+ else+ importOption = new GitHubIssueImportOption();+ return importOption;+ }++ @Override+ public String doImport(Project project, GitHubIssueImportSource importSource, GitHubIssueImportOption importOption,+ boolean dryRun, SimpleLogger logger) {+ logger.log("Importing issues from repository " + importSource.getRepository() + "...");+ Map<String, Optional<User>> users = new HashMap<>();+ return importIssues(importSource, importSource.getRepository(), project, false, importOption, users, dryRun, logger)+ .toHtml("Issues imported successfully");+ }++}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.javaindex 5330335a2d..21be1145de 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java@@ -8,6 +8,8 @@ import com.google.common.collect.Lists;import io.onedev.commons.launcher.loader.AbstractPluginModule;import io.onedev.server.web.page.project.imports.ProjectImporter;import io.onedev.server.web.page.project.imports.ProjectImporterContribution;+import io.onedev.server.web.page.project.issues.imports.IssueImporter;+import io.onedev.server.web.page.project.issues.imports.IssueImporterContribution;/*** NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.@@ -23,11 +25,19 @@ public class GitHubPluginModule extends AbstractPluginModule {@Overridepublic Collection<ProjectImporter<? extends Serializable, ? extends Serializable>> getImporters() {- return Lists.newArrayList(new GitHubImporter());+ return Lists.newArrayList(new GitHubProjectImporter());}});+ contribute(IssueImporterContribution.class, new IssueImporterContribution() {++ @Override+ public Collection<IssueImporter<? extends Serializable, ? extends Serializable>> getImporters() {+ return Lists.newArrayList(new GitHubIssueImporter());+ }++ });}}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportOption.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportOption.javanew file mode 100644index 0000000000..d4e144a0aa--- /dev/null+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportOption.java@@ -0,0 +1,89 @@+package io.onedev.server.plugin.imports.github;++import java.util.ArrayList;+import java.util.List;++import javax.annotation.Nullable;+import javax.validation.ConstraintValidatorContext;+import javax.validation.constraints.Size;++import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.entitymanager.RoleManager;+import io.onedev.server.model.Role;+import io.onedev.server.util.validation.Validatable;+import io.onedev.server.util.validation.annotation.ClassValidating;+import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.RoleChoice;++@Editable+@ClassValidating+public class GitHubProjectImportOption extends GitHubIssueImportOption implements Validatable {++ private static final long serialVersionUID = 1L;++ private List<ProjectMapping> projectMappings = new ArrayList<>();++ private String publicRoleName;++ @Editable(order=200, name="Public Role", description="If specified, all public repositories imported from GitHub "+ + "will use this as default role. Private repositories are not affected")+ @RoleChoice+ @Nullable+ public String getPublicRoleName() {+ return publicRoleName;+ }++ public void setPublicRoleName(String publicRoleName) {+ this.publicRoleName = publicRoleName;+ }++ @Nullable+ public Role getPublicRole() {+ if (publicRoleName != null)+ return OneDev.getInstance(RoleManager.class).find(publicRoleName);+ else+ return null;+ }++ @Editable(order=100, name="Repositories to Import")+ @Size(min=1, max=10000, message="No repositories to import")+ public List<ProjectMapping> getProjectMappings() {+ return projectMappings;+ }++ public void setProjectMappings(List<ProjectMapping> projectMappings) {+ this.projectMappings = projectMappings;+ }++ @Override+ public boolean isValid(ConstraintValidatorContext context) {+ boolean isValid = true;+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);+ for (int i=0; i<projectMappings.size(); i++) {+ if (projectManager.find(projectMappings.get(i).getOneDevProject()) != null) {+ context.buildConstraintViolationWithTemplate("Project name already used")+ .addPropertyNode("imports")+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)+ .inIterable().atIndex(i).addConstraintViolation();+ isValid = false;+ } else {+ for (int j=0; j<projectMappings.size(); j++) {+ if (j != i && projectMappings.get(j).getOneDevProject().equals(projectMappings.get(i).getOneDevProject())) {+ context.buildConstraintViolationWithTemplate("Duplicate project name")+ .addPropertyNode("imports")+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)+ .inIterable().atIndex(i).addConstraintViolation();+ isValid = false;+ break;+ }+ }+ }+ }++ if (!isValid)+ context.disableDefaultConstraintViolation();+ return isValid;+ }++}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportSource.javasimilarity index 83%rename from server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.javarename to server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportSource.javaindex 3827aad627..1ca2ea3a80 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImportSource.java@@ -17,16 +17,17 @@ import io.onedev.server.util.JerseyUtils;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.Password;@Editable@ClassValidating-public class GitHubImportSource implements Serializable, Validatable {+public class GitHubProjectImportSource implements Serializable, Validatable {private static final long serialVersionUID = 1L;- private static final String PROP_API_URL = "apiUrl";+ static final String PROP_API_URL = "apiUrl";- private static final String PROP_ACCESS_TOKEN = "accessToken";+ static final String PROP_ACCESS_TOKEN = "accessToken";private String apiUrl = "https://api.github.com";@@ -46,6 +47,7 @@ public class GitHubImportSource implements Serializable, Validatable {@Editable(order=100, name="GitHub Personal Access Token", description="GitHub personal access token should be generated with "+ "scope <b>repo</b> and <b>read:org</b>")+ @Password@NotEmptypublic String getAccessToken() {return accessToken;@@ -55,7 +57,7 @@ public class GitHubImportSource implements Serializable, Validatable {this.accessToken = accessToken;}- @Editable(order=200, description="If checked, import options will be pre-populated based on all accessible "+ @Editable(order=1000, description="If checked, import options will be pre-populated based on all accessible "+ "repositories and its settings")public boolean isPrepopulateImportOptions() {return prepopulateImportOptions;@@ -65,16 +67,21 @@ public class GitHubImportSource implements Serializable, Validatable {this.prepopulateImportOptions = prepopulateImportOptions;}- public String getApiEndpoint(String apiPath) {+ public static String getApiEndpoint(String apiUrl, String apiPath) {return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");}++ public String getApiEndpoint(String apiPath) {+ return getApiEndpoint(apiUrl, apiPath);+ }@Overridepublic boolean isValid(ConstraintValidatorContext context) {Client client = ClientBuilder.newClient();client.register(HttpAuthenticationFeature.basic("git", getAccessToken()));try {- WebTarget target = client.target(getApiEndpoint("/user"));+ String apiEndpoint = getApiEndpoint("/user");+ WebTarget target = client.target(apiEndpoint);Invocation.Builder builder = target.request();try (Response response = builder.get()) {if (response.getStatus() == 401) {@@ -84,7 +91,7 @@ public class GitHubImportSource implements Serializable, Validatable {.addPropertyNode(PROP_ACCESS_TOKEN).addConstraintViolation();return false;} else {- String errorMessage = JerseyUtils.checkStatus(response);+ String errorMessage = JerseyUtils.checkStatus(apiEndpoint, response);if (errorMessage != null) {context.disableDefaultConstraintViolation();context.buildConstraintViolationWithTemplate(errorMessage)diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.javanew file mode 100644index 0000000000..7b543a2c5c--- /dev/null+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java@@ -0,0 +1,160 @@+package io.onedev.server.plugin.imports.github;++import static io.onedev.server.plugin.imports.github.GitHubImportUtils.NAME;+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.buildImportOption;+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.get;+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.importIssues;+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.list;+import static io.onedev.server.plugin.imports.github.GitHubImportUtils.newClient;++import java.util.ArrayList;+import java.util.Collection;+import java.util.HashMap;+import java.util.Map;+import java.util.Optional;++import javax.annotation.Nullable;+import javax.ws.rs.client.Client;++import org.apache.http.client.utils.URIBuilder;+import org.joda.time.format.ISODateTimeFormat;++import com.fasterxml.jackson.databind.JsonNode;++import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.MilestoneManager;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.entitymanager.UserManager;+import io.onedev.server.model.Milestone;+import io.onedev.server.model.Project;+import io.onedev.server.model.User;+import io.onedev.server.storage.StorageManager;+import io.onedev.server.util.SimpleLogger;+import io.onedev.server.web.page.project.imports.ProjectImporter;++public class GitHubProjectImporter extends ProjectImporter<GitHubProjectImportSource, GitHubProjectImportOption> {++ private static final long serialVersionUID = 1L;++ @Override+ public String getName() {+ return NAME;+ }++ @Override+ public GitHubProjectImportOption getImportOption(GitHubProjectImportSource importSource, SimpleLogger logger) {+ GitHubProjectImportOption importOption = new GitHubProjectImportOption();+ if (importSource.isPrepopulateImportOptions()) {+ Client client = newClient(importSource);+ try {+ String apiEndpoint = importSource.getApiEndpoint("/user/repos");+ for (JsonNode repoNode: list(client, apiEndpoint, logger)) {+ String repoName = repoNode.get("name").asText();+ String ownerName = repoNode.get("owner").get("login").asText();+ ProjectMapping projectMapping = new ProjectMapping();+ projectMapping.setGitHubRepo(ownerName + "/" + repoName);+ projectMapping.setOneDevProject(ownerName + "-" + repoName);+ importOption.getProjectMappings().add(projectMapping);+ }+ GitHubIssueImportOption issueImportOption = buildImportOption(importSource, null, logger);+ importOption.setAssigneesIssueField(issueImportOption.getAssigneesIssueField());+ importOption.setClosedIssueState(issueImportOption.getClosedIssueState());+ importOption.setIssueLabelMappings(issueImportOption.getIssueLabelMappings());+ } finally {+ client.close();+ }+ }+ return importOption;+ }++ @Nullable+ private User getUser(Client client, GitHubProjectImportSource importSource,+ Map<String, Optional<User>> users, String login, SimpleLogger logger) {+ Optional<User> userOpt = users.get(login);+ if (userOpt == null) {+ String apiEndpoint = importSource.getApiEndpoint("/users/" + login);+ String email = get(client, apiEndpoint, logger).get("email").asText(null);+ if (email != null)+ userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByEmail(email));+ else+ userOpt = Optional.empty();+ users.put(login, userOpt);+ }+ return userOpt.orElse(null);+ }++ @Override+ public String doImport(GitHubProjectImportSource importSource, GitHubProjectImportOption importOption,+ boolean dryRun, SimpleLogger logger) {+ Collection<Long> projectIds = new ArrayList<>();+ Client client = newClient(importSource);+ try {+ Map<String, Optional<User>> users = new HashMap<>();+ GitHubImportResult result = new GitHubImportResult();+ for (ProjectMapping projectMapping: importOption.getProjectMappings()) {+ logger.log("Cloning code from repository " + projectMapping.getGitHubRepo() + "...");++ String apiEndpoint = importSource.getApiEndpoint("/repos/" + projectMapping.getGitHubRepo());+ JsonNode repoNode = get(client, apiEndpoint, logger);+ Project project = new Project();+ project.setName(projectMapping.getOneDevProject());+ project.setDescription(repoNode.get("description").asText(null));+ project.setIssueManagementEnabled(repoNode.get("has_issues").asBoolean());++ boolean isPrivate = repoNode.get("private").asBoolean();+ if (!isPrivate && importOption.getPublicRole() != null)+ project.setDefaultRole(importOption.getPublicRole());++ URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());+ if (isPrivate)+ builder.setUserInfo("git", importSource.getAccessToken());++ if (!dryRun) {+ OneDev.getInstance(ProjectManager.class).clone(project, builder.build().toString());+ projectIds.add(project.getId());+ }++ if (projectMapping.isImportIssues()) {+ Map<Long, Milestone> milestones = new HashMap<>();+ logger.log("Importing milestones from repository " + projectMapping.getGitHubRepo() + "...");+ apiEndpoint = importSource.getApiEndpoint("/repos/" + projectMapping.getGitHubRepo() + "/milestones?state=all");+ for (JsonNode milestoneNode: list(client, apiEndpoint, logger)) {+ Milestone milestone = new Milestone();+ milestone.setName(milestoneNode.get("title").asText());+ milestone.setDescription(milestoneNode.get("description").asText(null));+ milestone.setProject(project);+ String dueDateString = milestoneNode.get("due_on").asText(null);+ if (dueDateString != null)+ milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());+ if (milestoneNode.get("state").asText().equals("closed"))+ milestone.setClosed(true);++ milestones.put(milestoneNode.get("number").asLong(), milestone);+ project.getMilestones().add(milestone);++ if (!dryRun)+ OneDev.getInstance(MilestoneManager.class).save(milestone);+ }++ logger.log("Importing issues from repository " + projectMapping.getGitHubRepo() + "...");+ GitHubImportResult currentResult = importIssues(importSource, projectMapping.getGitHubRepo(),+ project, true, importOption, users, dryRun, logger);+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);+ result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);+ result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);++ result.issuesImported |= currentResult.issuesImported;+ }+ }++ return result.toHtml("Repositories imported successfully");+ } catch (Exception e) {+ for (Long projectId: projectIds)+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);+ throw new RuntimeException(e);+ } finally {+ client.close();+ }+ }++}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImport.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectMapping.javasimilarity index 95%rename from server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImport.javarename to server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectMapping.javaindex f480d74647..cb2b4393f0 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImport.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectMapping.java@@ -8,7 +8,7 @@ import io.onedev.server.util.validation.annotation.ProjectName;import io.onedev.server.web.editable.annotation.Editable;@Editable-public class GitHubImport implements Serializable {+public class ProjectMapping implements Serializable {private static final long serialVersionUID = 1L;diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImport.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.javasimilarity index 95%rename from server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImport.javarename to server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.javaindex 1f07e30919..9f302f2399 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImport.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.java@@ -8,7 +8,7 @@ import io.onedev.server.util.validation.annotation.ProjectName;import io.onedev.server.web.editable.annotation.Editable;@Editable-public class YouTrackImport implements Serializable {+public class ProjectMapping implements Serializable {private static final long serialVersionUID = 1L;diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportResult.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportResult.javanew file mode 100644index 0000000000..872833c197--- /dev/null+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportResult.java@@ -0,0 +1,76 @@+package io.onedev.server.plugin.imports.youtrack;++import java.util.LinkedHashMap;+import java.util.LinkedHashSet;+import java.util.Map;+import java.util.Set;++import org.unbescape.html.HtmlEscape;++public class YouTrackImportResult {++ Set<String> nonExistentLogins = new LinkedHashSet<>();++ Set<String> unmappedIssueTags = new LinkedHashSet<>();++ Set<String> unmappedIssueFields = new LinkedHashSet<>();++ Set<String> unmappedIssueStates = new LinkedHashSet<>();++ Map<String, String> mismatchedIssueFields = new LinkedHashMap<>();++ Set<String> tooLargeAttachments = new LinkedHashSet<>();++ public String toHtml(String leadingText) {+ StringBuilder feedback = new StringBuilder(HtmlEscape.escapeHtml5(leadingText));++ boolean hasNotes =+ !unmappedIssueStates.isEmpty()+ || !unmappedIssueFields.isEmpty()+ || !mismatchedIssueFields.isEmpty()+ || !unmappedIssueTags.isEmpty()+ || !nonExistentLogins.isEmpty()+ || !tooLargeAttachments.isEmpty();++ if (hasNotes)+ feedback.append("<br><br><b>NOTE:</b><ul>");++ if (!unmappedIssueStates.isEmpty()) {+ feedback.append("<li> Unmapped YouTrack issue states (using OneDev initial state): "+ + HtmlEscape.escapeHtml5(unmappedIssueStates.toString()));+ }+ if (!unmappedIssueFields.isEmpty()) {+ feedback.append("<li> Unmapped YouTrack issue fields: "+ + HtmlEscape.escapeHtml5(unmappedIssueFields.toString()));+ }+ if (!mismatchedIssueFields.isEmpty()) {+ feedback.append("<li> YouTrack issue fields mapped to wrong type of OneDev issue field: ");+ feedback.append("<ul>");+ for (Map.Entry<String, String> entry: mismatchedIssueFields.entrySet()) {+ feedback.append("<li>")+ .append(HtmlEscape.escapeHtml5(entry.getKey()))+ .append(" : ")+ .append(HtmlEscape.escapeHtml5(entry.getValue()));+ }+ feedback.append("</ul>");+ }+ if (!unmappedIssueTags.isEmpty()) {+ feedback.append("<li> YouTrack issue tags not mapped to OneDev custom field: "+ + HtmlEscape.escapeHtml5(unmappedIssueTags.toString()));+ }+ if (!nonExistentLogins.isEmpty()) {+ feedback.append("<li> YouTrack logins without email or email can not be mapped to OneDev account: "+ + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));+ }+ if (!tooLargeAttachments.isEmpty()) {+ feedback.append("<li> Too large attachments: "+ + HtmlEscape.escapeHtml5(tooLargeAttachments.toString()));+ }++ if (hasNotes)+ feedback.append("</ul>");++ return feedback.toString();+ }++}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportUtils.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportUtils.javanew file mode 100644index 0000000000..cf2c9cb68e--- /dev/null+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportUtils.java@@ -0,0 +1,882 @@+package io.onedev.server.plugin.imports.youtrack;++import java.io.IOException;+import java.io.InputStream;+import java.net.URI;+import java.net.URISyntaxException;+import java.util.ArrayList;+import java.util.Date;+import java.util.HashMap;+import java.util.LinkedHashMap;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.concurrent.atomic.AtomicInteger;++import javax.annotation.Nullable;+import javax.ws.rs.client.Client;+import javax.ws.rs.client.ClientBuilder;+import javax.ws.rs.client.Invocation;+import javax.ws.rs.client.WebTarget;+import javax.ws.rs.core.Response;++import org.apache.commons.lang3.StringUtils;+import org.apache.http.client.utils.URIBuilder;+import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;+import org.unbescape.html.HtmlEscape;++import com.fasterxml.jackson.databind.JsonNode;++import io.onedev.commons.utils.ExplicitException;+import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.IssueManager;+import io.onedev.server.entitymanager.SettingManager;+import io.onedev.server.entitymanager.UserManager;+import io.onedev.server.model.Issue;+import io.onedev.server.model.IssueComment;+import io.onedev.server.model.IssueField;+import io.onedev.server.model.Project;+import io.onedev.server.model.User;+import io.onedev.server.model.support.LastUpdate;+import io.onedev.server.model.support.administration.GlobalIssueSetting;+import io.onedev.server.model.support.inputspec.InputSpec;+import io.onedev.server.model.support.issue.StateSpec;+import io.onedev.server.model.support.issue.field.spec.ChoiceField;+import io.onedev.server.model.support.issue.field.spec.DateField;+import io.onedev.server.model.support.issue.field.spec.DateTimeField;+import io.onedev.server.model.support.issue.field.spec.FieldSpec;+import io.onedev.server.model.support.issue.field.spec.FloatField;+import io.onedev.server.model.support.issue.field.spec.IntegerField;+import io.onedev.server.model.support.issue.field.spec.TextField;+import io.onedev.server.model.support.issue.field.spec.UserChoiceField;+import io.onedev.server.model.support.issue.field.spec.WorkingPeriodField;+import io.onedev.server.persistence.dao.Dao;+import io.onedev.server.security.SecurityUtils;+import io.onedev.server.util.DateUtils;+import io.onedev.server.util.JerseyUtils;+import io.onedev.server.util.Pair;+import io.onedev.server.util.ReferenceMigrator;+import io.onedev.server.util.SimpleLogger;++public class YouTrackImportUtils {++ static final String NAME = "YouTrack";++ static final int PER_PAGE = 100;++ static YouTrackIssueImportOption buildImportOption(YouTrackProjectImportSource importSource, @Nullable String youTrackProjectId, SimpleLogger logger) {+ YouTrackIssueImportOption importOption = new YouTrackIssueImportOption();+ Client client = newClient(importSource);+ try {+ String fields = "customFields(field(name),bundle(values(name)))";++ Set<String> youTrackIssueStates = new LinkedHashSet<>();+ Set<String> youTrackIssueFields = new LinkedHashSet<>();++ List<JsonNode> projectNodes = new ArrayList<>();++ if (youTrackProjectId != null) {+ String apiEndpoint = importSource.getApiEndpoint("/admin/projects/" + youTrackProjectId + "?fields=" + fields);+ projectNodes.add(get(client, apiEndpoint, logger));+ } else {+ String apiEndpoint = importSource.getApiEndpoint("/admin/projects?fields=" + fields);+ for (JsonNode projectNode: list(client, apiEndpoint, logger))+ projectNodes.add(projectNode);+ }++ for (JsonNode projectNode: projectNodes) {+ for (JsonNode customFieldNode: projectNode.get("customFields")) {+ String fieldName = customFieldNode.get("field").get("name").asText();+ JsonNode bundleNode = customFieldNode.get("bundle");+ if (bundleNode != null) {+ String bundleType = bundleNode.get("$type").asText();+ if (bundleType.equals("StateBundle")) {+ for (JsonNode valueNode: bundleNode.get("values"))+ youTrackIssueStates.add(valueNode.get("name").asText());+ } else if (bundleType.equals("EnumBundle") || bundleType.equals("OwnedBundle")) {+ for (JsonNode valueNode: bundleNode.get("values"))+ youTrackIssueFields.add(fieldName + "::" + valueNode.get("name").asText());+ } else {+ youTrackIssueFields.add(fieldName);+ }+ } else {+ youTrackIssueFields.add(fieldName);+ }+ }+ }++ for (String youTrackIssueState: youTrackIssueStates) {+ IssueStateMapping mapping = new IssueStateMapping();+ mapping.setYouTrackIssueState(youTrackIssueState);+ importOption.getIssueStateMappings().add(mapping);+ }++ for (String youTrackIssueField: youTrackIssueFields) {+ IssueFieldMapping mapping = new IssueFieldMapping();+ mapping.setYouTrackIssueField(youTrackIssueField);+ importOption.getIssueFieldMappings().add(mapping);+ }++ String apiEndpoint = importSource.getApiEndpoint("/issueTags?fields=name");+ for (JsonNode tagNode: list(client, apiEndpoint, logger)) {+ IssueTagMapping mapping = new IssueTagMapping();+ mapping.setYouTrackIssueTag(tagNode.get("name").asText());+ importOption.getIssueTagMappings().add(mapping);+ }++ } finally {+ client.close();+ }+ return importOption;+ }++ static YouTrackImportResult importIssues(YouTrackProjectImportSource importSource, String youTrackProjectId,+ Project oneDevProject, boolean useExistingIssueNumbers, YouTrackIssueImportOption importOption,+ boolean dryRun, SimpleLogger logger) {+ Client client = newClient(importSource);+ try {+ String apiEndpoint = importSource.getApiEndpoint("/admin/projects/" + youTrackProjectId + "?fields=shortName");+ String youTrackProjectShortName = get(client, apiEndpoint, logger).get("shortName").asText();++ Map<String, String> stateMappings = new HashMap<>();+ Map<String, Pair<FieldSpec, String>> fieldMappings = new HashMap<>();+ Map<String, Pair<FieldSpec, String>> tagMappings = new HashMap<>();++ Map<Long, Long> issueNumberMappings = new HashMap<>();++ for (IssueStateMapping mapping: importOption.getIssueStateMappings())+ stateMappings.put(mapping.getYouTrackIssueState(), mapping.getOneDevIssueState());++ for (IssueFieldMapping mapping: importOption.getIssueFieldMappings()) {+ String oneDevFieldName;+ String oneDevFieldValue;+ if (mapping.getOneDevIssueField().contains("::")) {+ oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");+ oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");+ } else {+ oneDevFieldName = mapping.getOneDevIssueField();+ oneDevFieldValue = null;+ }+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);+ if (fieldSpec == null)+ throw new ExplicitException("No field spec found: " + oneDevFieldName);+ fieldMappings.put(mapping.getYouTrackIssueField(), new Pair<>(fieldSpec, oneDevFieldValue));+ }+ for (IssueTagMapping mapping: importOption.getIssueTagMappings()) {+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);+ if (fieldSpec == null)+ throw new ExplicitException("No field spec found: " + oneDevFieldName);+ tagMappings.put(mapping.getYouTrackIssueTag(), new Pair<>(fieldSpec, oneDevFieldValue));+ }++ Set<String> nonExistentLogins = new LinkedHashSet<>();+ Set<String> unmappedIssueTags = new LinkedHashSet<>();+ Set<String> unmappedIssueFields = new LinkedHashSet<>();+ Set<String> unmappedIssueStates = new LinkedHashSet<>();+ Map<String, String> mismatchedIssueFields = new LinkedHashMap<>();+ Set<String> tooLargeAttachments = new LinkedHashSet<>();++ AtomicInteger numOfImportedIssues = new AtomicInteger(0);++ List<Issue> issues = new ArrayList<>();++ String fields = ""+ + "idReadable,"+ + "numberInProject,"+ + "summary,"+ + "description,"+ + "created,"+ + "comments(created,text,author(login,name,email),attachments(name,url,size)),"+ + "attachments(name,url,comment,size),"+ + "reporter(login,name,email),"+ + "tags(name),"+ + "customFields(name,value(name,login,email,presentation,text),projectCustomField(field(fieldType(id)))),"+ + "links(direction,linkType(sourceToTarget,targetToSource),issues(numberInProject))";+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {++ @Nullable+ private String processAttachments(String issueUUID, String readableIssueId, @Nullable String markdown,+ List<JsonNode> attachmentNodes, Set<String> tooLargeAttachments) {+ if (markdown == null)+ markdown = "";++ Map<String, String> unreferencedAttachments = new LinkedHashMap<>();++ for (JsonNode attachmentNode: attachmentNodes) {+ String attachmentName = attachmentNode.get("name").asText(null);+ String attachmentUrl = attachmentNode.get("url").asText(null);+ long attachmentSize = attachmentNode.get("size").asLong(0);+ if (attachmentSize != 0 && attachmentName != null && attachmentUrl != null) {+ if (attachmentSize > Project.MAX_ATTACHMENT_SIZE) {+ tooLargeAttachments.add(readableIssueId + ":" + attachmentName);+ } else {+ if (!attachmentUrl.startsWith("/api"))+ throw new ExplicitException("Unexpected attachment url: " + attachmentUrl);+ attachmentUrl = attachmentUrl.substring("/api".length());++ String endpoint = importSource.getApiEndpoint(attachmentUrl);+ WebTarget target = client.target(endpoint);+ Invocation.Builder builder = target.request();+ try (Response response = builder.get()) {+ String errorMessage = JerseyUtils.checkStatus(endpoint, response);+ if (errorMessage != null) {+ throw new ExplicitException(String.format(+ "Error downloading attachment (url: %s, error message: %s)",+ endpoint, errorMessage));+ }+ try (InputStream is = response.readEntity(InputStream.class)) {+ String oneDevAttachmentName = oneDevProject.saveAttachment(issueUUID, attachmentName, is);+ String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);+ if (markdown.contains("(" + attachmentName + ")")) {+ markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");+ } else {+ unreferencedAttachments.put(attachmentName, oneDevAttachmentUrl);+ }+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }+ }+ }+ }++ if (!unreferencedAttachments.isEmpty()) {+ markdown += "\n\n";+ for (Map.Entry<String, String> entry: unreferencedAttachments.entrySet())+ markdown += "[" + entry.getKey() + "](" + entry.getValue() + ") ";+ }++ if (markdown.length() == 0)+ markdown = null;++ return markdown;+ }++ private String joinAsMultilineHtml(List<String> values) {+ List<String> escapedValues = new ArrayList<>();+ for (String value: values)+ escapedValues.add(HtmlEscape.escapeHtml5(value));+ return StringUtils.join(escapedValues, "<br>");+ }++ @Nullable+ private String getEmail(JsonNode userNode) {+ JsonNode emailNode = userNode.get("email");+ if (emailNode != null)+ return emailNode.asText(null);+ else+ return null;+ }++ @Override+ public void consume(List<JsonNode> pageData) throws InterruptedException {+ for (JsonNode issueNode: pageData) {+ if (Thread.interrupted())+ throw new InterruptedException();++ Issue issue = new Issue();+ String readableId = issueNode.get("idReadable").asText();+ Long oldNumber = issueNode.get("numberInProject").asLong();+ Long newNumber;+ if (dryRun || useExistingIssueNumbers)+ newNumber = oldNumber;+ else+ newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);+ issue.setNumber(newNumber);+ issueNumberMappings.put(oldNumber, newNumber);+ issue.setTitle(issueNode.get("summary").asText());+ issue.setDescription(issueNode.get("description").asText(null));+ issue.setSubmitDate(new Date(issueNode.get("created").asLong(System.currentTimeMillis())));+ issue.setProject(oneDevProject);+ issue.setNumberScope(oneDevProject);++ if (!dryRun) {+ List<JsonNode> attachmentNodes = new ArrayList<>();+ for (JsonNode attachmentNode: issueNode.get("attachments")) {+ if (!attachmentNode.hasNonNull("comment"))+ attachmentNodes.add(attachmentNode);+ }+ issue.setDescription(processAttachments(issue.getUUID(), readableId,+ issue.getDescription(), attachmentNodes, tooLargeAttachments));+ }++ Map<String, String> extraIssueInfo = new LinkedHashMap<>();++ if (issueNode.hasNonNull("reporter")) {+ JsonNode reporterNode = issueNode.get("reporter");+ String email = getEmail(reporterNode);+ String fullName = reporterNode.get("name").asText();+ String login = reporterNode.get("login").asText();+ if (email != null) {+ User user = OneDev.getInstance(UserManager.class).findByEmail(email);+ if (user != null) {+ issue.setSubmitter(user);+ } else {+ issue.setSubmitterName(fullName);+ nonExistentLogins.add(login);+ }+ } else {+ issue.setSubmitterName(fullName);+ nonExistentLogins.add(login);+ }+ } else {+ issue.setSubmitter(SecurityUtils.getUser());+ }++ LastUpdate lastUpdate = new LastUpdate();+ lastUpdate.setActivity("Opened");+ lastUpdate.setDate(issue.getSubmitDate());+ lastUpdate.setUser(issue.getSubmitter());+ lastUpdate.setUserName(issue.getSubmitterName());+ issue.setLastUpdate(lastUpdate);++ StateSpec initialState = getIssueSetting().getInitialStateSpec();+ for (JsonNode customFieldNode: issueNode.get("customFields")) {+ String fieldName = customFieldNode.get("name").asText();+ String fieldType = customFieldNode.get("$type").asText();+ switch (fieldType) {+ case "StateIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ String state = customFieldNode.get("value").get("name").asText();+ String mappedState = stateMappings.get(state);+ if (mappedState != null)+ issue.setState(mappedState);+ else+ unmappedIssueStates.add(state);+ }+ break;+ case "SingleBuildIssueCustomField":+ case "SingleGroupIssueCustomField":+ case "SingleVersionIssueCustomField":+ String fieldValue = customFieldNode.get("value").asText(null);+ if (fieldValue != null) {+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof TextField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.TEXT);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ }+ break;+ case "MultiBuildIssueCustomField":+ case "MultiGroupIssueCustomField":+ case "MultiVersionIssueCustomField":+ List<String> fieldValues = new ArrayList<>();+ for (JsonNode valueNode: customFieldNode.get("value"))+ fieldValues.add(valueNode.get("name").asText());+ if (!fieldValues.isEmpty()) {+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));+ } else if (!(mapped.getFirst() instanceof TextField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.TEXT);+ issueField.setValue(StringUtils.join(fieldValues, ", "));+ issue.getFields().add(issueField);+ }+ }+ break;+ case "SingleEnumIssueCustomField":+ case "SingleOwnedIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ fieldValue = customFieldNode.get("value").get("name").asText();+ String fieldNameWithValue = fieldName + "::" + fieldValue;+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);+ if (mapped == null) {+ unmappedIssueFields.add(fieldNameWithValue);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof ChoiceField) || mapped.getFirst().isAllowMultiple()) {+ mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a single-valued enum field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.ENUMERATION);+ issueField.setValue(mapped.getSecond());+ issue.getFields().add(issueField);+ }+ }+ break;+ case "MultiEnumIssueCustomField":+ case "MultiOwnedIssueCustomField":+ fieldValues = new ArrayList<>();+ for (JsonNode valueNode: customFieldNode.get("value"))+ fieldValues.add(valueNode.get("name").asText());++ boolean hasErrors = false;+ List<IssueField> issueFields = new ArrayList<>();+ for (String each: fieldValues) {+ String fieldNameWithValue = fieldName + "::" + each;+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);+ if (mapped == null) {+ unmappedIssueFields.add(fieldNameWithValue);+ hasErrors = true;+ } else if (!(mapped.getFirst() instanceof ChoiceField) || !mapped.getFirst().isAllowMultiple()) {+ mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a multi-valued enum field");+ hasErrors = true;+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.ENUMERATION);+ issueField.setValue(mapped.getSecond());+ issueFields.add(issueField);+ }+ }+ if (hasErrors)+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));+ else+ issue.getFields().addAll(issueFields);+ break;+ case "SingleUserIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ JsonNode valueNode = customFieldNode.get("value");++ String login = valueNode.get("login").asText();+ String fullName = valueNode.get("name").asText();+ String email = getEmail(valueNode);++ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fullName);+ } else if (!(mapped.getFirst() instanceof UserChoiceField) || mapped.getFirst().isAllowMultiple()) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a single-valued user field");+ extraIssueInfo.put(fieldName, fullName);+ } else {+ if (email != null) {+ User user = OneDev.getInstance(UserManager.class).findByEmail(email);+ if (user != null) {+ fieldValue = user.getName();+ } else {+ fieldValue = fullName;+ nonExistentLogins.add(login);+ }+ } else {+ fieldValue = fullName;+ nonExistentLogins.add(login);+ }++ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.USER);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ }+ break;+ case "MultiUserIssueCustomField":+ List<String> fullNames = new ArrayList<>();+ for (JsonNode valueNode: customFieldNode.get("value"))+ fullNames.add(valueNode.get("name").asText());+ if (!fullNames.isEmpty()) {+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));+ } else if (!(mapped.getFirst() instanceof UserChoiceField) || !mapped.getFirst().isAllowMultiple()) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-valued user field");+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));+ } else {+ for (JsonNode valueNode: customFieldNode.get("value")) {+ String login = valueNode.get("login").asText();+ String fullName = valueNode.get("name").asText();+ String email = getEmail(valueNode);++ if (email != null) {+ User user = OneDev.getInstance(UserManager.class).findByEmail(email);+ if (user != null) {+ fieldValue = user.getName();+ } else {+ fieldValue = fullName;+ nonExistentLogins.add(login);+ }+ } else {+ fieldValue = fullName;+ nonExistentLogins.add(login);+ }++ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.USER);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ }+ }+ break;+ case "DateIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ Date date = new Date(customFieldNode.get("value").asLong());+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, DateUtils.formatDate(date));+ } else if (!(mapped.getFirst() instanceof DateField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a date field");+ extraIssueInfo.put(fieldName, DateUtils.formatDate(date));+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.DATE);+ issueField.setValue(DateUtils.formatDate(date));+ issue.getFields().add(issueField);+ }+ }+ break;+ case "SimpleIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ JsonNode projectCustomField = customFieldNode.get("projectCustomField");+ JsonNode fieldNode = projectCustomField.get("field");+ if (fieldNode != null) {+ switch (fieldNode.get("fieldType").get("id").asText()) {+ case "date":+ Date date = new Date(customFieldNode.get("value").asLong());+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));+ } else if (!(mapped.getFirst() instanceof DateTimeField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a datetime field");+ extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.DATE_TIME);+ issueField.setValue(DateUtils.formatDateTime(date));+ issue.getFields().add(issueField);+ }+ break;+ case "float":+ fieldValue = customFieldNode.get("value").asText();+ mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof FloatField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a float field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.FLOAT);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ break;+ case "integer":+ fieldValue = customFieldNode.get("value").asText();+ mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof IntegerField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to an integer field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.INTEGER);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ break;+ case "string":+ fieldValue = customFieldNode.get("value").asText();+ mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof TextField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.TEXT);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ break;+ }+ }+ }+ break;+ case "TextIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ fieldValue = customFieldNode.get("value").get("text").asText();+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof TextField)+ || !((TextField)mapped.getFirst()).isMultiline()) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-line text field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.TEXT);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ }+ break;+ case "PeriodIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ fieldValue = customFieldNode.get("value").get("presentation").asText();+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof WorkingPeriodField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a working period field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.WORKING_PERIOD);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ }+ break;+ }+ }+ if (issue.getState() == null)+ issue.setState(initialState.getName());++ for (JsonNode linkNode: issueNode.get("links")) {+ List<String> linkedIssueLinks = new ArrayList<>();+ for (JsonNode linkedIssueNode: linkNode.get("issues"))+ linkedIssueLinks.add(youTrackProjectShortName + "-" + linkedIssueNode.get("numberInProject").asLong());++ if (!linkedIssueLinks.isEmpty() && linkNode.hasNonNull("linkType")) {+ JsonNode linkTypeNode = linkNode.get("linkType");+ switch (linkNode.get("direction").asText()) {+ case "BOTH":+ case "OUTWARD":+ String linkName = linkTypeNode.get("sourceToTarget").asText(null);+ if (linkName != null)+ extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));+ break;+ case "INWARD":+ linkName = linkTypeNode.get("targetToSource").asText(null);+ if (linkName != null)+ extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));+ break;+ }+ }+ }++ List<String> currentUnmappedTags = new ArrayList<>();+ for (JsonNode tagNode: issueNode.get("tags")) {+ String tagName = tagNode.get("name").asText();+ Pair<FieldSpec, String> mapped = tagMappings.get(tagName);+ if (mapped != null) {+ IssueField tagField = new IssueField();+ tagField.setIssue(issue);+ tagField.setName(mapped.getFirst().getName());+ tagField.setType(InputSpec.ENUMERATION);+ tagField.setValue(mapped.getSecond());+ tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));+ issue.getFields().add(tagField);+ } else {+ currentUnmappedTags.add(HtmlEscape.escapeHtml5(tagName));+ unmappedIssueTags.add(HtmlEscape.escapeHtml5(tagName));+ }+ }+ if (!currentUnmappedTags.isEmpty())+ extraIssueInfo.put("Tags", joinAsMultilineHtml(currentUnmappedTags));++ for (JsonNode commentNode: issueNode.get("comments")) {+ String commentContent = commentNode.get("text").asText(null);+ if (commentContent != null || !commentNode.get("attachments").isEmpty()) {+ IssueComment comment = new IssueComment();+ comment.setIssue(issue);+ if (!dryRun) {+ List<JsonNode> attachmentNodes = new ArrayList<>();+ for (JsonNode attachmentNode: commentNode.get("attachments"))+ attachmentNodes.add(attachmentNode);+ comment.setContent(processAttachments(issue.getUUID(), readableId,+ commentContent, attachmentNodes, tooLargeAttachments));+ }+ comment.setDate(new Date(commentNode.get("created").asLong(System.currentTimeMillis())));+ if (commentNode.hasNonNull("author")) {+ JsonNode authorNode = commentNode.get("author");+ String email = getEmail(authorNode);+ String fullName = authorNode.get("name").asText();+ String login = authorNode.get("login").asText();+ if (email != null) {+ User user = OneDev.getInstance(UserManager.class).findByEmail(email);+ if (user != null) {+ comment.setUser(user);+ } else {+ comment.setUserName(fullName);+ nonExistentLogins.add(login);+ }+ } else {+ comment.setUserName(fullName);+ nonExistentLogins.add(login);+ }+ } else {+ issue.setSubmitter(SecurityUtils.getUser());+ }+ issue.getComments().add(comment);+ }+ }++ issue.setCommentCount(issue.getComments().size());++ if (!extraIssueInfo.isEmpty()) {+ StringBuilder builder = new StringBuilder("|");+ for (String key: extraIssueInfo.keySet())+ builder.append(key).append("|");+ builder.append("\n|");+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));+ builder.append("\n|");+ for (String value: extraIssueInfo.values())+ builder.append(value).append("|");++ if (issue.getDescription() != null)+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());+ else+ issue.setDescription(builder.toString());+ }+ issues.add(issue);+ }+ logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");+ }++ };++ apiEndpoint = importSource.getApiEndpoint("/admin/projects/" + youTrackProjectId + "/issues?fields=" + fields);+ list(client, apiEndpoint, pageDataConsumer, logger);++ if (!dryRun) {+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);+ for (Issue issue: issues) {+ if (issue.getDescription() != null)+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), youTrackProjectShortName + "-"));++ OneDev.getInstance(IssueManager.class).save(issue);+ for (IssueField field: issue.getFields())+ OneDev.getInstance(Dao.class).persist(field);+ for (IssueComment comment: issue.getComments()) {+ comment.setContent(migrator.migratePrefixed(comment.getContent(), youTrackProjectShortName + "-"));+ OneDev.getInstance(Dao.class).persist(comment);+ }+ }+ }++ YouTrackImportResult result = new YouTrackImportResult();+ result.mismatchedIssueFields.putAll(mismatchedIssueFields);+ result.nonExistentLogins.addAll(nonExistentLogins);+ result.tooLargeAttachments.addAll(tooLargeAttachments);+ result.unmappedIssueFields.addAll(unmappedIssueFields);+ result.unmappedIssueStates.addAll(unmappedIssueStates);+ result.unmappedIssueTags.addAll(unmappedIssueTags);++ return result;+ } finally {+ client.close();+ }+ }++ static GlobalIssueSetting getIssueSetting() {+ return OneDev.getInstance(SettingManager.class).getIssueSetting();+ }++ static List<JsonNode> list(Client client, String apiEndpoint, SimpleLogger logger) {+ List<JsonNode> result = new ArrayList<>();+ list(client, apiEndpoint, new PageDataConsumer() {++ @Override+ public void consume(List<JsonNode> pageData) {+ result.addAll(pageData);+ }++ }, logger);+ return result;+ }++ static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer, SimpleLogger logger) {+ URI uri;+ try {+ uri = new URIBuilder(apiEndpoint)+ .addParameter("$top", String.valueOf(PER_PAGE)).build();+ } catch (URISyntaxException e) {+ throw new RuntimeException(e);+ }++ int page = 0;+ while (true) {+ try {+ URIBuilder builder = new URIBuilder(uri);+ builder.addParameter("$skip", String.valueOf(page*PER_PAGE));+ List<JsonNode> pageData = new ArrayList<>();+ for (JsonNode each: get(client, builder.build().toString(), logger))+ pageData.add(each);+ pageDataConsumer.consume(pageData);+ if (pageData.size() < PER_PAGE)+ break;+ page++;+ } catch (URISyntaxException|InterruptedException e) {+ throw new RuntimeException(e);+ }+ }+ }++ static JsonNode get(Client client, String apiEndpoint, SimpleLogger logger) {+ WebTarget target = client.target(apiEndpoint);+ Invocation.Builder builder = target.request();+ try (Response response = builder.get()) {+ String errorMessage = JerseyUtils.checkStatus(apiEndpoint, response);+ if (errorMessage != null)+ throw new ExplicitException(errorMessage);+ else+ return response.readEntity(JsonNode.class);+ }+ }++ static Client newClient(YouTrackProjectImportSource importSource) {+ Client client = ClientBuilder.newClient();+ client.register(HttpAuthenticationFeature.basic(importSource.getUserName(), importSource.getPassword()));+ return client;+ }++ private static interface PageDataConsumer {++ void consume(List<JsonNode> pageData) throws InterruptedException;++ }++}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.javadeleted file mode 100644index 9f3237b82d..0000000000--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.java+++ /dev/null@@ -1,938 +0,0 @@-package io.onedev.server.plugin.imports.youtrack;--import java.io.IOException;-import java.io.InputStream;-import java.net.URI;-import java.net.URISyntaxException;-import java.util.ArrayList;-import java.util.Collection;-import java.util.Date;-import java.util.HashMap;-import java.util.LinkedHashMap;-import java.util.LinkedHashSet;-import java.util.List;-import java.util.Map;-import java.util.Set;-import java.util.concurrent.atomic.AtomicInteger;--import javax.annotation.Nullable;-import javax.ws.rs.client.Client;-import javax.ws.rs.client.ClientBuilder;-import javax.ws.rs.client.Invocation;-import javax.ws.rs.client.WebTarget;-import javax.ws.rs.core.Response;--import org.apache.commons.lang3.StringUtils;-import org.apache.http.client.utils.URIBuilder;-import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;-import org.unbescape.html.HtmlEscape;--import com.fasterxml.jackson.databind.JsonNode;--import io.onedev.commons.utils.ExceptionUtils;-import io.onedev.commons.utils.ExplicitException;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.IssueManager;-import io.onedev.server.entitymanager.ProjectManager;-import io.onedev.server.entitymanager.SettingManager;-import io.onedev.server.entitymanager.UserManager;-import io.onedev.server.model.Issue;-import io.onedev.server.model.IssueComment;-import io.onedev.server.model.IssueField;-import io.onedev.server.model.Project;-import io.onedev.server.model.User;-import io.onedev.server.model.support.LastUpdate;-import io.onedev.server.model.support.administration.GlobalIssueSetting;-import io.onedev.server.model.support.inputspec.InputSpec;-import io.onedev.server.model.support.issue.StateSpec;-import io.onedev.server.model.support.issue.field.spec.ChoiceField;-import io.onedev.server.model.support.issue.field.spec.DateField;-import io.onedev.server.model.support.issue.field.spec.DateTimeField;-import io.onedev.server.model.support.issue.field.spec.FieldSpec;-import io.onedev.server.model.support.issue.field.spec.FloatField;-import io.onedev.server.model.support.issue.field.spec.IntegerField;-import io.onedev.server.model.support.issue.field.spec.TextField;-import io.onedev.server.model.support.issue.field.spec.UserChoiceField;-import io.onedev.server.model.support.issue.field.spec.WorkingPeriodField;-import io.onedev.server.persistence.dao.Dao;-import io.onedev.server.security.SecurityUtils;-import io.onedev.server.storage.StorageManager;-import io.onedev.server.util.DateUtils;-import io.onedev.server.util.JerseyUtils;-import io.onedev.server.util.Pair;-import io.onedev.server.util.SimpleLogger;-import io.onedev.server.web.page.project.imports.ProjectImporter;--public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouTrackImportOption> {-- private static final long serialVersionUID = 1L;-- public static final String NAME = "YouTrack";-- private static final int PER_PAGE = 100;-- @Override- public String getName() {- return NAME;- }-- @Override- public YouTrackImportOption getImportOption(YouTrackImportSource importSource, SimpleLogger logger) {- YouTrackImportOption importOption = new YouTrackImportOption();- if (importSource.isPrepopulateImportOptions()) {- Client client = newClient(importSource);- try {- String fields = "name,customFields(field(name),bundle(values(name)))";-- Set<String> youTrackIssueStates = new LinkedHashSet<>();- Set<String> youTrackIssueFields = new LinkedHashSet<>();-- for (JsonNode projectNode: list(client, importSource, "/admin/projects?fields=" + fields, logger)) {- YouTrackImport aImport = new YouTrackImport();- aImport.setYouTrackProject(projectNode.get("name").asText());- aImport.setOneDevProject(aImport.getYouTrackProject().replace(' ', '-'));- importOption.getImports().add(aImport);-- for (JsonNode customFieldNode: projectNode.get("customFields")) {- String fieldName = customFieldNode.get("field").get("name").asText();- JsonNode bundleNode = customFieldNode.get("bundle");- if (bundleNode != null) {- String bundleType = bundleNode.get("$type").asText();- if (bundleType.equals("StateBundle")) {- for (JsonNode valueNode: bundleNode.get("values"))- youTrackIssueStates.add(valueNode.get("name").asText());- } else if (bundleType.equals("EnumBundle") || bundleType.equals("OwnedBundle")) {- for (JsonNode valueNode: bundleNode.get("values"))- youTrackIssueFields.add(fieldName + "::" + valueNode.get("name").asText());- } else {- youTrackIssueFields.add(fieldName);- }- } else {- youTrackIssueFields.add(fieldName);- }- }- }-- for (String youTrackIssueState: youTrackIssueStates) {- IssueStateMapping mapping = new IssueStateMapping();- mapping.setYouTrackIssueState(youTrackIssueState);- importOption.getIssueStateMappings().add(mapping);- }-- for (String youTrackIssueField: youTrackIssueFields) {- IssueFieldMapping mapping = new IssueFieldMapping();- mapping.setYouTrackIssueField(youTrackIssueField);- importOption.getIssueFieldMappings().add(mapping);- }-- for (JsonNode tagNode: list(client, importSource, "/issueTags?fields=name", logger)) {- IssueTagMapping mapping = new IssueTagMapping();- mapping.setYouTrackIssueTag(tagNode.get("name").asText());- importOption.getIssueTagMappings().add(mapping);- }-- } finally {- client.close();- }- }- return importOption;- }-- @Override- public String doImport(YouTrackImportSource importSource, YouTrackImportOption importOption, boolean dryRun,- SimpleLogger logger) {- Collection<Long> projectIds = new ArrayList<>();- Client client = newClient(importSource);- try {- Map<String, String> stateMappings = new HashMap<>();- Map<String, Pair<FieldSpec, String>> fieldMappings = new HashMap<>();- Map<String, Pair<FieldSpec, String>> tagMappings = new HashMap<>();-- for (IssueStateMapping mapping: importOption.getIssueStateMappings())- stateMappings.put(mapping.getYouTrackIssueState(), mapping.getOneDevIssueState());-- for (IssueFieldMapping mapping: importOption.getIssueFieldMappings()) {- String oneDevFieldName;- String oneDevFieldValue;- if (mapping.getOneDevIssueField().contains("::")) {- oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");- oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");- } else {- oneDevFieldName = mapping.getOneDevIssueField();- oneDevFieldValue = null;- }- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);- if (fieldSpec == null)- throw new ExplicitException("No field spec found: " + oneDevFieldName);- fieldMappings.put(mapping.getYouTrackIssueField(), new Pair<>(fieldSpec, oneDevFieldValue));- }- for (IssueTagMapping mapping: importOption.getIssueTagMappings()) {- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);- if (fieldSpec == null)- throw new ExplicitException("No field spec found: " + oneDevFieldName);- tagMappings.put(mapping.getYouTrackIssueTag(), new Pair<>(fieldSpec, oneDevFieldValue));- }-- Map<String, String> youTrackProjectIds = new HashMap<>();- Map<String, String> projectDescriptions = new HashMap<>();- for (JsonNode projectNode: list(client, importSource, "/admin/projects?fields=id,name,description", logger)) {- String projectName = projectNode.get("name").asText();- youTrackProjectIds.put(projectName, projectNode.get("id").asText());- projectDescriptions.put(projectName, projectNode.get("description").asText(null));- }-- Set<String> nonExistentLogins = new LinkedHashSet<>();- Set<String> unmappedIssueTags = new LinkedHashSet<>();- Set<String> unmappedIssueFields = new LinkedHashSet<>();- Set<String> unmappedIssueStates = new LinkedHashSet<>();- Map<String, String> mismatchedIssueFields = new LinkedHashMap<>();- Set<String> tooLargeAttachments = new LinkedHashSet<>();-- for (YouTrackImport aImport: importOption.getImports()) {- logger.log("Importing project '" + aImport.getYouTrackProject() + "'...");-- String youTrackProjectId = youTrackProjectIds.get(aImport.getYouTrackProject());- if (youTrackProjectId == null)- throw new ExplicitException("Unable to find YouTrack project: " + aImport.getYouTrackProject());-- Project project = new Project();- project.setName(aImport.getOneDevProject());- project.setDescription(projectDescriptions.get(aImport.getYouTrackProject()));- project.setIssueManagementEnabled(true);-- if (!dryRun) {- OneDev.getInstance(ProjectManager.class).create(project);- projectIds.add(project.getId());- }-- AtomicInteger numOfImportedIssues = new AtomicInteger(0);-- String fields = ""- + "idReadable,"- + "numberInProject,"- + "summary,"- + "description,"- + "created,"- + "comments(created,text,author(login,name,email),attachments(name,url,size)),"- + "attachments(name,url,comment,size),"- + "reporter(login,name,email),"- + "tags(name),"- + "customFields(name,value(name,login,email,presentation,text),projectCustomField(field(fieldType(id)))),"- + "links(direction,linkType(sourceToTarget,targetToSource),issues(numberInProject))";- PageDataConsumer pageDataConsumer = new PageDataConsumer() {-- @Nullable- private String processAttachments(String issueUUID, String readableIssueId, @Nullable String markdown,- List<JsonNode> attachmentNodes, Set<String> tooLargeAttachments) {- if (markdown == null)- markdown = "";-- Map<String, String> unreferencedAttachments = new LinkedHashMap<>();-- for (JsonNode attachmentNode: attachmentNodes) {- String attachmentName = attachmentNode.get("name").asText(null);- String attachmentUrl = attachmentNode.get("url").asText(null);- long attachmentSize = attachmentNode.get("size").asLong(0);- if (attachmentSize != 0 && attachmentName != null && attachmentUrl != null) {- if (attachmentSize > Project.MAX_ATTACHMENT_SIZE) {- tooLargeAttachments.add(readableIssueId + ":" + attachmentName);- } else {- if (!attachmentUrl.startsWith("/api"))- throw new ExplicitException("Unexpected attachment url: " + attachmentUrl);- attachmentUrl = attachmentUrl.substring("/api".length());-- String endpoint = importSource.getApiEndpoint(attachmentUrl);- WebTarget target = client.target(endpoint);- Invocation.Builder builder = target.request();- try (Response response = builder.get()) {- String errorMessage = JerseyUtils.checkStatus(response);- if (errorMessage != null) {- throw new ExplicitException(String.format(- "Error downloading attachment (url: %s, error message: %s)",- endpoint, errorMessage));- }- try (InputStream is = response.readEntity(InputStream.class)) {- String oneDevAttachmentName = project.saveAttachment(issueUUID, attachmentName, is);- String oneDevAttachmentUrl = project.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);- if (markdown.contains("(" + attachmentName + ")")) {- markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");- } else {- unreferencedAttachments.put(attachmentName, oneDevAttachmentUrl);- }- } catch (IOException e) {- throw new RuntimeException(e);- }- }- }- }- }-- if (!unreferencedAttachments.isEmpty()) {- markdown += "\n\n";- for (Map.Entry<String, String> entry: unreferencedAttachments.entrySet())- markdown += "[" + entry.getKey() + "](" + entry.getValue() + ") ";- }-- if (markdown.length() == 0)- markdown = null;-- return markdown;- }-- private String joinAsMultilineHtml(List<String> values) {- List<String> escapedValues = new ArrayList<>();- for (String value: values)- escapedValues.add(HtmlEscape.escapeHtml5(value));- return StringUtils.join(escapedValues, "<br>");- }-- @Nullable- private String getEmail(JsonNode userNode) {- JsonNode emailNode = userNode.get("email");- if (emailNode != null)- return emailNode.asText(null);- else- return null;- }-- @Override- public void consume(List<JsonNode> pageData) throws InterruptedException {- for (JsonNode issueNode: pageData) {- if (Thread.interrupted())- throw new InterruptedException();-- Issue issue = new Issue();- String readableId = issueNode.get("idReadable").asText();- issue.setNumber(issueNode.get("numberInProject").asLong());- issue.setTitle(issueNode.get("summary").asText());- issue.setDescription(issueNode.get("description").asText(null));- issue.setSubmitDate(new Date(issueNode.get("created").asLong(System.currentTimeMillis())));- issue.setProject(project);- issue.setNumberScope(project);-- if (!dryRun) {- List<JsonNode> attachmentNodes = new ArrayList<>();- for (JsonNode attachmentNode: issueNode.get("attachments")) {- if (!attachmentNode.hasNonNull("comment"))- attachmentNodes.add(attachmentNode);- }- issue.setDescription(processAttachments(issue.getUUID(), readableId,- issue.getDescription(), attachmentNodes, tooLargeAttachments));- }-- Map<String, String> extraIssueInfo = new LinkedHashMap<>();-- if (issueNode.hasNonNull("reporter")) {- JsonNode reporterNode = issueNode.get("reporter");- String email = getEmail(reporterNode);- String fullName = reporterNode.get("name").asText();- String login = reporterNode.get("login").asText();- if (email != null) {- User user = OneDev.getInstance(UserManager.class).findByEmail(email);- if (user != null) {- issue.setSubmitter(user);- } else {- issue.setSubmitterName(fullName);- nonExistentLogins.add(login);- }- } else {- issue.setSubmitterName(fullName);- nonExistentLogins.add(login);- }- } else {- issue.setSubmitter(SecurityUtils.getUser());- }-- LastUpdate lastUpdate = new LastUpdate();- lastUpdate.setActivity("Opened");- lastUpdate.setDate(issue.getSubmitDate());- lastUpdate.setUser(issue.getSubmitter());- lastUpdate.setUserName(issue.getSubmitterName());- issue.setLastUpdate(lastUpdate);-- StateSpec initialState = getIssueSetting().getInitialStateSpec();- for (JsonNode customFieldNode: issueNode.get("customFields")) {- String fieldName = customFieldNode.get("name").asText();- String fieldType = customFieldNode.get("$type").asText();- switch (fieldType) {- case "StateIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- String state = customFieldNode.get("value").get("name").asText();- String mappedState = stateMappings.get(state);- if (mappedState != null)- issue.setState(mappedState);- else- unmappedIssueStates.add(state);- }- break;- case "SingleBuildIssueCustomField":- case "SingleGroupIssueCustomField":- case "SingleVersionIssueCustomField":- String fieldValue = customFieldNode.get("value").asText(null);- if (fieldValue != null) {- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof TextField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.TEXT);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- }- break;- case "MultiBuildIssueCustomField":- case "MultiGroupIssueCustomField":- case "MultiVersionIssueCustomField":- List<String> fieldValues = new ArrayList<>();- for (JsonNode valueNode: customFieldNode.get("value"))- fieldValues.add(valueNode.get("name").asText());- if (!fieldValues.isEmpty()) {- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));- } else if (!(mapped.getFirst() instanceof TextField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.TEXT);- issueField.setValue(StringUtils.join(fieldValues, ", "));- issue.getFields().add(issueField);- }- }- break;- case "SingleEnumIssueCustomField":- case "SingleOwnedIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- fieldValue = customFieldNode.get("value").get("name").asText();- String fieldNameWithValue = fieldName + "::" + fieldValue;- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);- if (mapped == null) {- unmappedIssueFields.add(fieldNameWithValue);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof ChoiceField) || mapped.getFirst().isAllowMultiple()) {- mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a single-valued enum field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.ENUMERATION);- issueField.setValue(mapped.getSecond());- issue.getFields().add(issueField);- }- }- break;- case "MultiEnumIssueCustomField":- case "MultiOwnedIssueCustomField":- fieldValues = new ArrayList<>();- for (JsonNode valueNode: customFieldNode.get("value"))- fieldValues.add(valueNode.get("name").asText());-- boolean hasErrors = false;- List<IssueField> issueFields = new ArrayList<>();- for (String each: fieldValues) {- String fieldNameWithValue = fieldName + "::" + each;- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);- if (mapped == null) {- unmappedIssueFields.add(fieldNameWithValue);- hasErrors = true;- } else if (!(mapped.getFirst() instanceof ChoiceField) || !mapped.getFirst().isAllowMultiple()) {- mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a multi-valued enum field");- hasErrors = true;- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.ENUMERATION);- issueField.setValue(mapped.getSecond());- issueFields.add(issueField);- }- }- if (hasErrors)- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));- else- issue.getFields().addAll(issueFields);- break;- case "SingleUserIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- JsonNode valueNode = customFieldNode.get("value");-- String login = valueNode.get("login").asText();- String fullName = valueNode.get("name").asText();- String email = getEmail(valueNode);-- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fullName);- } else if (!(mapped.getFirst() instanceof UserChoiceField) || mapped.getFirst().isAllowMultiple()) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a single-valued user field");- extraIssueInfo.put(fieldName, fullName);- } else {- if (email != null) {- User user = OneDev.getInstance(UserManager.class).findByEmail(email);- if (user != null) {- fieldValue = user.getName();- } else {- fieldValue = fullName;- nonExistentLogins.add(login);- }- } else {- fieldValue = fullName;- nonExistentLogins.add(login);- }-- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.USER);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- }- break;- case "MultiUserIssueCustomField":- List<String> fullNames = new ArrayList<>();- for (JsonNode valueNode: customFieldNode.get("value"))- fullNames.add(valueNode.get("name").asText());- if (!fullNames.isEmpty()) {- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));- } else if (!(mapped.getFirst() instanceof UserChoiceField) || !mapped.getFirst().isAllowMultiple()) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-valued user field");- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));- } else {- for (JsonNode valueNode: customFieldNode.get("value")) {- String login = valueNode.get("login").asText();- String fullName = valueNode.get("name").asText();- String email = getEmail(valueNode);-- if (email != null) {- User user = OneDev.getInstance(UserManager.class).findByEmail(email);- if (user != null) {- fieldValue = user.getName();- } else {- fieldValue = fullName;- nonExistentLogins.add(login);- }- } else {- fieldValue = fullName;- nonExistentLogins.add(login);- }-- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.USER);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- }- }- break;- case "DateIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- Date date = new Date(customFieldNode.get("value").asLong());- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, DateUtils.formatDate(date));- } else if (!(mapped.getFirst() instanceof DateField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a date field");- extraIssueInfo.put(fieldName, DateUtils.formatDate(date));- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.DATE);- issueField.setValue(DateUtils.formatDate(date));- issue.getFields().add(issueField);- }- }- break;- case "SimpleIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- JsonNode projectCustomField = customFieldNode.get("projectCustomField");- JsonNode fieldNode = projectCustomField.get("field");- if (fieldNode != null) {- switch (fieldNode.get("fieldType").get("id").asText()) {- case "date":- Date date = new Date(customFieldNode.get("value").asLong());- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));- } else if (!(mapped.getFirst() instanceof DateTimeField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a datetime field");- extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.DATE_TIME);- issueField.setValue(DateUtils.formatDateTime(date));- issue.getFields().add(issueField);- }- break;- case "float":- fieldValue = customFieldNode.get("value").asText();- mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof FloatField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a float field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.FLOAT);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- break;- case "integer":- fieldValue = customFieldNode.get("value").asText();- mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof IntegerField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to an integer field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.INTEGER);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- break;- case "string":- fieldValue = customFieldNode.get("value").asText();- mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof TextField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.TEXT);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- break;- }- }- }- break;- case "TextIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- fieldValue = customFieldNode.get("value").get("text").asText();- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof TextField)- || !((TextField)mapped.getFirst()).isMultiline()) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-line text field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.TEXT);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- }- break;- case "PeriodIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- fieldValue = customFieldNode.get("value").get("presentation").asText();- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof WorkingPeriodField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a working period field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.WORKING_PERIOD);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- }- break;- }- }- if (issue.getState() == null)- issue.setState(initialState.getName());-- for (JsonNode linkNode: issueNode.get("links")) {- List<String> linkedIssueLinks = new ArrayList<>();- for (JsonNode linkedIssueNode: linkNode.get("issues"))- linkedIssueLinks.add("issue #" + linkedIssueNode.get("numberInProject").asLong());-- if (!linkedIssueLinks.isEmpty() && linkNode.hasNonNull("linkType")) {- JsonNode linkTypeNode = linkNode.get("linkType");- switch (linkNode.get("direction").asText()) {- case "BOTH":- case "OUTWARD":- String linkName = linkTypeNode.get("sourceToTarget").asText(null);- if (linkName != null)- extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));- break;- case "INWARD":- linkName = linkTypeNode.get("targetToSource").asText(null);- if (linkName != null)- extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));- break;- }- }- }-- List<String> currentUnmappedTags = new ArrayList<>();- for (JsonNode tagNode: issueNode.get("tags")) {- String tagName = tagNode.get("name").asText();- Pair<FieldSpec, String> mapped = tagMappings.get(tagName);- if (mapped != null) {- IssueField tagField = new IssueField();- tagField.setIssue(issue);- tagField.setName(mapped.getFirst().getName());- tagField.setType(InputSpec.ENUMERATION);- tagField.setValue(mapped.getSecond());- tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));- issue.getFields().add(tagField);- } else {- currentUnmappedTags.add(HtmlEscape.escapeHtml5(tagName));- unmappedIssueTags.add(HtmlEscape.escapeHtml5(tagName));- }- }- if (!currentUnmappedTags.isEmpty())- extraIssueInfo.put("Tags", joinAsMultilineHtml(currentUnmappedTags));-- for (JsonNode commentNode: issueNode.get("comments")) {- String commentContent = commentNode.get("text").asText(null);- if (commentContent != null || !commentNode.get("attachments").isEmpty()) {- IssueComment comment = new IssueComment();- comment.setIssue(issue);- if (!dryRun) {- List<JsonNode> attachmentNodes = new ArrayList<>();- for (JsonNode attachmentNode: commentNode.get("attachments"))- attachmentNodes.add(attachmentNode);- comment.setContent(processAttachments(issue.getUUID(), readableId,- commentContent, attachmentNodes, tooLargeAttachments));- }- comment.setDate(new Date(commentNode.get("created").asLong(System.currentTimeMillis())));- if (commentNode.hasNonNull("author")) {- JsonNode authorNode = commentNode.get("author");- String email = getEmail(authorNode);- String fullName = authorNode.get("name").asText();- String login = authorNode.get("login").asText();- if (email != null) {- User user = OneDev.getInstance(UserManager.class).findByEmail(email);- if (user != null) {- comment.setUser(user);- } else {- comment.setUserName(fullName);- nonExistentLogins.add(login);- }- } else {- comment.setUserName(fullName);- nonExistentLogins.add(login);- }- } else {- issue.setSubmitter(SecurityUtils.getUser());- }- issue.getComments().add(comment);- }- }-- issue.setCommentCount(issue.getComments().size());-- if (!extraIssueInfo.isEmpty()) {- StringBuilder builder = new StringBuilder("|");- for (String key: extraIssueInfo.keySet())- builder.append(key).append("|");- builder.append("\n|");- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));- builder.append("\n|");- for (String value: extraIssueInfo.values())- builder.append(value).append("|");-- if (issue.getDescription() != null)- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());- else- issue.setDescription(builder.toString());- }-- if (!dryRun) {- OneDev.getInstance(IssueManager.class).save(issue);- for (IssueField field: issue.getFields())- OneDev.getInstance(Dao.class).persist(field);- for (IssueComment comment: issue.getComments())- OneDev.getInstance(Dao.class).persist(comment);- }-- }- logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");- }-- };- list(client, importSource, "/admin/projects/" + youTrackProjectId + "/issues?fields=" + fields,- pageDataConsumer, logger);- }- StringBuilder feedback = new StringBuilder("Projects imported successfully");-- boolean hasNotes =- !unmappedIssueStates.isEmpty()- || !unmappedIssueFields.isEmpty()- || !mismatchedIssueFields.isEmpty()- || !unmappedIssueTags.isEmpty()- || !nonExistentLogins.isEmpty()- || !tooLargeAttachments.isEmpty();-- if (hasNotes)- feedback.append("<br><br><b>NOTE:</b><ul>");-- if (!unmappedIssueStates.isEmpty()) {- feedback.append("<li> Unmapped YouTrack issue states (using OneDev initial state): "- + HtmlEscape.escapeHtml5(unmappedIssueStates.toString()));- }- if (!unmappedIssueFields.isEmpty()) {- feedback.append("<li> Unmapped YouTrack issue fields (mentioned as extra info in issue description): "- + HtmlEscape.escapeHtml5(unmappedIssueFields.toString()));- }- if (!mismatchedIssueFields.isEmpty()) {- feedback.append("<li> YouTrack issue fields mapped to wrong type of OneDev issue field (mentioned as extra info in issue description):");- feedback.append("<ul>");- for (Map.Entry<String, String> entry: mismatchedIssueFields.entrySet()) {- feedback.append("<li>")- .append(HtmlEscape.escapeHtml5(entry.getKey()))- .append(" : ")- .append(HtmlEscape.escapeHtml5(entry.getValue()));- }- feedback.append("</ul>");- }- if (!unmappedIssueTags.isEmpty()) {- feedback.append("<li> YouTrack issue tags not mapped to OneDev custom field (mentioned as extra info in issue description): "- + HtmlEscape.escapeHtml5(unmappedIssueTags.toString()));- }- if (!nonExistentLogins.isEmpty()) {- feedback.append("<li> YouTrack logins without email or email can not be mapped to OneDev account: "- + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));- }- if (!tooLargeAttachments.isEmpty()) {- feedback.append("<li> Too large attachments: "- + HtmlEscape.escapeHtml5(tooLargeAttachments.toString()));- }-- if (hasNotes)- feedback.append("</ul>");-- return feedback.toString();- } catch (Exception e) {- for (Long projectId: projectIds)- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);- throw ExceptionUtils.unchecked(e);- } finally {- client.close();- }- }-- private GlobalIssueSetting getIssueSetting() {- return OneDev.getInstance(SettingManager.class).getIssueSetting();- }-- private List<JsonNode> list(Client client, YouTrackImportSource importSource,- String apiPath, SimpleLogger logger) {- List<JsonNode> result = new ArrayList<>();- list(client, importSource, apiPath, new PageDataConsumer() {-- @Override- public void consume(List<JsonNode> pageData) {- result.addAll(pageData);- }-- }, logger);- return result;- }-- private void list(Client client, YouTrackImportSource importSource, String apiPath,- PageDataConsumer pageDataConsumer, SimpleLogger logger) {- URI uri;- try {- uri = new URIBuilder(importSource.getApiEndpoint(apiPath))- .addParameter("$top", String.valueOf(PER_PAGE)).build();- } catch (URISyntaxException e) {- throw new RuntimeException(e);- }-- int page = 0;- while (true) {- try {- URIBuilder builder = new URIBuilder(uri);- builder.addParameter("$skip", String.valueOf(page*PER_PAGE));- List<JsonNode> pageData = new ArrayList<>();- for (JsonNode each: get(client, builder.build().toString(), logger))- pageData.add(each);- pageDataConsumer.consume(pageData);- if (pageData.size() < PER_PAGE)- break;- page++;- } catch (URISyntaxException|InterruptedException e) {- throw new RuntimeException(e);- }- }- }-- private JsonNode get(Client client, String apiEndpoint, SimpleLogger logger) {- WebTarget target = client.target(apiEndpoint);- Invocation.Builder builder = target.request();- try (Response response = builder.get()) {- String errorMessage = JerseyUtils.checkStatus(response);- if (errorMessage != null)- throw new ExplicitException(errorMessage);- else- return response.readEntity(JsonNode.class);- }- }-- private Client newClient(YouTrackImportSource importSource) {- Client client = ClientBuilder.newClient();- client.register(HttpAuthenticationFeature.basic(importSource.getUserName(), importSource.getPassword()));- return client;- }-- private static interface PageDataConsumer {-- void consume(List<JsonNode> pageData) throws InterruptedException;-- }--}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportOption.javasimilarity index 51%rename from server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.javarename to server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportOption.javaindex b03e6e16a9..327e3a5c76 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportOption.java@@ -4,39 +4,19 @@ import java.io.Serializable;import java.util.ArrayList;import java.util.List;-import javax.validation.ConstraintValidatorContext;-import javax.validation.constraints.Size;--import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.ProjectManager;-import io.onedev.server.util.validation.Validatable;-import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;@Editable-@ClassValidating-public class YouTrackImportOption implements Serializable, Validatable {+public class YouTrackIssueImportOption implements Serializable {private static final long serialVersionUID = 1L;- private List<YouTrackImport> imports = new ArrayList<>();-private List<IssueStateMapping> issueStateMappings = new ArrayList<>();private List<IssueFieldMapping> issueFieldMappings = new ArrayList<>();private List<IssueTagMapping> issueTagMappings = new ArrayList<>();- @Editable(order=100, name="Projects to Import")- @Size(min=1, max=10000, message="No projects to import")- public List<YouTrackImport> getImports() {- return imports;- }-- public void setImports(List<YouTrackImport> imports) {- this.imports = imports;- }-@Editable(order=200, description="Specify how to map YouTrack issue state to OneDev issue state. "+ "Unmapped states will use the initial state in OneDev")public List<IssueStateMapping> getIssueStateMappings() {@@ -70,34 +50,4 @@ public class YouTrackImportOption implements Serializable, Validatable {this.issueTagMappings = issueTagMappings;}- @Override- public boolean isValid(ConstraintValidatorContext context) {- boolean isValid = true;- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);- for (int i=0; i<imports.size(); i++) {- if (projectManager.find(imports.get(i).getOneDevProject()) != null) {- context.buildConstraintViolationWithTemplate("Project name already used")- .addPropertyNode("imports")- .addPropertyNode(YouTrackImport.PROP_ONEDEV_PROJECT)- .inIterable().atIndex(i).addConstraintViolation();- isValid = false;- } else {- for (int j=0; j<imports.size(); j++) {- if (j != i && imports.get(j).getOneDevProject().equals(imports.get(i).getOneDevProject())) {- context.buildConstraintViolationWithTemplate("Duplicate project name")- .addPropertyNode("imports")- .addPropertyNode(YouTrackImport.PROP_ONEDEV_PROJECT)- .inIterable().atIndex(i).addConstraintViolation();- isValid = false;- break;- }- }- }- }-- if (!isValid)- context.disableDefaultConstraintViolation();- return isValid;- }-}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportSource.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportSource.javanew file mode 100644index 0000000000..caf0896744--- /dev/null+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImportSource.java@@ -0,0 +1,85 @@+package io.onedev.server.plugin.imports.youtrack;++import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.get;+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.list;++import java.util.ArrayList;+import java.util.Collections;+import java.util.List;++import javax.ws.rs.client.Client;+import javax.ws.rs.client.ClientBuilder;++import org.apache.commons.lang3.StringUtils;+import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;+import org.hibernate.validator.constraints.NotEmpty;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import com.fasterxml.jackson.databind.JsonNode;++import io.onedev.server.util.EditContext;+import io.onedev.server.util.SimpleLogger;+import io.onedev.server.web.editable.annotation.ChoiceProvider;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public class YouTrackIssueImportSource extends YouTrackProjectImportSource {++ private static final long serialVersionUID = 1L;++ private static final Logger logger = LoggerFactory.getLogger(YouTrackIssueImportSource.class);++ private String project;++ @Editable(order=400, name="YouTrack Project", description="Choose YouTrack project to "+ + "import issues from. Available projects will be populated when above options "+ + "are specified")+ @ChoiceProvider("getProjectChoices")+ @NotEmpty+ public String getProject() {+ return project;+ }++ public void setProject(String project) {+ this.project = project;+ }++ @SuppressWarnings("unused")+ private static List<String> getProjectChoices() {+ List<String> choices = new ArrayList<>();++ String apiUrl = (String) EditContext.get().getInputValue(PROP_API_URL);+ String userName = (String) EditContext.get().getInputValue(PROP_USER_NAME);+ String password = (String) EditContext.get().getInputValue(PROP_PASSWORD);++ if (StringUtils.isNotBlank(apiUrl) && StringUtils.isNotBlank(userName)+ && StringUtils.isNotBlank(password)) {+ Client client = ClientBuilder.newClient();+ try {+ client.register(HttpAuthenticationFeature.basic(userName, password));+ SimpleLogger logger = new SimpleLogger() {++ @Override+ public void log(String message) {+ YouTrackIssueImportSource.logger.info(message);+ }++ };+ String apiEndpoint = getApiEndpoint(apiUrl, "/users/me?fields=guest");+ if (!get(client, apiEndpoint, logger).get("guest").asBoolean()) {+ apiEndpoint = getApiEndpoint(apiUrl, "/admin/projects?fields=name");+ for (JsonNode projectNode: list(client, apiEndpoint, logger))+ choices.add(projectNode.get("name").asText());+ }+ } catch (Exception e) {+ } finally {+ client.close();+ }+ }++ Collections.sort(choices);+ return choices;+ }++}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.javanew file mode 100644index 0000000000..cc7bbdc6df--- /dev/null+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java@@ -0,0 +1,57 @@+package io.onedev.server.plugin.imports.youtrack;++import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.*;++import javax.ws.rs.client.Client;++import com.fasterxml.jackson.databind.JsonNode;++import io.onedev.commons.utils.ExplicitException;+import io.onedev.server.model.Project;+import io.onedev.server.util.SimpleLogger;+import io.onedev.server.web.page.project.issues.imports.IssueImporter;++public class YouTrackIssueImporter extends IssueImporter<YouTrackIssueImportSource, YouTrackIssueImportOption> {++ private static final long serialVersionUID = 1L;++ @Override+ public String getName() {+ return NAME;+ }++ private String getYouTrackProjectId(YouTrackProjectImportSource importSource,+ String youTrackProjectName, SimpleLogger logger) {+ Client client = newClient(importSource);+ try {+ String apiEndpoint = importSource.getApiEndpoint("/admin/projects?fields=id,name");+ for (JsonNode projectNode: list(client, apiEndpoint, logger)) {+ if (youTrackProjectName.equals(projectNode.get("name").asText()))+ return projectNode.get("id").asText();+ }+ throw new ExplicitException("Unable to find YouTrack project: " + youTrackProjectName);+ } finally {+ client.close();+ }+ }++ @Override+ public YouTrackIssueImportOption getImportOption(YouTrackIssueImportSource importSource, SimpleLogger logger) {+ if (importSource.isPrepopulateImportOptions()) {+ String youTrackProjectId = getYouTrackProjectId(importSource, importSource.getProject(), logger);+ return buildImportOption(importSource, youTrackProjectId, logger);+ } else {+ return new YouTrackIssueImportOption();+ }+ }++ @Override+ public String doImport(Project project, YouTrackIssueImportSource importSource,+ YouTrackIssueImportOption importOption, boolean dryRun, SimpleLogger logger) {+ logger.log("Importing issues from '" + importSource.getProject() + "'...");+ String youTrackProjectId = getYouTrackProjectId(importSource, importSource.getProject(), logger);+ return importIssues(importSource, youTrackProjectId, project, false, importOption, dryRun, logger)+ .toHtml("Issues imported successfully");+ }++}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.javaindex ce5c2ef6fd..4f7170b203 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java@@ -8,6 +8,8 @@ import com.google.common.collect.Lists;import io.onedev.commons.launcher.loader.AbstractPluginModule;import io.onedev.server.web.page.project.imports.ProjectImporter;import io.onedev.server.web.page.project.imports.ProjectImporterContribution;+import io.onedev.server.web.page.project.issues.imports.IssueImporter;+import io.onedev.server.web.page.project.issues.imports.IssueImporterContribution;/*** NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.@@ -24,7 +26,16 @@ public class YouTrackPluginModule extends AbstractPluginModule {@Overridepublic Collection<ProjectImporter<? extends Serializable, ? extends Serializable>> getImporters() {- return Lists.newArrayList(new YouTrackImporter());+ return Lists.newArrayList(new YouTrackProjectImporter());+ }++ });++ contribute(IssueImporterContribution.class, new IssueImporterContribution() {++ @Override+ public Collection<IssueImporter<? extends Serializable, ? extends Serializable>> getImporters() {+ return Lists.newArrayList(new YouTrackIssueImporter());}});diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportOption.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportOption.javanew file mode 100644index 0000000000..14e4397376--- /dev/null+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportOption.java@@ -0,0 +1,63 @@+package io.onedev.server.plugin.imports.youtrack;++import java.util.ArrayList;+import java.util.List;++import javax.validation.ConstraintValidatorContext;+import javax.validation.constraints.Size;++import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.util.validation.Validatable;+import io.onedev.server.util.validation.annotation.ClassValidating;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+@ClassValidating+public class YouTrackProjectImportOption extends YouTrackIssueImportOption implements Validatable {++ private static final long serialVersionUID = 1L;++ private List<ProjectMapping> projecMappings = new ArrayList<>();++ @Editable(order=100, name="Projects to Import")+ @Size(min=1, max=10000, message="No projects to import")+ public List<ProjectMapping> getProjectMappings() {+ return projecMappings;+ }++ public void setProjectMappings(List<ProjectMapping> projectMappings) {+ this.projecMappings = projectMappings;+ }++ @Override+ public boolean isValid(ConstraintValidatorContext context) {+ boolean isValid = true;+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);+ for (int i=0; i<projecMappings.size(); i++) {+ if (projectManager.find(projecMappings.get(i).getOneDevProject()) != null) {+ context.buildConstraintViolationWithTemplate("Project name already used")+ .addPropertyNode("imports")+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)+ .inIterable().atIndex(i).addConstraintViolation();+ isValid = false;+ } else {+ for (int j=0; j<projecMappings.size(); j++) {+ if (j != i && projecMappings.get(j).getOneDevProject().equals(projecMappings.get(i).getOneDevProject())) {+ context.buildConstraintViolationWithTemplate("Duplicate project name")+ .addPropertyNode("imports")+ .addPropertyNode(ProjectMapping.PROP_ONEDEV_PROJECT)+ .inIterable().atIndex(i).addConstraintViolation();+ isValid = false;+ break;+ }+ }+ }+ }++ if (!isValid)+ context.disableDefaultConstraintViolation();+ return isValid;+ }++}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportSource.javasimilarity index 83%rename from server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.javarename to server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportSource.javaindex e7d052383e..8722357070 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImportSource.java@@ -23,15 +23,15 @@ import io.onedev.server.web.editable.annotation.Password;@Editable@ClassValidating-public class YouTrackImportSource implements Serializable, Validatable {+public class YouTrackProjectImportSource implements Serializable, Validatable {private static final long serialVersionUID = 1L;- private static final String PROP_API_URL = "apiUrl";+ protected static final String PROP_API_URL = "apiUrl";- private static final String PROP_USER_NAME = "userName";+ protected static final String PROP_USER_NAME = "userName";- private static final String PROP_PASSWORD = "password";+ protected static final String PROP_PASSWORD = "password";private String apiUrl;@@ -77,7 +77,7 @@ public class YouTrackImportSource implements Serializable, Validatable {this.password = password;}- @Editable(order=400, description="If checked, import options will be pre-populated based on all accessible "+ @Editable(order=1000, description="If checked, import options will be pre-populated based on all accessible "+ "projects and its settings")public boolean isPrepopulateImportOptions() {return prepopulateImportOptions;@@ -87,19 +87,24 @@ public class YouTrackImportSource implements Serializable, Validatable {this.prepopulateImportOptions = prepopulateImportOptions;}- public String getApiEndpoint(String apiPath) {+ static String getApiEndpoint(String apiUrl, String apiPath) {return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");}+ public String getApiEndpoint(String apiPath) {+ return getApiEndpoint(apiUrl, apiPath);+ }+@Overridepublic boolean isValid(ConstraintValidatorContext context) {Client client = ClientBuilder.newClient();- client.register(HttpAuthenticationFeature.basic(getUserName(), getPassword()));try {- WebTarget target = client.target(getApiEndpoint("/users/me?fields=guest"));+ client.register(HttpAuthenticationFeature.basic(getUserName(), getPassword()));+ String apiEndpoint = getApiEndpoint("/users/me?fields=guest");+ WebTarget target = client.target(apiEndpoint);Invocation.Builder builder = target.request();try (Response response = builder.get()) {- String errorMessage = JerseyUtils.checkStatus(response);+ String errorMessage = JerseyUtils.checkStatus(apiEndpoint, response);if (errorMessage != null) {context.disableDefaultConstraintViolation();context.buildConstraintViolationWithTemplate(errorMessage)diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.javanew file mode 100644index 0000000000..963c0f5e14--- /dev/null+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java@@ -0,0 +1,116 @@+package io.onedev.server.plugin.imports.youtrack;++import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.NAME;+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.buildImportOption;+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.importIssues;+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.list;+import static io.onedev.server.plugin.imports.youtrack.YouTrackImportUtils.newClient;++import java.util.ArrayList;+import java.util.Collection;+import java.util.HashMap;+import java.util.Map;++import javax.ws.rs.client.Client;++import com.fasterxml.jackson.databind.JsonNode;++import io.onedev.commons.utils.ExceptionUtils;+import io.onedev.commons.utils.ExplicitException;+import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.model.Project;+import io.onedev.server.storage.StorageManager;+import io.onedev.server.util.SimpleLogger;+import io.onedev.server.web.page.project.imports.ProjectImporter;++public class YouTrackProjectImporter extends ProjectImporter<YouTrackProjectImportSource, YouTrackProjectImportOption> {++ private static final long serialVersionUID = 1L;++ @Override+ public String getName() {+ return NAME;+ }++ @Override+ public YouTrackProjectImportOption getImportOption(YouTrackProjectImportSource importSource, SimpleLogger logger) {+ YouTrackProjectImportOption importOption = new YouTrackProjectImportOption();+ if (importSource.isPrepopulateImportOptions()) {+ Client client = newClient(importSource);+ try {+ String apiEndpoint = importSource.getApiEndpoint("/admin/projects?fields=name");+ for (JsonNode projectNode: list(client, apiEndpoint, logger)) {+ ProjectMapping projectMapping = new ProjectMapping();+ projectMapping.setYouTrackProject(projectNode.get("name").asText());+ projectMapping.setOneDevProject(projectMapping.getYouTrackProject().replace(' ', '-'));+ importOption.getProjectMappings().add(projectMapping);+ }+ YouTrackIssueImportOption issueImportOption = buildImportOption(importSource, null, logger);+ importOption.setIssueFieldMappings(issueImportOption.getIssueFieldMappings());+ importOption.setIssueStateMappings(issueImportOption.getIssueStateMappings());+ importOption.setIssueTagMappings(issueImportOption.getIssueTagMappings());+ } finally {+ client.close();+ }+ }+ return importOption;+ }++ @Override+ public String doImport(YouTrackProjectImportSource importSource, YouTrackProjectImportOption importOption,+ boolean dryRun, SimpleLogger logger) {++ Map<String, String> youTrackProjectIds = new HashMap<>();+ Map<String, String> youTrackProjectDescriptions = new HashMap<>();+ Collection<Long> projectIds = new ArrayList<>();++ Client client = newClient(importSource);+ try {+ String apiEndpoint = importSource.getApiEndpoint("/admin/projects?fields=id,name,description");+ for (JsonNode projectNode: list(client, apiEndpoint, logger)) {+ String projectName = projectNode.get("name").asText();+ youTrackProjectIds.put(projectName, projectNode.get("id").asText());+ youTrackProjectDescriptions.put(projectName, projectNode.get("description").asText(null));+ }++ YouTrackImportResult result = new YouTrackImportResult();++ for (ProjectMapping projectMapping: importOption.getProjectMappings()) {+ logger.log("Importing project '" + projectMapping.getYouTrackProject() + "'...");++ String youTrackProjectId = youTrackProjectIds.get(projectMapping.getYouTrackProject());+ if (youTrackProjectId == null)+ throw new ExplicitException("Unable to find YouTrack project: " + projectMapping.getYouTrackProject());++ Project project = new Project();+ project.setName(projectMapping.getOneDevProject());+ project.setDescription(youTrackProjectDescriptions.get(projectMapping.getYouTrackProject()));+ project.setIssueManagementEnabled(true);++ if (!dryRun) {+ OneDev.getInstance(ProjectManager.class).create(project);+ projectIds.add(project.getId());+ }++ YouTrackImportResult currentResult = importIssues(importSource, youTrackProjectId, project,+ true, importOption, dryRun, logger);+ result.mismatchedIssueFields.putAll(currentResult.mismatchedIssueFields);+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);+ result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);+ result.unmappedIssueFields.addAll(currentResult.unmappedIssueFields);+ result.unmappedIssueStates.addAll(currentResult.unmappedIssueStates);+ result.unmappedIssueTags.addAll(currentResult.unmappedIssueTags);+ }++ return result.toHtml("Projects imported successfully");+ } catch (Exception e) {+ for (Long projectId: projectIds)+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);+ throw ExceptionUtils.unchecked(e);+ } finally {+ client.close();+ }+ }++}\ No newline at end of file